dev: refactor items and update repo deploy (#278)

moonD4rk · web-flow · commit eed1d12282a8 · 2024-01-13T15:58:53.000+08:00
* refactor: Refactor file paths and use map to store item names - Refactored file paths for various browsing data types to use a consistent method of generating temporary file names - Modified parsing functions in many browsing data types to use the new temporary file naming scheme - Renamed `FileName` to `Filename` for consistency in the `item` package - Removed unnecessary code and comments throughout various files - Made minor improvements to some Item methods such as `TempFilename` and `FileName` * ci: Optimize GitHub actions and update Go versions. (#274) * ci: Optimize GitHub actions and update Go versions. - Add GitHub actions for unit testing, linting, building, and releasing - Use a matrix strategy to test on different versions and platforms - Update setup-go and go-version options for compatibility - Format code and build zip files for different operating systems - Upload releases to GitHub with specific asset names and types * fix: Refactor item file naming convention - Update `filename()` function to return `UnsupportedItem` in specific cases - Replace `UnknownItem` with `UnsupportedItem` in `Filename` method of `Item` struct - Refactor code for clarity and consistency * ci: Update GitHub workflow with latest setup-go version - Update setup-go action to v3 in lint.yml GitHub workflow - Omits some big changes in file diff summary - Improve overall workflow reliability and efficiency * ci: Improve GitHub actions across platforms - Improve Windows compatibility in build workflow - Optimize unit testing for pull requests - Upgrade Coveralls GitHub action to v2 for improved coverage tracking * build: Optimize build process for consistency and efficiency - Ensure consistency of line endings by disabling Git's automatic conversion - Add format check for Windows systems - Update Go version in strategy matrix to `1.21.x` - Remove unused dependencies from build process - Include all packages in repository in build command * ci: Refactor GitHub workflow configuration - Remove unnecessary checks for `windows-latest` in github workflow - Change `gofmt` check to `diff` for formatting - Remove unneeded Git configuration for encoding of line endings - Close #273 * ci: Update default branch references in GitHub Actions workflows (#277) - Update Github Actions workflows to use `main` branch instead of `master`. - Rename `master` branch to `main` in `lint.yml` and `build.yml` files. - Change default branch to `main` in `contributors.yml` workflow file.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -1,34 +1,42 @@
 name: build
+
 on:
   push:
     branches:
-      - master
+      - main
   pull_request:
+
 jobs:
   build:
-    name: Build
-    runs-on: ubuntu-latest
+    name: Build on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        goVer: ["1.21"]
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        goVer: ["1.21.x"]
 
     steps:
-    - name: Set up Go ${{ matrix.goVer }}
-      uses: actions/setup-go@v1
-      with:
-        go-version: ${{ matrix.goVer }}
-      id: go
-
-    - name: Check out code into the Go module directory
-      uses: actions/checkout@v2
-
-    - name: Get dependencies
-      run: |
-        go get -v -t -d ./...
-        go get gopkg.in/check.v1
-        
-    - name: Format
-      run: diff -u <(echo -n) <(gofmt -d .)    
-      
-    - name: Build
-      run: go build -v ./...
+      - name: Set up Go ${{ matrix.goVer }}
+        uses: actions/setup-go@v1
+        with:
+          go-version: ${{ matrix.goVer }}
+        id: go
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Set Git to handle line endings consistently
+        run: git config --global core.autocrlf false
+
+      - name: Format Check
+        if: matrix.os != 'windows-latest'
+        run: |
+          diff -u <(echo -n) <(gofmt -d .)
+
+      - name: Get dependencies
+        run: |
+          go get -v -t -d ./...
+          go get gopkg.in/check.v1
+
+      - name: Build
+        run: go build -v ./...
diff --git a/.github/workflows/contributors.yml b/.github/workflows/contributors.yml
@@ -4,7 +4,7 @@ on:
     - cron: '0 1 * * 0' # At 01:00 on Sunday.
   push:
     branches:
-      - master
+      - main
   workflow_dispatch:
     inputs:
       logLevel:
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
@@ -2,9 +2,8 @@ name: Lint
 on:
   push:
     branches:
-      - master
+      - main
   pull_request:
-  
 jobs:
   lint:
     name: Lint
@@ -13,7 +12,7 @@ jobs:
       - name: Set Golang
         uses: actions/setup-go@v3
         with:
-          go-version: "1.21"
+          go-version: "1.21.x"
       - name: Checkout code
         uses: actions/checkout@v3
       - name: Lint
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -13,7 +13,7 @@ jobs:
       - name: Use Golang
         uses: actions/setup-go@v3
         with:
-          go-version: "1.21"
+          go-version: "1.21.x"
 
       - name: Build with xgo
         uses: crazy-max/ghaction-xgo@v2
diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml
@@ -0,0 +1,43 @@
+on:
+  [pull_request]
+
+name: run tests
+jobs:
+  test:
+    strategy:
+      matrix:
+        go-version: [ "1.21.x" ]
+        platform: [ubuntu-latest]
+    runs-on: ${{ matrix.platform }}
+    steps:
+      - name: Install Go
+        if: success()
+        uses: actions/setup-go@v3
+        with:
+          go-version: ${{ matrix.go-version }}
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Run tests
+        run: go test -v ./... -covermode=count
+
+  coverage:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install Go
+        if: success()
+        uses: actions/setup-go@v3
+        with:
+          go-version: "1.21.x"
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Calc coverage
+        run: |
+          go test -v ./... -covermode=count -coverprofile=coverage.out
+
+      - name: Convert coverage.out to coverage.lcov
+        uses: jandelgado/gcov2lcov-action@v1
+      - name: Coveralls
+        uses: coverallsapp/github-action@v2
+        with:
+          github-token: ${{ secrets.github_token }}
+          path-to-lcov: coverage.lcov
diff --git a/.gitignore b/.gitignore
@@ -194,3 +194,6 @@ hack-browser-data
 !/browsingdata/history
 !/browsingdata/history/history.go
 !/browsingdata/history/history_test.go
+
+# github action
+!/.github/workflows/unittest.yml
diff --git a/browser/chromium/chromium.go b/browser/chromium/chromium.go
@@ -76,7 +76,7 @@ func (c *Chromium) BrowsingData(isFullExport bool) (*browsingdata.Data, error) {
 
 func (c *Chromium) copyItemToLocal() error {
 	for i, path := range c.itemPaths {
-		filename := i.String()
+		filename := i.TempFilename()
 		var err error
 		switch {
 		case fileutil.IsDirExists(path):
@@ -109,7 +109,7 @@ func (c *Chromium) userItemPaths(profilePath string, items []item.Item) (map[str
 	var dir string
 	for userDir, v := range multiItemPaths {
 		for _, p := range v {
-			if strings.HasSuffix(p, item.ChromiumKey.FileName()) {
+			if strings.HasSuffix(p, item.ChromiumKey.Filename()) {
 				keyPath = p
 				dir = userDir
 				break
@@ -132,7 +132,7 @@ func (c *Chromium) userItemPaths(profilePath string, items []item.Item) (map[str
 func chromiumWalkFunc(items []item.Item, multiItemPaths map[string]map[item.Item]string) filepath.WalkFunc {
 	return func(path string, info fs.FileInfo, err error) error {
 		for _, v := range items {
-			if info.Name() == v.FileName() {
+			if info.Name() == v.Filename() {
 				if strings.Contains(path, "System Profile") {
 					continue
 				}
@@ -153,7 +153,7 @@ func chromiumWalkFunc(items []item.Item, multiItemPaths map[string]map[item.Item
 
 func fillLocalStoragePath(itemPaths map[item.Item]string, storage item.Item) {
 	if p, ok := itemPaths[item.ChromiumHistory]; ok {
-		lsp := filepath.Join(filepath.Dir(p), storage.FileName())
+		lsp := filepath.Join(filepath.Dir(p), storage.Filename())
 		if fileutil.IsDirExists(lsp) {
 			itemPaths[item.ChromiumLocalStorage] = lsp
 		}
diff --git a/browser/chromium/chromium_darwin.go b/browser/chromium/chromium_darwin.go
@@ -24,7 +24,7 @@ var (
 
 func (c *Chromium) GetMasterKey() ([]byte, error) {
 	// don't need chromium key file for macOS
-	defer os.Remove(item.TempChromiumKey)
+	defer os.Remove(item.ChromiumKey.TempFilename())
 	// Get the master key from the keychain
 	// $ security find-generic-password -wa 'Chrome'
 	var (
diff --git a/browser/chromium/chromium_linux.go b/browser/chromium/chromium_linux.go
@@ -18,7 +18,7 @@ import (
 func (c *Chromium) GetMasterKey() ([]byte, error) {
 	// what is d-bus @https://dbus.freedesktop.org/
 	// don't need chromium key file for Linux
-	defer os.Remove(item.TempChromiumKey)
+	defer os.Remove(item.ChromiumKey.TempFilename())
 
 	conn, err := dbus.SessionBus()
 	if err != nil {
diff --git a/browser/chromium/chromium_windows.go b/browser/chromium/chromium_windows.go
@@ -18,11 +18,11 @@ import (
 var errDecodeMasterKeyFailed = errors.New("decode master key failed")
 
 func (c *Chromium) GetMasterKey() ([]byte, error) {
-	b, err := fileutil.ReadFile(item.TempChromiumKey)
+	b, err := fileutil.ReadFile(item.ChromiumKey.TempFilename())
 	if err != nil {
 		return nil, err
 	}
-	defer os.Remove(item.TempChromiumKey)
+	defer os.Remove(item.ChromiumKey.TempFilename())
 
 	encryptedKey := gjson.Get(b, "os_crypt.encrypted_key")
 	if !encryptedKey.Exists() {
diff --git a/browser/firefox/firefox.go b/browser/firefox/firefox.go
@@ -43,7 +43,7 @@ func New(profilePath string, items []item.Item) ([]*Firefox, error) {
 
 func (f *Firefox) copyItemToLocal() error {
 	for i, path := range f.itemPaths {
-		filename := i.String()
+		filename := i.TempFilename()
 		if err := fileutil.CopyFile(path, filename); err != nil {
 			return err
 		}
@@ -54,7 +54,7 @@ func (f *Firefox) copyItemToLocal() error {
 func firefoxWalkFunc(items []item.Item, multiItemPaths map[string]map[item.Item]string) fs.WalkDirFunc {
 	return func(path string, info fs.DirEntry, err error) error {
 		for _, v := range items {
-			if info.Name() == v.FileName() {
+			if info.Name() == v.Filename() {
 				parentBaseDir := fileutil.ParentBaseDir(path)
 				if _, exist := multiItemPaths[parentBaseDir]; exist {
 					multiItemPaths[parentBaseDir][v] = path
diff --git a/browsingdata/bookmark/bookmark.go b/browsingdata/bookmark/bookmark.go
@@ -27,11 +27,11 @@ type bookmark struct {
 }
 
 func (c *ChromiumBookmark) Parse(_ []byte) error {
-	bookmarks, err := fileutil.ReadFile(item.TempChromiumBookmark)
+	bookmarks, err := fileutil.ReadFile(item.ChromiumBookmark.TempFilename())
 	if err != nil {
 		return err
 	}
-	defer os.Remove(item.TempChromiumBookmark)
+	defer os.Remove(item.ChromiumBookmark.TempFilename())
 	r := gjson.Parse(bookmarks)
 	if r.Exists() {
 		roots := r.Get("roots")
@@ -94,11 +94,11 @@ const (
 )
 
 func (f *FirefoxBookmark) Parse(_ []byte) error {
-	db, err := sql.Open("sqlite3", item.TempFirefoxBookmark)
+	db, err := sql.Open("sqlite3", item.FirefoxBookmark.TempFilename())
 	if err != nil {
 		return err
 	}
-	defer os.Remove(item.TempFirefoxBookmark)
+	defer os.Remove(item.FirefoxBookmark.TempFilename())
 	defer db.Close()
 	_, err = db.Exec(closeJournalMode)
 	if err != nil {
diff --git a/browsingdata/cookie/cookie.go b/browsingdata/cookie/cookie.go
@@ -36,11 +36,11 @@ const (
 )
 
 func (c *ChromiumCookie) Parse(masterKey []byte) error {
-	db, err := sql.Open("sqlite3", item.TempChromiumCookie)
+	db, err := sql.Open("sqlite3", item.ChromiumCookie.TempFilename())
 	if err != nil {
 		return err
 	}
-	defer os.Remove(item.TempChromiumCookie)
+	defer os.Remove(item.ChromiumCookie.TempFilename())
 	defer db.Close()
 	rows, err := db.Query(queryChromiumCookie)
 	if err != nil {
@@ -104,11 +104,11 @@ const (
 )
 
 func (f *FirefoxCookie) Parse(_ []byte) error {
-	db, err := sql.Open("sqlite3", item.TempFirefoxCookie)
+	db, err := sql.Open("sqlite3", item.FirefoxCookie.TempFilename())
 	if err != nil {
 		return err
 	}
-	defer os.Remove(item.TempFirefoxCookie)
+	defer os.Remove(item.FirefoxCookie.TempFilename())
 	defer db.Close()
 
 	rows, err := db.Query(queryFirefoxCookie)
diff --git a/browsingdata/creditcard/creditcard.go b/browsingdata/creditcard/creditcard.go
@@ -29,11 +29,11 @@ const (
 )
 
 func (c *ChromiumCreditCard) Parse(masterKey []byte) error {
-	db, err := sql.Open("sqlite3", item.TempChromiumCreditCard)
+	db, err := sql.Open("sqlite3", item.ChromiumCreditCard.TempFilename())
 	if err != nil {
 		return err
 	}
-	defer os.Remove(item.TempChromiumCreditCard)
+	defer os.Remove(item.ChromiumCreditCard.TempFilename())
 	defer db.Close()
 
 	rows, err := db.Query(queryChromiumCredit)
@@ -85,11 +85,11 @@ func (c *ChromiumCreditCard) Len() int {
 type YandexCreditCard []card
 
 func (c *YandexCreditCard) Parse(masterKey []byte) error {
-	db, err := sql.Open("sqlite3", item.TempYandexCreditCard)
+	db, err := sql.Open("sqlite3", item.YandexCreditCard.TempFilename())
 	if err != nil {
 		return err
 	}
-	defer os.Remove(item.TempYandexCreditCard)
+	defer os.Remove(item.YandexCreditCard.TempFilename())
 	defer db.Close()
 	rows, err := db.Query(queryChromiumCredit)
 	if err != nil {
diff --git a/browsingdata/download/download.go b/browsingdata/download/download.go
@@ -32,11 +32,11 @@ const (
 )
 
 func (c *ChromiumDownload) Parse(_ []byte) error {
-	db, err := sql.Open("sqlite3", item.TempChromiumDownload)
+	db, err := sql.Open("sqlite3", item.ChromiumDownload.TempFilename())
 	if err != nil {
 		return err
 	}
-	defer os.Remove(item.TempChromiumDownload)
+	defer os.Remove(item.ChromiumDownload.TempFilename())
 	defer db.Close()
 	rows, err := db.Query(queryChromiumDownload)
 	if err != nil {
@@ -83,11 +83,11 @@ const (
 )
 
 func (f *FirefoxDownload) Parse(_ []byte) error {
-	db, err := sql.Open("sqlite3", item.TempFirefoxDownload)
+	db, err := sql.Open("sqlite3", item.FirefoxDownload.TempFilename())
 	if err != nil {
 		return err
 	}
-	defer os.Remove(item.TempFirefoxDownload)
+	defer os.Remove(item.FirefoxDownload.TempFilename())
 	defer db.Close()
 
 	_, err = db.Exec(closeJournalMode)
diff --git a/browsingdata/extension/extension.go b/browsingdata/extension/extension.go
@@ -25,11 +25,11 @@ type extension struct {
 }
 
 func (c *ChromiumExtension) Parse(_ []byte) error {
-	extensionFile, err := fileutil.ReadFile(item.TempChromiumExtension)
+	extensionFile, err := fileutil.ReadFile(item.ChromiumExtension.TempFilename())
 	if err != nil {
 		return err
 	}
-	defer os.Remove(item.TempChromiumExtension)
+	defer os.Remove(item.ChromiumExtension.TempFilename())
 
 	result, err := parseChromiumExtensions(extensionFile)
 	if err != nil {
@@ -114,11 +114,11 @@ type FirefoxExtension []*extension
 var lang = language.Und
 
 func (f *FirefoxExtension) Parse(_ []byte) error {
-	s, err := fileutil.ReadFile(item.TempFirefoxExtension)
+	s, err := fileutil.ReadFile(item.FirefoxExtension.TempFilename())
 	if err != nil {
 		return err
 	}
-	_ = os.Remove(item.TempFirefoxExtension)
+	_ = os.Remove(item.FirefoxExtension.TempFilename())
 	j := gjson.Parse(s)
 	for _, v := range j.Get("addons").Array() {
 		// https://searchfox.org/mozilla-central/source/toolkit/mozapps/extensions/internal/XPIDatabase.jsm#157
diff --git a/browsingdata/history/history.go b/browsingdata/history/history.go
diff --git a/browsingdata/localstorage/localstorage.go b/browsingdata/localstorage/localstorage.go
diff --git a/browsingdata/password/password.go b/browsingdata/password/password.go
diff --git a/browsingdata/sessionstorage/sessionstorage.go b/browsingdata/sessionstorage/sessionstorage.go
diff --git a/item/filename.go b/item/filename.go
diff --git a/item/item.go b/item/item.go
diff --git a/item/item_test.go b/item/item_test.go
