package Filesystem import ( "context" "fmt" "os" "path/filepath" "runtime" "sync" "github.com/vbauerster/mpb" bolt "go.etcd.io/bbolt" "golang.org/x/sync/semaphore" "PackageManager/Client/Database" "PackageManager/Client/ProgressBar" "PackageManager/Color" "PackageManager/Variables" ) var ( fsStatusWG sync.WaitGroup FsWalkWG sync.WaitGroup ) type FilesystemStatus struct { NewFiles []string PickedFiles []string ModifiedFiles []string MissingFiles []string } func ShowFilesystemDiff(root string) error { var ( fsStatus FilesystemStatus picksBucket *bolt.Bucket pickedFiles []string e error ) fsStatus, e = GetFilesystemDiff(root) if e != nil { return e } e = Database.FsDB.View(func(tx *bolt.Tx) error { picksBucket = tx.Bucket(Variables.FsHashPicksBucket) return picksBucket.ForEach(func(key, _ []byte) error { pickedFiles = append(pickedFiles, string(key)) return nil }) }) fmt.Println("New files:") PrintFilesOrLength(fsStatus.NewFiles, Color.Green) fmt.Println("Added files:") PrintFilesOrLength(pickedFiles, Color.Green) fmt.Println("Modified files:") PrintFilesOrLength(fsStatus.ModifiedFiles, Color.Warning) fmt.Println("Deleted files:") PrintFilesOrLength(fsStatus.MissingFiles, Color.Fatal) return nil } func GetFilesystemLength(root string) (int, error) { var ( rootStat os.FileInfo fsCount int = 0 e error ) rootStat, e = os.Stat(root) if e != nil { return fsCount, e } if rootStat.IsDir() && root[len(root)-1:] != "/" { root = root + "/" } filepath.Walk(root, func(p string, i os.FileInfo, _ error) error { // Ignore path in Variables.PruneRegexPaths if i.IsDir() && matchAny(p, PruneRegex) { return filepath.SkipDir } // Ignore path in Variables.IgnoreRegexPaths if matchAny(p, IgnoreRegex) { return nil } if !i.Mode().IsRegular() && (i.Mode()&os.ModeSymlink == 0) { return nil } fsCount++ return nil }) return fsCount, e } func (fsStatus *FilesystemStatus) parseFile(indexBucket, picksBucket *bolt.Bucket, p string, bar *mpb.Bar) { var ( newFileObject FileObject knownFileObject FileObject pick, known []byte e error ) defer func() { bar.Increment() fsStatusWG.Done() }() pick = picksBucket.Get([]byte(p)) known = indexBucket.Get([]byte(p)) if pick != nil { fsStatusWG.Wait() fsStatusWG.Add(1) fsStatus.PickedFiles = append(fsStatus.PickedFiles, p) return } if known != nil { newFileObject, e = CreateFileObject(p) if e != nil { return } knownFileObject, e = FromBytes(known) if e != nil { return } e = newFileObject.IsDifferent(knownFileObject) if e != nil { fsStatusWG.Wait() fsStatusWG.Add(1) fsStatus.ModifiedFiles = append(fsStatus.ModifiedFiles, p) } return } fsStatusWG.Wait() fsStatusWG.Add(1) fsStatus.NewFiles = append(fsStatus.NewFiles, p) return } func GetFilesystemDiff(root string) (FilesystemStatus, error) { var ( fsStatus FilesystemStatus = FilesystemStatus{} sem *semaphore.Weighted picksBucket *bolt.Bucket indexBucket *bolt.Bucket rootStat os.FileInfo bar *mpb.Bar fsCount int poolSize int e error ) poolSize = runtime.NumCPU() sem = semaphore.NewWeighted(int64(poolSize)) rootStat, e = os.Stat(root) if e != nil { return fsStatus, e } if rootStat.IsDir() && root[len(root)-1:] != "/" { root = root + "/" } fsCount, e = GetFilesystemLength(root) if e != nil { return fsStatus, e } bar = ProgressBar.InitBar("Scanning...", fsCount) e = Database.FsDB.View(func(tx *bolt.Tx) error { picksBucket = tx.Bucket(Variables.FsHashPicksBucket) indexBucket = tx.Bucket(Variables.FsHashIndexBucket) filepath.Walk(root, func(p string, i os.FileInfo, _ error) error { // Ignore path in Variables.PruneRegexPaths if i.IsDir() && matchAny(p, PruneRegex) { return filepath.SkipDir } // Ignore path in Variables.IgnoreRegexPaths if matchAny(p, IgnoreRegex) { return nil } if !i.Mode().IsRegular() && (i.Mode()&os.ModeSymlink == 0) { return nil } FsWalkWG.Add(1) sem.Acquire(context.Background(), 1) go func() { fsStatus.parseFile(indexBucket, picksBucket, p, bar) sem.Release(1) FsWalkWG.Done() }() FsWalkWG.Wait() return nil }) indexBucket.ForEach(func(k, v []byte) error { _, e = os.Lstat(string(k)) if os.IsNotExist(e) { fsStatus.MissingFiles = append(fsStatus.MissingFiles, string(k)) } return nil }) return nil }) return fsStatus, e }