From a12d1b6d2ecd49078304fd8f139f1da440b05c39 Mon Sep 17 00:00:00 2001 From: Tovi Jaeschke-Rogers Date: Mon, 19 Jul 2021 17:12:14 +0930 Subject: [PATCH] WIP --- Client/Filesystem/CommitFiles.go | 4 +- Client/Filesystem/Config.go | 3 - Client/Filesystem/FileObject.go | 11 ++- Client/Filesystem/FilesystemDiff.go | 106 ++++++++++++++++---------- Client/Filesystem/ManageFileBucket.go | 2 +- Client/Filesystem/PickFiles.go | 69 ++++++++++++++--- Client/Filesystem/Print.go | 3 +- Client/ProgressBar/Bar.go | 8 +- Client/main.go | 37 ++++++++- Variables/Variables.go | 5 ++ 10 files changed, 176 insertions(+), 72 deletions(-) diff --git a/Client/Filesystem/CommitFiles.go b/Client/Filesystem/CommitFiles.go index 0b47f19..8cb066b 100644 --- a/Client/Filesystem/CommitFiles.go +++ b/Client/Filesystem/CommitFiles.go @@ -29,10 +29,10 @@ func CommitFiles() error { if len(fsStatus.PickedFiles) > 0 { bar = ProgressBar.InitBar("Commiting...", len(fsStatus.PickedFiles)) for _, f = range fsStatus.PickedFiles { - e = AddFileToBucket(indexBucket, f) bar.Increment() + e = AddFileToBucket(indexBucket, f) if e != nil { - return e + return nil } } diff --git a/Client/Filesystem/Config.go b/Client/Filesystem/Config.go index fb192c2..b8fabd5 100644 --- a/Client/Filesystem/Config.go +++ b/Client/Filesystem/Config.go @@ -1,15 +1,12 @@ package Filesystem import ( - "crypto/sha1" "regexp" "PackageManager/Variables" ) var ( - sha1Hash = sha1.New() - PruneRegex []*regexp.Regexp IgnoreRegex []*regexp.Regexp ) diff --git a/Client/Filesystem/FileObject.go b/Client/Filesystem/FileObject.go index b7c498b..0cf1e3e 100644 --- a/Client/Filesystem/FileObject.go +++ b/Client/Filesystem/FileObject.go @@ -2,10 +2,12 @@ package Filesystem import ( "bytes" + "crypto/sha1" "encoding/gob" "encoding/hex" "errors" "fmt" + "hash" "io" "os" "path/filepath" @@ -141,10 +143,11 @@ func (f FileObject) IsDifferent(fn FileObject) error { func CreateFileObject(f string) (FileObject, error) { var ( - fo FileObject - fi os.FileInfo - file *os.File - e error + sha1Hash hash.Hash = sha1.New() + fo FileObject + fi os.FileInfo + file *os.File + e error ) fi, e = os.Lstat(f) diff --git a/Client/Filesystem/FilesystemDiff.go b/Client/Filesystem/FilesystemDiff.go index 5d023b1..73eb41f 100644 --- a/Client/Filesystem/FilesystemDiff.go +++ b/Client/Filesystem/FilesystemDiff.go @@ -1,15 +1,18 @@ package Filesystem import ( - "PackageManager/Client/Database" - "PackageManager/Color" - "PackageManager/Variables" "fmt" - "log" "os" "path/filepath" + "github.com/vbauerster/mpb" + "github.com/zenthangplus/goccm" bolt "go.etcd.io/bbolt" + + "PackageManager/Client/Database" + "PackageManager/Client/ProgressBar" + "PackageManager/Color" + "PackageManager/Variables" ) type FilesystemStatus struct { @@ -66,7 +69,6 @@ func GetFilesystemLength(root string) (int, error) { // Ignore path in Variables.PruneRegexPaths if i.IsDir() && matchAny(p, PruneRegex) { - log.Println("Prune", p) return filepath.SkipDir } @@ -87,6 +89,51 @@ func GetFilesystemLength(root string) (int, error) { return fsCount, e } +func (fsStatus *FilesystemStatus) parseFile(indexBucket, picksBucket *bolt.Bucket, p string, bar *mpb.Bar, c goccm.ConcurrencyManager) { + var ( + newFileObject FileObject + knownFileObject FileObject + pick, known []byte + e error + ) + + defer func() { + bar.Increment() + c.Done() + }() + + pick = picksBucket.Get([]byte(p)) + known = indexBucket.Get([]byte(p)) + + if pick != nil { + fsStatus.PickedFiles = append(fsStatus.PickedFiles, p) + return + } + + if known != nil { + newFileObject, e = CreateFileObject(p) + if e != nil { + return + } + + knownFileObject, e = FromBytes(known) + if e != nil { + return + } + + e = newFileObject.IsDifferent(knownFileObject) + if e != nil { + fsStatus.ModifiedFiles = append(fsStatus.ModifiedFiles, p) + } + + return + } + + fsStatus.NewFiles = append(fsStatus.NewFiles, p) + + return +} + func GetFilesystemDiff(root string) (FilesystemStatus, error) { var ( fsStatus FilesystemStatus = FilesystemStatus{} @@ -96,10 +143,11 @@ func GetFilesystemDiff(root string) (FilesystemStatus, error) { picksBucket *bolt.Bucket indexBucket *bolt.Bucket - pick, known []byte + bar *mpb.Bar - newFileObject FileObject - knownFileObject FileObject + fsCount int + + c goccm.ConcurrencyManager e error ) @@ -113,6 +161,13 @@ func GetFilesystemDiff(root string) (FilesystemStatus, error) { root = root + "/" } + fsCount, e = GetFilesystemLength(root) + if e != nil { + return fsStatus, e + } + + bar = ProgressBar.InitBar("Scanning...", fsCount) + e = Database.FsDB.View(func(tx *bolt.Tx) error { picksBucket = tx.Bucket(Variables.FsHashPicksBucket) @@ -122,7 +177,6 @@ func GetFilesystemDiff(root string) (FilesystemStatus, error) { // Ignore path in Variables.PruneRegexPaths if i.IsDir() && matchAny(p, PruneRegex) { - log.Println("Prune", p) return filepath.SkipDir } @@ -135,37 +189,7 @@ func GetFilesystemDiff(root string) (FilesystemStatus, error) { return nil } - pick = picksBucket.Get([]byte(p)) - known = indexBucket.Get([]byte(p)) - - if pick != nil { - fsStatus.PickedFiles = append(fsStatus.PickedFiles, p) - return nil - } - - if known != nil { - newFileObject, e = CreateFileObject(p) - if os.IsNotExist(e) { - return nil - } - if e != nil { - return e - } - - knownFileObject, e = FromBytes(known) - if e != nil { - return e - } - - e = newFileObject.IsDifferent(knownFileObject) - if e != nil { - fsStatus.ModifiedFiles = append(fsStatus.ModifiedFiles, p) - } - - return nil - } - - fsStatus.NewFiles = append(fsStatus.NewFiles, p) + go fsStatus.parseFile(indexBucket, picksBucket, p, bar, c) return nil }) @@ -178,6 +202,8 @@ func GetFilesystemDiff(root string) (FilesystemStatus, error) { return nil }) + ProgressBar.CloseBar(bar) + return nil }) diff --git a/Client/Filesystem/ManageFileBucket.go b/Client/Filesystem/ManageFileBucket.go index 8cd907e..4a0c8b2 100644 --- a/Client/Filesystem/ManageFileBucket.go +++ b/Client/Filesystem/ManageFileBucket.go @@ -17,7 +17,7 @@ func AddFileToBucket(bucket *bolt.Bucket, filePath string) error { return nil } if e != nil { - return e + return nil } fileObjectBytes, e = fileObject.ToBytes() if e != nil { diff --git a/Client/Filesystem/PickFiles.go b/Client/Filesystem/PickFiles.go index fd908a0..984b7a7 100644 --- a/Client/Filesystem/PickFiles.go +++ b/Client/Filesystem/PickFiles.go @@ -1,20 +1,42 @@ package Filesystem import ( - "PackageManager/Client/Database" - "PackageManager/Client/ProgressBar" - "PackageManager/Variables" + "os" "github.com/vbauerster/mpb" bolt "go.etcd.io/bbolt" + + "PackageManager/Client/Database" + "PackageManager/Client/ProgressBar" + "PackageManager/Variables" ) -func PickFiles(rootPath string) error { +func pickFilesSingle(rootPath string) error { + var ( + indexBucket *bolt.Bucket + picksBucket *bolt.Bucket + e error + ) + e = Database.FsDB.Batch(func(tx *bolt.Tx) error { + indexBucket = tx.Bucket(Variables.FsHashIndexBucket) + picksBucket = tx.Bucket(Variables.FsHashPicksBucket) + + e = AddFileToBucket(picksBucket, rootPath) + if e != nil { + return e + } + return RemoveFileFromBucket(indexBucket, rootPath) + }) + return e +} + +func pickFilesRecursive(rootPath string) error { var ( fsStatus FilesystemStatus + indexBucket *bolt.Bucket picksBucket *bolt.Bucket - totalLen int bar *mpb.Bar + totalLen int f string e error ) @@ -33,34 +55,39 @@ func PickFiles(rootPath string) error { bar = ProgressBar.InitBar("Adding...", totalLen) e = Database.FsDB.Batch(func(tx *bolt.Tx) error { + indexBucket = tx.Bucket(Variables.FsHashIndexBucket) picksBucket = tx.Bucket(Variables.FsHashPicksBucket) if len(fsStatus.NewFiles) > 0 { for _, f = range fsStatus.NewFiles { - e = AddFileToBucket(picksBucket, f) bar.Increment() + e = AddFileToBucket(picksBucket, f) if e != nil { - return e + return nil } } } if len(fsStatus.ModifiedFiles) > 0 { for _, f = range fsStatus.ModifiedFiles { - e = AddFileToBucket(picksBucket, f) bar.Increment() + e = AddFileToBucket(picksBucket, f) if e != nil { - return e + return nil } } } if len(fsStatus.MissingFiles) > 0 { - for _, f = range fsStatus.NewFiles { - e = RemoveFileFromBucket(picksBucket, f) + for _, f = range fsStatus.MissingFiles { bar.Increment() + e = RemoveFileFromBucket(indexBucket, f) if e != nil { - return e + return nil + } + e = RemoveFileFromBucket(picksBucket, f) + if e != nil { + return nil } } } @@ -73,6 +100,24 @@ func PickFiles(rootPath string) error { return e } +func PickFiles(rootPath string) error { + var ( + rootStat os.FileInfo + e error + ) + + rootStat, e = os.Stat(rootPath) + if e != nil { + return e + } + + if !rootStat.IsDir() { + return pickFilesSingle(rootPath) + } + + return pickFilesRecursive(rootPath) +} + func ResetAllPickedFiles() error { var ( e error diff --git a/Client/Filesystem/Print.go b/Client/Filesystem/Print.go index cfeaef0..668cb61 100644 --- a/Client/Filesystem/Print.go +++ b/Client/Filesystem/Print.go @@ -1,6 +1,7 @@ package Filesystem import ( + "PackageManager/Variables" "fmt" ) @@ -16,7 +17,7 @@ func PrintFilesLength(files []string) { } func PrintFilesOrLength(files []string, color func(...interface{}) string) { - if len(files) < 25 && len(files) > 0 { + if (Variables.VerboseOutput && len(files) != 0) || (len(files) < 25 && len(files) > 0) { PrintFiles(files, color) return } diff --git a/Client/ProgressBar/Bar.go b/Client/ProgressBar/Bar.go index 2e0ade7..e60e3cd 100644 --- a/Client/ProgressBar/Bar.go +++ b/Client/ProgressBar/Bar.go @@ -1,15 +1,12 @@ package ProgressBar import ( - "sync" - "github.com/vbauerster/mpb" "github.com/vbauerster/mpb/decor" ) var ( - BarWG sync.WaitGroup - P = mpb.New(mpb.WithWaitGroup(&BarWG)) + P = mpb.New() ) func InitBar(name string, total int) *mpb.Bar { @@ -17,8 +14,6 @@ func InitBar(name string, total int) *mpb.Bar { bar *mpb.Bar ) - BarWG.Add(1) - bar = P.AddBar(int64(total), mpb.PrependDecorators( decor.Name(name), @@ -37,5 +32,4 @@ func InitBar(name string, total int) *mpb.Bar { func CloseBar(bar *mpb.Bar) { bar.Abort(false) - BarWG.Done() } diff --git a/Client/main.go b/Client/main.go index 967d4fb..f820c15 100644 --- a/Client/main.go +++ b/Client/main.go @@ -3,6 +3,7 @@ package main import ( "flag" "fmt" + "os" "PackageManager/Client/Database" "PackageManager/Client/Filesystem" @@ -11,6 +12,29 @@ import ( "PackageManager/Variables" ) +func HelpMsg() { + var helpMsg string + helpMsg = `Usage of %s: + + -Af | -add-files + Add files + + -Cf | -commit + Add files + + -Fd | -fs-diff + Filesystem diff + + -Rf | -reset + Reset added files + + -V | -verbose + Verbose output +` + helpMsg = fmt.Sprintf(helpMsg, os.Args[0]) + fmt.Println(helpMsg) +} + func main() { var ( getFilesystemDiffFlag bool @@ -22,9 +46,14 @@ func main() { resetAddedFilesFlag bool resetAddedFilesFlagLong bool + verboseOutputFlag bool + verboseOutputFlagLong bool + e error ) + flag.Usage = HelpMsg + e = Helper.CheckRoot() if e != nil { fmt.Println(Color.Fatal(e)) @@ -39,6 +68,8 @@ func main() { // TODO: Rework usage function // Initialise flags + flag.BoolVar(&verboseOutputFlag, "V", false, "Verbose output") + flag.BoolVar(&verboseOutputFlagLong, "verbose", false, "Verbose output") flag.BoolVar(&getFilesystemDiffFlag, "Fd", false, "Filesystem diff") flag.BoolVar(&getFilesystemDiffFlagLong, "fs-diff", false, "Filesystem diff") @@ -46,14 +77,16 @@ func main() { flag.BoolVar(&addFileDiffFlag, "Af", false, "Add files") flag.BoolVar(&addFileDiffFlagLong, "add-files", false, "Add files") - flag.BoolVar(&commitAddedFilesFlag, "Cf", false, "Add files") - flag.BoolVar(&commitAddedFilesFlagLong, "commit", false, "Add files") + flag.BoolVar(&commitAddedFilesFlag, "Cf", false, "Commit files") + flag.BoolVar(&commitAddedFilesFlagLong, "commit", false, "Commit files") flag.BoolVar(&resetAddedFilesFlag, "Rf", false, "Reset added files") flag.BoolVar(&resetAddedFilesFlagLong, "reset", false, "Reset added files") flag.Parse() + Variables.VerboseOutput = verboseOutputFlag || verboseOutputFlagLong + if getFilesystemDiffFlag || getFilesystemDiffFlagLong { var rootPath string = Variables.RootDir if len(flag.Args()) > 1 { diff --git a/Variables/Variables.go b/Variables/Variables.go index a52aa55..15482d8 100644 --- a/Variables/Variables.go +++ b/Variables/Variables.go @@ -2,6 +2,7 @@ package Variables import ( "os" + "sync" ) const ( @@ -10,6 +11,9 @@ const ( ) var ( + WG sync.WaitGroup + + VerboseOutput bool = false RootDir string = "/" FsHashPicksBucket []byte = []byte("FilesystemPicks") FsHashIndexBucket []byte = []byte("FilesystemIndex") @@ -46,6 +50,7 @@ var ( } IgnoreRegexPaths []string = []string{ + "^/swapfile$", "^/etc/passwd$", "^/etc/passwd-$", "^/etc/group$",