From 8154034d5aabcedac227ab5eb75a6b08cca4f4a7 Mon Sep 17 00:00:00 2001 From: Tovi Jaeschke-Rogers Date: Thu, 8 Jul 2021 08:45:47 +0930 Subject: [PATCH 01/12] Get filesystem diff --- Client/Database/Filesystem.go | 143 ++++++++++++++++++++++++++++ Client/Database/Init.go | 26 ++++- Client/Filesystem/HashFilesystem.go | 128 +++++++++++++++++++++++++ Client/main.go | 28 +++++- Variables/Variables.go | 5 + package_manager.db | Bin 0 -> 12288 bytes 6 files changed, 321 insertions(+), 9 deletions(-) create mode 100644 Client/Database/Filesystem.go create mode 100644 Client/Filesystem/HashFilesystem.go create mode 100644 Variables/Variables.go create mode 100644 package_manager.db diff --git a/Client/Database/Filesystem.go b/Client/Database/Filesystem.go new file mode 100644 index 0000000..489879d --- /dev/null +++ b/Client/Database/Filesystem.go @@ -0,0 +1,143 @@ +package Database + +import ( + "database/sql" + "fmt" + "time" +) + +type FilesystemHashRow struct { + Path string + Hash string + UpdatedAt time.Time +} + +func FindOrCreateFileHash(rows []FilesystemHashRow) error { + var ( + stmtString string + stmt *sql.Stmt + values []interface{} = []interface{}{} + e error + ) + + stmtString = "INSERT OR REPLACE INTO filesystem_hash (id, path, hash, updated_at) VALUES " + + for _, row := range rows { + stmtString += `( + (SELECT id FROM filesystem_hash WHERE path = ?), + ?, + ?, + ? + ),` + values = append(values, row.Path, row.Path, row.Hash, row.UpdatedAt.String()) + } + + stmtString = stmtString[0 : len(stmtString)-1] + + stmt, e = DB.Prepare(stmtString) + if e != nil { + return e + } + + _, e = stmt.Exec(values...) + + return e +} + +func FindModifiedFiles(hashes []string) ([]string, error) { + var ( + stmtString string + stmt *sql.Stmt + values []interface{} = []interface{}{} + result *sql.Rows + dirtyFiles []string = []string{} + e error + ) + + stmtString = "SELECT id, path FROM filesystem_hash WHERE hash NOT IN (" + + for _, row := range hashes { + stmtString += "?," + values = append(values, row) + } + + stmtString = stmtString[0:len(stmtString)-1] + ")" + + stmt, e = DB.Prepare(stmtString) + if e != nil { + return dirtyFiles, e + } + + result, e = stmt.Query(values...) + if e != nil { + return dirtyFiles, e + } + + defer result.Close() + for result.Next() { + var id string + var hash string + e = result.Scan(&id, &hash) + if e != nil { + return dirtyFiles, e + } + fmt.Println(id, hash) + dirtyFiles = append(dirtyFiles, hash) + } + + e = result.Err() + return dirtyFiles, e +} + +/* +func FindNewFiles(files []string) ([]string, error) { + var ( + stmtString string + stmt *sql.Stmt + values []interface{} = []interface{}{} + result *sql.Rows + dirtyFiles []string = []string{} + e error + ) + + stmtString = `select * +from ( + values (4),(5),(6) +) as v(id) +where not exists (select * + from images i + where i.id = v.id);` + + for _, row := range hashes { + stmtString += "?," + values = append(values, row) + } + + stmtString = stmtString[0:len(stmtString)-1] + ")" + + stmt, e = DB.Prepare(stmtString) + if e != nil { + return dirtyFiles, e + } + + result, e = stmt.Query(values...) + if e != nil { + return dirtyFiles, e + } + + defer result.Close() + for result.Next() { + var id string + var hash string + e = result.Scan(&id, &hash) + if e != nil { + return dirtyFiles, e + } + fmt.Println(id, hash) + dirtyFiles = append(dirtyFiles, hash) + } + + e = result.Err() + return dirtyFiles, e +} +*/ diff --git a/Client/Database/Init.go b/Client/Database/Init.go index 9410e25..1aa7d2c 100644 --- a/Client/Database/Init.go +++ b/Client/Database/Init.go @@ -2,9 +2,10 @@ package Database import ( "database/sql" - "log" _ "github.com/mattn/go-sqlite3" + + "PackageManager/Variables" ) var ( @@ -13,12 +14,29 @@ var ( func init() { var e error - DB, e = sql.Open("sqlite3", "./foo.db") + DB, e = sql.Open("sqlite3", Variables.DatabaseName) if e != nil { panic(e) } } -func InitDB() { - log.Println("Initialising Database...") +func InitDB() error { + var ( + sqlStmt string + e error + ) + sqlStmt = ` + CREATE TABLE IF NOT EXISTS filesystem_hash ( + id INTEGER NOT NULL PRIMARY KEY, + path VARCHAR(256), + hash VARCHAR(64), + created_at DATE DEFAULT CURRENT_TIMESTAMP, + updated_at DATE + ) + ` + _, e = DB.Exec(sqlStmt) + if e != nil { + return e + } + return nil } diff --git a/Client/Filesystem/HashFilesystem.go b/Client/Filesystem/HashFilesystem.go new file mode 100644 index 0000000..262e7c6 --- /dev/null +++ b/Client/Filesystem/HashFilesystem.go @@ -0,0 +1,128 @@ +package Filesystem + +import ( + "PackageManager/Client/Database" + "crypto/md5" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" +) + +func HashFile(path string) (string, error) { + var ( + Md5Hash string + hashBytes [16]byte + file *os.File + e error + ) + + file, e = os.Open(path) + if e != nil { + panic(e) + } + + defer file.Close() + + body, e := ioutil.ReadAll(file) + if e != nil { + panic(e) + } + + //Get the 16 bytes hash + hashBytes = md5.Sum(body) + + //Convert the bytes to a string + Md5Hash = fmt.Sprintf("%x", hashBytes) + + return Md5Hash, nil +} + +func UpdateFilesystemHash() error { + var ( + fileHash string + rows []Database.FilesystemHashRow + e error + ) + + e = filepath.Walk(".", func(path string, info os.FileInfo, e error) error { + if e != nil { + return e + } + + // Ignore hidden files + if strings.HasPrefix(info.Name(), ".") || strings.HasPrefix(path, ".") { + return nil + } + + // Ignore directories + if info.IsDir() { + return nil + } + + fileHash, e = HashFile(path) + + if e != nil { + return e + } + + rows = append(rows, Database.FilesystemHashRow{ + Path: path, + Hash: fileHash, + UpdatedAt: info.ModTime(), + }) + + // TODO: If len(rows) > x, update the db and clear out rows, and continue + + return nil + }) + + if e != nil { + panic(e) + } + + return Database.FindOrCreateFileHash(rows) +} + +func GetFilesystemDiff() error { + var ( + fileHash string + hashes []string = []string{} + e error + ) + + e = filepath.Walk(".", func(path string, info os.FileInfo, e error) error { + if e != nil { + return e + } + + // Ignore hidden files + if strings.HasPrefix(info.Name(), ".") || strings.HasPrefix(path, ".") { + return nil + } + + // Ignore directories + if info.IsDir() { + return nil + } + + fileHash, e = HashFile(path) + + if e != nil { + return e + } + + hashes = append(hashes, fileHash) + + // TODO: If len(rows) > x, update the db and clear out rows, and continue + + return nil + }) + + dirty, e := Database.FindModifiedFiles(hashes) + + fmt.Println(dirty) + + return e +} diff --git a/Client/main.go b/Client/main.go index 4675230..e68248b 100644 --- a/Client/main.go +++ b/Client/main.go @@ -1,13 +1,31 @@ package main import ( - "fmt" - - "PackageManager/Archive" + "PackageManager/Client/Database" + "PackageManager/Client/Filesystem" ) func main() { + var e error + + e = Database.InitDB() + if e != nil { + panic(e) + } + + /* + e = Filesystem.UpdateFilesystemHash() + if e != nil { + panic(e) + } + */ + + e = Filesystem.GetFilesystemDiff() + if e != nil { + panic(e) + } + //e := Archive.TarGzip("/tmp/test", "/tmp/test.tar.gz") - e := Archive.UntarGzip("/tmp/test.tar.gz", "/tmp/test") - fmt.Println(e) + //e := Archive.UntarGzip("/tmp/test.tar.gz", "/tmp/test") + //fmt.Println(e) } diff --git a/Variables/Variables.go b/Variables/Variables.go new file mode 100644 index 0000000..8592aed --- /dev/null +++ b/Variables/Variables.go @@ -0,0 +1,5 @@ +package Variables + +const ( + DatabaseName string = "package_manager.db" +) diff --git a/package_manager.db b/package_manager.db new file mode 100644 index 0000000000000000000000000000000000000000..01f184cd7c6f7a42f1eaab2744c85aa26073fca8 GIT binary patch literal 12288 zcmeH~O>f&q5Qa%frlTOX8lXUcoQNI*8+NJv{!lJeWF;{wyQw6n$f0nT zNe=;%Yme=p=s)YN*PeH070x7U7N`|QTA;J|?d zkN^@u0!RP}AOR$R1dsp{00Q%CQ9$?jw`VT<-aYk)CvFPUEXmW%Na_lfI+-e`thJOe ztuvkxo~4C$j4=8+(Qw6#b>Jd;;W9DN9SJU#aI1`_#+rDGSWe<@Z!~_s8U+?aAG_J4 z>elA%g_S%#HJ%s3i9)GNNbRZ4U8c381!q)qnJ-j6Doi^XPtj6a||g+&iB5 zY1uw;lPL&NCS^_y;SvHACimV_s(r4w%W{{Js|6ubmS_VzCAD@b)7OoaC>UKR-d}W^ z?T63+UMwLD;1SUsVun7Q=U-=m1S)1d7dlEbxPBeyMJ}!TqJ~7CoYv*(tlQq0tx!vS#~t}!M6{l z?wti`OJ}Jdne#*lZM|Z-^Nv$*v`?wG>WZ{w2Y^$WK%hb?a;ebf;k)1JT=wr7L^Drq zCWWTFfUH3qirZNwwImuuFaDtqSGVkNV674*xR6Gv0VIF~ skN^@u0!RP}AOR$R1dsp{Kmz|`0xR`KGYD5!YmG1rYxPDHHky_37geoMK>z>% literal 0 HcmV?d00001 From 67488fbd703f520c102c14620a3e456c170ae8e3 Mon Sep 17 00:00:00 2001 From: Tovi Jaeschke-Rogers Date: Fri, 9 Jul 2021 03:27:45 +0930 Subject: [PATCH 02/12] Start on package creation --- .gitignore | 1 + Client/Database/Filesystem.go | 73 ++++++------------ Client/Database/Init.go | 4 +- Client/Filesystem/HashFilesystem.go | 43 +++++++++-- Client/Package/CreatePackage.go | 115 ++++++++++++++++++++++++++++ Client/main.go | 41 ++++++++-- Color/Color.go | 58 ++++++++++++++ Helper/Input.go | 24 ++++++ package_manager.db | Bin 12288 -> 0 bytes 9 files changed, 293 insertions(+), 66 deletions(-) create mode 100644 .gitignore create mode 100644 Client/Package/CreatePackage.go create mode 100644 Color/Color.go create mode 100644 Helper/Input.go delete mode 100644 package_manager.db diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..98e6ef6 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +*.db diff --git a/Client/Database/Filesystem.go b/Client/Database/Filesystem.go index 489879d..11b56eb 100644 --- a/Client/Database/Filesystem.go +++ b/Client/Database/Filesystem.go @@ -2,7 +2,6 @@ package Database import ( "database/sql" - "fmt" "time" ) @@ -29,7 +28,7 @@ func FindOrCreateFileHash(rows []FilesystemHashRow) error { ?, ? ),` - values = append(values, row.Path, row.Path, row.Hash, row.UpdatedAt.String()) + values = append(values, row.Path, row.Path, row.Hash, row.UpdatedAt.Unix()) } stmtString = stmtString[0 : len(stmtString)-1] @@ -44,13 +43,14 @@ func FindOrCreateFileHash(rows []FilesystemHashRow) error { return e } -func FindModifiedFiles(hashes []string) ([]string, error) { +func FindModifiedFiles(hashes []string) (map[int]string, error) { var ( stmtString string stmt *sql.Stmt values []interface{} = []interface{}{} result *sql.Rows - dirtyFiles []string = []string{} + dirtyFiles map[int]string = make(map[int]string) + counter int = 0 e error ) @@ -76,68 +76,39 @@ func FindModifiedFiles(hashes []string) ([]string, error) { defer result.Close() for result.Next() { var id string - var hash string - e = result.Scan(&id, &hash) + var path string + e = result.Scan(&id, &path) if e != nil { return dirtyFiles, e } - fmt.Println(id, hash) - dirtyFiles = append(dirtyFiles, hash) + dirtyFiles[counter] = path + counter++ } e = result.Err() return dirtyFiles, e } -/* -func FindNewFiles(files []string) ([]string, error) { +func GetMostRecentTimestamp() (time.Time, error) { var ( - stmtString string - stmt *sql.Stmt - values []interface{} = []interface{}{} - result *sql.Rows - dirtyFiles []string = []string{} - e error + stmt *sql.Stmt + result *sql.Row + lastUpdatedAt int64 + e error ) - stmtString = `select * -from ( - values (4),(5),(6) -) as v(id) -where not exists (select * - from images i - where i.id = v.id);` - - for _, row := range hashes { - stmtString += "?," - values = append(values, row) - } - - stmtString = stmtString[0:len(stmtString)-1] + ")" - - stmt, e = DB.Prepare(stmtString) + stmt, e = DB.Prepare(` + SELECT updated_at FROM filesystem_hash + ORDER BY updated_at DESC + LIMIT 1; + `) if e != nil { - return dirtyFiles, e + return time.Now(), e } - result, e = stmt.Query(values...) - if e != nil { - return dirtyFiles, e - } + result = stmt.QueryRow() - defer result.Close() - for result.Next() { - var id string - var hash string - e = result.Scan(&id, &hash) - if e != nil { - return dirtyFiles, e - } - fmt.Println(id, hash) - dirtyFiles = append(dirtyFiles, hash) - } + result.Scan(&lastUpdatedAt) - e = result.Err() - return dirtyFiles, e + return time.Unix(lastUpdatedAt, 0), nil } -*/ diff --git a/Client/Database/Init.go b/Client/Database/Init.go index 1aa7d2c..98b2596 100644 --- a/Client/Database/Init.go +++ b/Client/Database/Init.go @@ -30,8 +30,8 @@ func InitDB() error { id INTEGER NOT NULL PRIMARY KEY, path VARCHAR(256), hash VARCHAR(64), - created_at DATE DEFAULT CURRENT_TIMESTAMP, - updated_at DATE + created_at INTEGER DEFAULT CURRENT_TIMESTAMP, + updated_at INTEGER ) ` _, e = DB.Exec(sqlStmt) diff --git a/Client/Filesystem/HashFilesystem.go b/Client/Filesystem/HashFilesystem.go index 262e7c6..bca43c7 100644 --- a/Client/Filesystem/HashFilesystem.go +++ b/Client/Filesystem/HashFilesystem.go @@ -8,6 +8,7 @@ import ( "os" "path/filepath" "strings" + "time" ) func HashFile(path string) (string, error) { @@ -85,13 +86,24 @@ func UpdateFilesystemHash() error { return Database.FindOrCreateFileHash(rows) } -func GetFilesystemDiff() error { +func GetFilesystemDiff() (map[int]string, map[int]string, error) { var ( - fileHash string - hashes []string = []string{} - e error + fileHash string + hashes []string = []string{} + lastUpdatedAt time.Time + dirtyFiles map[int]string + newFiles map[int]string = make(map[int]string) + newFilesTmp map[string]string = make(map[string]string) + counter int + ok bool + e error ) + lastUpdatedAt, e = Database.GetMostRecentTimestamp() + if e != nil { + return dirtyFiles, newFiles, e + } + e = filepath.Walk(".", func(path string, info os.FileInfo, e error) error { if e != nil { return e @@ -115,14 +127,31 @@ func GetFilesystemDiff() error { hashes = append(hashes, fileHash) + if info.ModTime().After(lastUpdatedAt) { + newFilesTmp[path] = path + } + // TODO: If len(rows) > x, update the db and clear out rows, and continue return nil }) + if e != nil { + return dirtyFiles, newFiles, e + } - dirty, e := Database.FindModifiedFiles(hashes) + dirtyFiles, e = Database.FindModifiedFiles(hashes) + if e != nil { + return dirtyFiles, newFiles, e + } - fmt.Println(dirty) + counter = len(dirtyFiles) + + for _, file := range dirtyFiles { + _, ok = newFilesTmp[file] + if !ok { + newFiles[counter] = file + } + } - return e + return dirtyFiles, newFiles, e } diff --git a/Client/Package/CreatePackage.go b/Client/Package/CreatePackage.go new file mode 100644 index 0000000..31ab612 --- /dev/null +++ b/Client/Package/CreatePackage.go @@ -0,0 +1,115 @@ +package Package + +import ( + "errors" + "fmt" + "strconv" + "strings" + + "PackageManager/Client/Filesystem" + "PackageManager/Color" + "PackageManager/Helper" +) + +func CreatePackage() error { + var ( + dirtyFiles map[int]string + newFiles map[int]string + pkgFiles map[int]string = make(map[int]string) + choices string + choicesSplit []string + filePath string + pkgName string + pkgVersion string + index int + ok bool + e error + ) + + fmt.Println("Initialising package creation...") + + dirtyFiles, newFiles, e = Filesystem.GetFilesystemDiff() + if e != nil { + return e + } + + fmt.Println("\nModified files...") + for i, file := range dirtyFiles { + fmt.Printf( + "\t%d - %s\n", + i, + Color.Red(file), + ) + } + + fmt.Println("\nNew files...") + for i, file := range newFiles { + fmt.Printf( + "\t%d - %s\n", + i, + Color.Red(file), + ) + } + + fmt.Println("Please select the files you would like to use to create the package. Leave empty for all.") + choices = Helper.Input() + + if choices == "" { + for i, file := range dirtyFiles { + pkgFiles[i] = file + } + for i, file := range newFiles { + pkgFiles[i] = file + } + } else { + + choicesSplit = strings.Split(choices, ",") + + for _, i := range choicesSplit { + index, e = strconv.Atoi(i) + if e != nil { + // TODO: Handle this error + panic(e) + } + filePath, ok = dirtyFiles[index] + if !ok { + filePath, ok = dirtyFiles[index] + if !ok { + return errors.New("Invalid package selection") + } + } + pkgFiles[index] = filePath + } + } + + fmt.Println("Please enter the package name:") + pkgName = Helper.Input() + if pkgName == "" { + return errors.New("Invalid package name") + } + + fmt.Println("Please enter the package version:") + pkgVersion = Helper.Input() + if pkgVersion == "" { + return errors.New("Invalid package name") + } + + fmt.Printf("Package Name: %s\n", pkgName) + fmt.Printf("Package Version: %s\n", pkgVersion) + + fmt.Println("Files to be added") + for i, file := range pkgFiles { + fmt.Printf( + "\t%d - %s\n", + i, + Color.Green(file), + ) + } + + fmt.Println("Is this correct? [y/N]") + if strings.ToLower(Helper.Input()) != "y" { + return errors.New("User aborted") + } + + return nil +} diff --git a/Client/main.go b/Client/main.go index e68248b..faec932 100644 --- a/Client/main.go +++ b/Client/main.go @@ -1,30 +1,59 @@ package main import ( + "flag" + "fmt" + "PackageManager/Client/Database" "PackageManager/Client/Filesystem" + "PackageManager/Client/Package" + "PackageManager/Color" ) func main() { - var e error + var ( + updateFilesytemFlag bool + updateFilesytemFlagLong bool + createPackageFlag bool + createPackageFlagLong bool + + e error + ) e = Database.InitDB() if e != nil { panic(e) } - /* + // Initialise flags + flag.BoolVar(&updateFilesytemFlag, "Uf", false, "Update filesystem database") + flag.BoolVar(&updateFilesytemFlagLong, "update-filesystem", false, "Update filesystem database") + + flag.BoolVar(&createPackageFlag, "Cp", false, "Create package") + flag.BoolVar(&createPackageFlagLong, "create-package", false, "Create Package") + + flag.Parse() + + if updateFilesytemFlag || updateFilesytemFlagLong { e = Filesystem.UpdateFilesystemHash() if e != nil { panic(e) } - */ - e = Filesystem.GetFilesystemDiff() - if e != nil { - panic(e) + return } + if createPackageFlag || createPackageFlagLong { + e = Package.CreatePackage() + if e != nil { + panic(e) + } + return + } + + flag.Usage() + fmt.Println(Color.Fatal("Nothing to do")) + //e := Archive.TarGzip("/tmp/test", "/tmp/test.tar.gz") //e := Archive.UntarGzip("/tmp/test.tar.gz", "/tmp/test") //fmt.Println(e) diff --git a/Color/Color.go b/Color/Color.go new file mode 100644 index 0000000..3d4a9d2 --- /dev/null +++ b/Color/Color.go @@ -0,0 +1,58 @@ +package Color + +import ( + "fmt" + "regexp" + "runtime" +) + +var ( + Success = Green + Info = Teal + Warning = Yellow + Fatal = Red +) + +var ( + Black = Color("\033[1;30m%s\033[0m") + Red = Color("\033[1;31m%s\033[0m") + Green = Color("\033[1;32m%s\033[0m") + Yellow = Color("\033[1;33m%s\033[0m") + Purple = Color("\033[1;34m%s\033[0m") + Magenta = Color("\033[1;35m%s\033[0m") + Teal = Color("\033[1;36m%s\033[0m") + White = Color("\033[1;37m%s\033[0m") +) + +func init() { + if runtime.GOOS != "windows" { + return + } + + Black = Color("%s") + Red = Color("%s") + Green = Color("%s") + Yellow = Color("%s") + Purple = Color("%s") + Magenta = Color("%s") + Teal = Color("%s") + White = Color("%s") +} + +func Color(colorString string) func(...interface{}) string { + sprint := func(args ...interface{}) string { + return fmt.Sprintf(colorString, + fmt.Sprint(args...)) + } + return sprint +} + +func Strip(s string) string { + var ( + reg *regexp.Regexp + res string + ) + reg = regexp.MustCompile("\\033\\[.{1,4}m") + res = reg.ReplaceAllString(s, "${1}") + return res +} diff --git a/Helper/Input.go b/Helper/Input.go new file mode 100644 index 0000000..369778e --- /dev/null +++ b/Helper/Input.go @@ -0,0 +1,24 @@ +package Helper + +import ( + "bufio" + "os" + "strings" +) + +var ( + reader *bufio.Reader = bufio.NewReader(os.Stdin) +) + +func Input() string { + var ( + text string + e error + ) + text, e = reader.ReadString('\n') + if e != nil { + panic(e) + } + // convert CRLF to LF + return strings.Replace(text, "\n", "", -1) +} diff --git a/package_manager.db b/package_manager.db deleted file mode 100644 index 01f184cd7c6f7a42f1eaab2744c85aa26073fca8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeH~O>f&q5Qa%frlTOX8lXUcoQNI*8+NJv{!lJeWF;{wyQw6n$f0nT zNe=;%Yme=p=s)YN*PeH070x7U7N`|QTA;J|?d zkN^@u0!RP}AOR$R1dsp{00Q%CQ9$?jw`VT<-aYk)CvFPUEXmW%Na_lfI+-e`thJOe ztuvkxo~4C$j4=8+(Qw6#b>Jd;;W9DN9SJU#aI1`_#+rDGSWe<@Z!~_s8U+?aAG_J4 z>elA%g_S%#HJ%s3i9)GNNbRZ4U8c381!q)qnJ-j6Doi^XPtj6a||g+&iB5 zY1uw;lPL&NCS^_y;SvHACimV_s(r4w%W{{Js|6ubmS_VzCAD@b)7OoaC>UKR-d}W^ z?T63+UMwLD;1SUsVun7Q=U-=m1S)1d7dlEbxPBeyMJ}!TqJ~7CoYv*(tlQq0tx!vS#~t}!M6{l z?wti`OJ}Jdne#*lZM|Z-^Nv$*v`?wG>WZ{w2Y^$WK%hb?a;ebf;k)1JT=wr7L^Drq zCWWTFfUH3qirZNwwImuuFaDtqSGVkNV674*xR6Gv0VIF~ skN^@u0!RP}AOR$R1dsp{Kmz|`0xR`KGYD5!YmG1rYxPDHHky_37geoMK>z>% From feaad6a0399187caae8f25ae353206104cd6fd46 Mon Sep 17 00:00:00 2001 From: Tovi Jaeschke-Rogers Date: Fri, 9 Jul 2021 04:46:08 +0930 Subject: [PATCH 03/12] Create packages from filesystem diff --- .gitignore | 2 ++ Archive/Archive.go | 24 ++++------------ Archive/Unarchive.go | 9 ++++-- Client/Filesystem/CopyFile.go | 50 +++++++++++++++++++++++++++++++++ Client/Package/CreatePackage.go | 50 +++++++++++++++++++++++++-------- 5 files changed, 103 insertions(+), 32 deletions(-) create mode 100644 Client/Filesystem/CopyFile.go diff --git a/.gitignore b/.gitignore index 98e6ef6..4762cbb 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ *.db +*.tar +*.tar.gz diff --git a/Archive/Archive.go b/Archive/Archive.go index 8d0e0b3..ef8c24e 100644 --- a/Archive/Archive.go +++ b/Archive/Archive.go @@ -44,7 +44,6 @@ func Tar(source, target string) error { var ( tarfile, file *os.File tarball *tar.Writer - info os.FileInfo header *tar.Header e error ) @@ -58,16 +57,6 @@ func Tar(source, target string) error { tarball = tar.NewWriter(tarfile) defer tarball.Close() - info, e = os.Stat(source) - if e != nil { - return e - } - - var baseDir string - if info.IsDir() { - baseDir = filepath.Base(source) - } - return filepath.Walk(source, func(path string, info os.FileInfo, e error) error { if e != nil { @@ -79,14 +68,11 @@ func Tar(source, target string) error { return e } - if baseDir != "" { - header.Name = filepath.Join( - baseDir, - strings.TrimPrefix( - strings.Join(strings.Split(path, "/")[1:], "/"), - source, - ), - ) + // TODO change "/" to work cross platform + header.Name = strings.TrimPrefix(strings.TrimPrefix(path, source), "/") + + if header.Name == "" { + return nil } e = tarball.WriteHeader(header) diff --git a/Archive/Unarchive.go b/Archive/Unarchive.go index eaa2530..f609d89 100644 --- a/Archive/Unarchive.go +++ b/Archive/Unarchive.go @@ -69,7 +69,7 @@ func Untar(tarball, target string) error { return e } - path = "/" + strings.Join(strings.Split(header.Name, "/")[1:], "/") + path = filepath.Join(target, header.Name) info = header.FileInfo() if info.IsDir() { @@ -107,5 +107,10 @@ func UntarGzip(source, target string) error { return e } - return Untar(tarPath, target) + e = Untar(tarPath, target) + if e != nil { + return e + } + + return os.Remove(tarPath) } diff --git a/Client/Filesystem/CopyFile.go b/Client/Filesystem/CopyFile.go new file mode 100644 index 0000000..57519e8 --- /dev/null +++ b/Client/Filesystem/CopyFile.go @@ -0,0 +1,50 @@ +package Filesystem + +import ( + "fmt" + "io/fs" + "io/ioutil" + "os" + "path/filepath" +) + +func CopyFile(src, dest string) error { + var ( + input []byte + fileInfo fs.FileInfo + srcBasePath string + destBasePath string + e error + ) + + srcBasePath = filepath.Dir(src) + destBasePath = filepath.Dir(dest) + + fileInfo, e = os.Stat(srcBasePath) + if e != nil { + return e + } + + e = os.MkdirAll(destBasePath, fileInfo.Mode()) + if e != nil { + return e + } + + fileInfo, e = os.Stat(src) + if e != nil { + return e + } + + input, e = ioutil.ReadFile(src) + if e != nil { + return e + } + + e = ioutil.WriteFile(dest, input, fileInfo.Mode()) + if e != nil { + fmt.Println(e) + return e + } + + return nil +} diff --git a/Client/Package/CreatePackage.go b/Client/Package/CreatePackage.go index 31ab612..50e878f 100644 --- a/Client/Package/CreatePackage.go +++ b/Client/Package/CreatePackage.go @@ -3,9 +3,13 @@ package Package import ( "errors" "fmt" + "io/ioutil" + "os" + "path/filepath" "strconv" "strings" + "PackageManager/Archive" "PackageManager/Client/Filesystem" "PackageManager/Color" "PackageManager/Helper" @@ -13,17 +17,19 @@ import ( func CreatePackage() error { var ( - dirtyFiles map[int]string - newFiles map[int]string - pkgFiles map[int]string = make(map[int]string) - choices string - choicesSplit []string - filePath string - pkgName string - pkgVersion string - index int - ok bool - e error + dirtyFiles map[int]string + newFiles map[int]string + pkgFiles map[int]string = make(map[int]string) + choices string + choicesSplit []string + filePath string + pkgName string + pkgVersion string + pkgNameVersion string + tmpDir string + index int + ok bool + e error ) fmt.Println("Initialising package creation...") @@ -111,5 +117,27 @@ func CreatePackage() error { return errors.New("User aborted") } + pkgNameVersion = fmt.Sprintf("%s-%s", pkgName, pkgVersion) + + tmpDir, e = ioutil.TempDir("/tmp", pkgNameVersion) + if e != nil { + return e + } + defer os.RemoveAll(tmpDir) + + for _, file := range pkgFiles { + Filesystem.CopyFile(file, filepath.Join(tmpDir, file)) + } + + e = Archive.TarGzip(tmpDir, pkgNameVersion+".tar.gz") + if e != nil { + return e + } + + fmt.Printf( + Color.Green("\nSuccessfully created package %s\n"), + pkgNameVersion, + ) + return nil } From 300fadce37e5c64419dc9b3fce77584ae9836426 Mon Sep 17 00:00:00 2001 From: Tovi Jaeschke-Rogers Date: Fri, 9 Jul 2021 05:25:06 +0930 Subject: [PATCH 04/12] WIP - Add install package --- Client/Package/CreatePackage.go | 2 ++ Client/Package/InstallPackage.go | 29 ++++++++++++++++++++++ Client/main.go | 32 +++++++++++++++++++----- Helper/CheckRoot.go | 42 ++++++++++++++++++++++++++++++++ 4 files changed, 99 insertions(+), 6 deletions(-) create mode 100644 Client/Package/InstallPackage.go create mode 100644 Helper/CheckRoot.go diff --git a/Client/Package/CreatePackage.go b/Client/Package/CreatePackage.go index 50e878f..27e3c2c 100644 --- a/Client/Package/CreatePackage.go +++ b/Client/Package/CreatePackage.go @@ -129,6 +129,8 @@ func CreatePackage() error { Filesystem.CopyFile(file, filepath.Join(tmpDir, file)) } + // TODO: Add dependancy management here + e = Archive.TarGzip(tmpDir, pkgNameVersion+".tar.gz") if e != nil { return e diff --git a/Client/Package/InstallPackage.go b/Client/Package/InstallPackage.go new file mode 100644 index 0000000..136d38a --- /dev/null +++ b/Client/Package/InstallPackage.go @@ -0,0 +1,29 @@ +package Package + +import ( + "errors" + "fmt" + "os" + + "PackageManager/Archive" +) + +func InstallPackage(pkgs []string) error { + var ( + pkg string + e error + ) + + for _, pkg = range pkgs { + _, e = os.Stat(pkg) + if os.IsNotExist(e) { + return errors.New(fmt.Sprintf("Invalid package %s", pkg)) + } + } + + for _, pkg = range pkgs { + e = Archive.UntarGzip(pkg, "/") + } + + return nil +} diff --git a/Client/main.go b/Client/main.go index faec932..b875ccc 100644 --- a/Client/main.go +++ b/Client/main.go @@ -8,18 +8,27 @@ import ( "PackageManager/Client/Filesystem" "PackageManager/Client/Package" "PackageManager/Color" + "PackageManager/Helper" ) func main() { var ( - updateFilesytemFlag bool - updateFilesytemFlagLong bool - createPackageFlag bool - createPackageFlagLong bool + updateFilesytemFlag bool + updateFilesytemFlagLong bool + createPackageFlag bool + createPackageFlagLong bool + installLocalPackageFlag bool + installLocalPackageFlagLong bool e error ) + e = Helper.CheckRoot() + if e != nil { + fmt.Println(Color.Fatal(e)) + return + } + e = Database.InitDB() if e != nil { panic(e) @@ -27,10 +36,13 @@ func main() { // Initialise flags flag.BoolVar(&updateFilesytemFlag, "Uf", false, "Update filesystem database") - flag.BoolVar(&updateFilesytemFlagLong, "update-filesystem", false, "Update filesystem database") + flag.BoolVar(&updateFilesytemFlagLong, "-update-filesystem", false, "Update filesystem database") flag.BoolVar(&createPackageFlag, "Cp", false, "Create package") - flag.BoolVar(&createPackageFlagLong, "create-package", false, "Create Package") + flag.BoolVar(&createPackageFlagLong, "-create-package", false, "Create Package") + + flag.BoolVar(&installLocalPackageFlag, "Il", false, "Install local package") + flag.BoolVar(&installLocalPackageFlagLong, "-install-local-package", false, "Install local Package") flag.Parse() @@ -51,6 +63,14 @@ func main() { return } + if installLocalPackageFlag || installLocalPackageFlagLong { + e = Package.InstallPackage(flag.Args()) + if e != nil { + panic(e) + } + return + } + flag.Usage() fmt.Println(Color.Fatal("Nothing to do")) diff --git a/Helper/CheckRoot.go b/Helper/CheckRoot.go new file mode 100644 index 0000000..925b90c --- /dev/null +++ b/Helper/CheckRoot.go @@ -0,0 +1,42 @@ +package Helper + +import ( + "errors" + "os/exec" + "strconv" +) + +func CheckRoot() error { + var ( + cmd *exec.Cmd + output []byte + i int + e error + ) + + // TODO Make cross platform + + cmd = exec.Command("id", "-u") + output, e = cmd.Output() + + if e != nil { + return e + } + + // output has trailing \n + // need to remove the \n + // otherwise it will cause error for strconv.Atoi + // log.Println(output[:len(output)-1]) + + // 0 = root, 501 = non-root user + i, e = strconv.Atoi(string(output[:len(output)-1])) + if e != nil { + return e + } + + if i != 0 { + return errors.New("Please run as root") + } + + return nil +} From 16d689d5259ff0c26735b0d74f960fc906f92ad6 Mon Sep 17 00:00:00 2001 From: Tovi Jaeschke-Rogers Date: Fri, 9 Jul 2021 05:48:40 +0930 Subject: [PATCH 05/12] Fix new files calculation --- Client/Filesystem/HashFilesystem.go | 13 +++++++++---- Client/Package/CreatePackage.go | 2 +- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/Client/Filesystem/HashFilesystem.go b/Client/Filesystem/HashFilesystem.go index bca43c7..99f937b 100644 --- a/Client/Filesystem/HashFilesystem.go +++ b/Client/Filesystem/HashFilesystem.go @@ -144,14 +144,19 @@ func GetFilesystemDiff() (map[int]string, map[int]string, error) { return dirtyFiles, newFiles, e } - counter = len(dirtyFiles) - for _, file := range dirtyFiles { _, ok = newFilesTmp[file] - if !ok { - newFiles[counter] = file + if ok { + delete(newFilesTmp, file) } } + counter = len(dirtyFiles) + + for _, file := range newFilesTmp { + newFiles[counter] = file + counter++ + } + return dirtyFiles, newFiles, e } diff --git a/Client/Package/CreatePackage.go b/Client/Package/CreatePackage.go index 27e3c2c..26e49b3 100644 --- a/Client/Package/CreatePackage.go +++ b/Client/Package/CreatePackage.go @@ -79,7 +79,7 @@ func CreatePackage() error { } filePath, ok = dirtyFiles[index] if !ok { - filePath, ok = dirtyFiles[index] + filePath, ok = newFiles[index] if !ok { return errors.New("Invalid package selection") } From 1872b493b66e502b991564cfddb623876e6b121e Mon Sep 17 00:00:00 2001 From: Tovi Jaeschke-Rogers Date: Fri, 9 Jul 2021 07:01:57 +0930 Subject: [PATCH 06/12] Add DestDir variable --- Client/Package/InstallPackage.go | 5 ++++- Variables/Variables.go | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/Client/Package/InstallPackage.go b/Client/Package/InstallPackage.go index 136d38a..320f97f 100644 --- a/Client/Package/InstallPackage.go +++ b/Client/Package/InstallPackage.go @@ -6,6 +6,7 @@ import ( "os" "PackageManager/Archive" + "PackageManager/Variables" ) func InstallPackage(pkgs []string) error { @@ -22,7 +23,9 @@ func InstallPackage(pkgs []string) error { } for _, pkg = range pkgs { - e = Archive.UntarGzip(pkg, "/") + fmt.Printf("Installing %s...", pkg) + e = Archive.UntarGzip(pkg, Variables.DestDir) + fmt.Printf("%s successfully installed", pkg) } return nil diff --git a/Variables/Variables.go b/Variables/Variables.go index 8592aed..3808ad0 100644 --- a/Variables/Variables.go +++ b/Variables/Variables.go @@ -1,5 +1,23 @@ package Variables +import ( + "os" +) + const ( DatabaseName string = "package_manager.db" ) + +var ( + DestDir string = "/" +) + +func init() { + var ( + destDir string + ) + destDir = os.Getenv("DESTDIR") + if destDir != "" { + DestDir = destDir + } +} From 7b6a860b32241f2ab1d57bfde4ee213065f2f222 Mon Sep 17 00:00:00 2001 From: Tovi Jaeschke-Rogers Date: Fri, 9 Jul 2021 07:04:38 +0930 Subject: [PATCH 07/12] Add go.mod and go.sum --- go.mod | 5 +++++ go.sum | 2 ++ 2 files changed, 7 insertions(+) create mode 100644 go.mod create mode 100644 go.sum diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..f73a3a7 --- /dev/null +++ b/go.mod @@ -0,0 +1,5 @@ +module PackageManager + +go 1.16 + +require github.com/mattn/go-sqlite3 v1.14.7 diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..96ff824 --- /dev/null +++ b/go.sum @@ -0,0 +1,2 @@ +github.com/mattn/go-sqlite3 v1.14.7 h1:fxWBnXkxfM6sRiuH3bqJ4CfzZojMOLVc0UTsTglEghA= +github.com/mattn/go-sqlite3 v1.14.7/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= From d0f53fd27f3a171d6bfd7ad25384f7d722673311 Mon Sep 17 00:00:00 2001 From: Tovi Jaeschke-Rogers Date: Fri, 9 Jul 2021 07:26:15 +0930 Subject: [PATCH 08/12] Add Variables.InstallDirs for accurately scanning system Containing /bin, /etc, /lib, /lib64, /sbin, /usr --- Client/Filesystem/HashFilesystem.go | 58 ++++++++++++++++------------- Client/Package/InstallPackage.go | 10 ++++- Variables/Variables.go | 3 +- 3 files changed, 43 insertions(+), 28 deletions(-) diff --git a/Client/Filesystem/HashFilesystem.go b/Client/Filesystem/HashFilesystem.go index 99f937b..2a00261 100644 --- a/Client/Filesystem/HashFilesystem.go +++ b/Client/Filesystem/HashFilesystem.go @@ -2,6 +2,7 @@ package Filesystem import ( "PackageManager/Client/Database" + "PackageManager/Variables" "crypto/md5" "fmt" "io/ioutil" @@ -44,43 +45,50 @@ func UpdateFilesystemHash() error { var ( fileHash string rows []Database.FilesystemHashRow + dir string e error ) - e = filepath.Walk(".", func(path string, info os.FileInfo, e error) error { - if e != nil { - return e + for _, dir = range Variables.InstallDirs { + _, e = os.Stat(dir) + if os.IsNotExist(e) { + continue } + e = filepath.Walk(dir, func(path string, info os.FileInfo, e error) error { + if e != nil { + return e + } - // Ignore hidden files - if strings.HasPrefix(info.Name(), ".") || strings.HasPrefix(path, ".") { - return nil - } + // Ignore hidden files + if strings.HasPrefix(info.Name(), ".") || strings.HasPrefix(path, ".") { + return nil + } - // Ignore directories - if info.IsDir() { - return nil - } + // Ignore directories + if info.IsDir() { + return nil + } - fileHash, e = HashFile(path) + fileHash, e = HashFile(path) - if e != nil { - return e - } + if e != nil { + return e + } - rows = append(rows, Database.FilesystemHashRow{ - Path: path, - Hash: fileHash, - UpdatedAt: info.ModTime(), - }) + rows = append(rows, Database.FilesystemHashRow{ + Path: path, + Hash: fileHash, + UpdatedAt: info.ModTime(), + }) - // TODO: If len(rows) > x, update the db and clear out rows, and continue + // TODO: If len(rows) > x, update the db and clear out rows, and continue - return nil - }) + return nil + }) - if e != nil { - panic(e) + if e != nil { + panic(e) + } } return Database.FindOrCreateFileHash(rows) diff --git a/Client/Package/InstallPackage.go b/Client/Package/InstallPackage.go index 320f97f..f514594 100644 --- a/Client/Package/InstallPackage.go +++ b/Client/Package/InstallPackage.go @@ -23,9 +23,15 @@ func InstallPackage(pkgs []string) error { } for _, pkg = range pkgs { - fmt.Printf("Installing %s...", pkg) + fmt.Printf( + "Installing %s...\n", + pkg, + ) e = Archive.UntarGzip(pkg, Variables.DestDir) - fmt.Printf("%s successfully installed", pkg) + fmt.Printf( + "%s successfully installed\n", + pkg, + ) } return nil diff --git a/Variables/Variables.go b/Variables/Variables.go index 3808ad0..f0959cf 100644 --- a/Variables/Variables.go +++ b/Variables/Variables.go @@ -9,7 +9,8 @@ const ( ) var ( - DestDir string = "/" + DestDir string = "/" + InstallDirs []string = []string{"/bin", "/etc", "/lib", "/lib64", "/sbin", "/usr"} ) func init() { From 4686a536543745f74ca8e233db4d999396da53c8 Mon Sep 17 00:00:00 2001 From: Tovi Jaeschke-Rogers Date: Sat, 10 Jul 2021 14:56:17 +0930 Subject: [PATCH 09/12] Add bolt db --- .gitignore | 1 + Client/Database/Filesystem.go | 114 -------------- Client/Database/Init.go | 46 ++++-- Client/Filesystem/CommitFiles.go | 48 ++++++ Client/Filesystem/Config.go | 77 +++++++++ Client/Filesystem/FileObject.go | 211 +++++++++++++++++++++++++ Client/Filesystem/FilesystemDiff.go | 171 ++++++++++++++++++++ Client/Filesystem/HashFilesystem.go | 170 -------------------- Client/Filesystem/ManageFileBucket.go | 31 ++++ Client/Filesystem/PickFiles.go | 88 +++++++++++ Client/Package/CreatePackage.go | 217 ++++++++++++-------------- Client/Package/InstallPackage.go | 2 +- Client/main.go | 89 ++++++++--- Makefile | 14 ++ Variables/Variables.go | 39 ++++- 15 files changed, 879 insertions(+), 439 deletions(-) delete mode 100644 Client/Database/Filesystem.go create mode 100644 Client/Filesystem/CommitFiles.go create mode 100644 Client/Filesystem/Config.go create mode 100644 Client/Filesystem/FileObject.go create mode 100644 Client/Filesystem/FilesystemDiff.go delete mode 100644 Client/Filesystem/HashFilesystem.go create mode 100644 Client/Filesystem/ManageFileBucket.go create mode 100644 Client/Filesystem/PickFiles.go create mode 100644 Makefile diff --git a/.gitignore b/.gitignore index 4762cbb..8b84a56 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ *.db *.tar *.tar.gz +tjpkg diff --git a/Client/Database/Filesystem.go b/Client/Database/Filesystem.go deleted file mode 100644 index 11b56eb..0000000 --- a/Client/Database/Filesystem.go +++ /dev/null @@ -1,114 +0,0 @@ -package Database - -import ( - "database/sql" - "time" -) - -type FilesystemHashRow struct { - Path string - Hash string - UpdatedAt time.Time -} - -func FindOrCreateFileHash(rows []FilesystemHashRow) error { - var ( - stmtString string - stmt *sql.Stmt - values []interface{} = []interface{}{} - e error - ) - - stmtString = "INSERT OR REPLACE INTO filesystem_hash (id, path, hash, updated_at) VALUES " - - for _, row := range rows { - stmtString += `( - (SELECT id FROM filesystem_hash WHERE path = ?), - ?, - ?, - ? - ),` - values = append(values, row.Path, row.Path, row.Hash, row.UpdatedAt.Unix()) - } - - stmtString = stmtString[0 : len(stmtString)-1] - - stmt, e = DB.Prepare(stmtString) - if e != nil { - return e - } - - _, e = stmt.Exec(values...) - - return e -} - -func FindModifiedFiles(hashes []string) (map[int]string, error) { - var ( - stmtString string - stmt *sql.Stmt - values []interface{} = []interface{}{} - result *sql.Rows - dirtyFiles map[int]string = make(map[int]string) - counter int = 0 - e error - ) - - stmtString = "SELECT id, path FROM filesystem_hash WHERE hash NOT IN (" - - for _, row := range hashes { - stmtString += "?," - values = append(values, row) - } - - stmtString = stmtString[0:len(stmtString)-1] + ")" - - stmt, e = DB.Prepare(stmtString) - if e != nil { - return dirtyFiles, e - } - - result, e = stmt.Query(values...) - if e != nil { - return dirtyFiles, e - } - - defer result.Close() - for result.Next() { - var id string - var path string - e = result.Scan(&id, &path) - if e != nil { - return dirtyFiles, e - } - dirtyFiles[counter] = path - counter++ - } - - e = result.Err() - return dirtyFiles, e -} - -func GetMostRecentTimestamp() (time.Time, error) { - var ( - stmt *sql.Stmt - result *sql.Row - lastUpdatedAt int64 - e error - ) - - stmt, e = DB.Prepare(` - SELECT updated_at FROM filesystem_hash - ORDER BY updated_at DESC - LIMIT 1; - `) - if e != nil { - return time.Now(), e - } - - result = stmt.QueryRow() - - result.Scan(&lastUpdatedAt) - - return time.Unix(lastUpdatedAt, 0), nil -} diff --git a/Client/Database/Init.go b/Client/Database/Init.go index 98b2596..7f199a4 100644 --- a/Client/Database/Init.go +++ b/Client/Database/Init.go @@ -2,41 +2,59 @@ package Database import ( "database/sql" + "time" _ "github.com/mattn/go-sqlite3" + bolt "go.etcd.io/bbolt" "PackageManager/Variables" ) var ( - DB *sql.DB + DB *sql.DB + FsDB *bolt.DB ) func init() { var e error + // Initialise sqlite3 database for package versioning DB, e = sql.Open("sqlite3", Variables.DatabaseName) if e != nil { panic(e) } + + // Initialise bolt db for filesystem hashing + FsDB, e = bolt.Open(Variables.FsHashDatabaseName, 0600, &bolt.Options{ + Timeout: 5 * time.Second, + }) + if e != nil { + panic(e) + } } func InitDB() error { var ( - sqlStmt string - e error + tx *bolt.Tx + e error ) - sqlStmt = ` - CREATE TABLE IF NOT EXISTS filesystem_hash ( - id INTEGER NOT NULL PRIMARY KEY, - path VARCHAR(256), - hash VARCHAR(64), - created_at INTEGER DEFAULT CURRENT_TIMESTAMP, - updated_at INTEGER - ) - ` - _, e = DB.Exec(sqlStmt) + + tx, e = FsDB.Begin(true) + if e != nil { + return e + } + defer tx.Rollback() + + _, e = tx.CreateBucketIfNotExists(Variables.FsHashIndexBucket) + if e != nil { + return e + } + + _, e = tx.CreateBucketIfNotExists(Variables.FsHashPicksBucket) if e != nil { return e } - return nil + + e = tx.Commit() + + return e } diff --git a/Client/Filesystem/CommitFiles.go b/Client/Filesystem/CommitFiles.go new file mode 100644 index 0000000..5886fce --- /dev/null +++ b/Client/Filesystem/CommitFiles.go @@ -0,0 +1,48 @@ +package Filesystem + +import ( + "PackageManager/Client/Database" + "PackageManager/Variables" + "fmt" + + bolt "go.etcd.io/bbolt" +) + +func CommitFiles() error { + var ( + fsStatus FilesystemStatus + indexBucket *bolt.Bucket + f string + e error + ) + + fsStatus, e = GetFilesystemDiff(Variables.RootDir) + if e != nil { + return e + } + + e = Database.FsDB.Batch(func(tx *bolt.Tx) error { + indexBucket = tx.Bucket(Variables.FsHashIndexBucket) + + if len(fsStatus.PickedFiles) > 0 { + for _, f = range fsStatus.PickedFiles { + fmt.Println(f) + e = AddFileToBucket(indexBucket, f) + if e != nil { + return e + } + } + + /* + e = tx.DeleteBucket(Variables.FsHashPicksBucket) + if e != nil { + return e + } + */ + } + + return nil + }) + + return e +} diff --git a/Client/Filesystem/Config.go b/Client/Filesystem/Config.go new file mode 100644 index 0000000..fb192c2 --- /dev/null +++ b/Client/Filesystem/Config.go @@ -0,0 +1,77 @@ +package Filesystem + +import ( + "crypto/sha1" + "regexp" + + "PackageManager/Variables" +) + +var ( + sha1Hash = sha1.New() + + PruneRegex []*regexp.Regexp + IgnoreRegex []*regexp.Regexp +) + +func init() { + var e error + + e = InitPruneRegex() + if e != nil { + panic(e) + } + + e = InitIgnoreRegex() + if e != nil { + panic(e) + } +} + +func InitPruneRegex() error { + var ( + r *regexp.Regexp + s string + e error + ) + + for _, s = range Variables.PruneRegexPaths { + r, e = regexp.Compile(s) + if e != nil { + return e + } + PruneRegex = append(PruneRegex, r) + } + return nil +} + +func InitIgnoreRegex() error { + var ( + r *regexp.Regexp + s string + e error + ) + + for _, s = range Variables.IgnoreRegexPaths { + r, e = regexp.Compile(s) + if e != nil { + return e + } + IgnoreRegex = append(IgnoreRegex, r) + } + return nil +} + +func matchAny(p string, a []*regexp.Regexp) bool { + var ( + regex *regexp.Regexp + match bool + ) + for _, regex = range a { + match = regex.MatchString(p) + if match == true { + return true + } + } + return false +} diff --git a/Client/Filesystem/FileObject.go b/Client/Filesystem/FileObject.go new file mode 100644 index 0000000..b7c498b --- /dev/null +++ b/Client/Filesystem/FileObject.go @@ -0,0 +1,211 @@ +package Filesystem + +import ( + "bytes" + "encoding/gob" + "encoding/hex" + "errors" + "fmt" + "io" + "os" + "path/filepath" +) + +var ( + // TODO: Where do I put this + Data string +) + +type Package struct { + Name string + Version string +} + +type FileObject struct { + FileMode os.FileMode + Size int64 + Package Package + Ref string + Sha1 []byte +} + +type ByName []Package + +func (f FileObject) IsLink() bool { + return f.FileMode&os.ModeSymlink != 0 +} + +func (f FileObject) objFile() string { + return filepath.Join(f.objDir(), hex.EncodeToString(f.Sha1)) +} + +func (f FileObject) objDir() string { + return filepath.Join(Data, hex.EncodeToString(f.Sha1[:2])) +} + +func (f FileObject) Reset(dst string) error { + var e error + + if f.IsLink() { + _, e = os.Lstat(dst) + if !os.IsNotExist(e) { + e = os.Remove(dst) + if e != nil { + return e + } + } + + e = os.Symlink(f.Ref, dst) + if e != nil { + return e + } + + return nil + } + + f.cp(f.objFile(), dst) + e = os.Chmod(dst, f.FileMode) + if e != nil { + return e + } + + return nil +} + +func (f FileObject) Stov(src string) error { + var e error + + if f.IsLink() { + return nil + } + + e = os.MkdirAll(f.objDir(), 0744) + if e != nil { + return e + } + + f.cp(src, f.objFile()) + return nil +} + +func (f FileObject) cp(src string, dst string) error { + var ( + srcFile, dstFile *os.File + e error + ) + + fmt.Println("cp ", src, dst) + + srcFile, e = os.Open(src) + if e != nil { + return e + } + defer srcFile.Close() + + dstFile, e = os.Create(dst) + if e != nil { + return e + } + defer dstFile.Close() + + _, e = io.Copy(dstFile, srcFile) + if e != nil { + return e + } + + return dstFile.Sync() +} + +func (f FileObject) IsDifferent(fn FileObject) error { + if f.FileMode != fn.FileMode { + return errors.New("Mode does not match") + } + + if f.IsLink() { + if f.Ref != fn.Ref { + return errors.New("Ref does not match") + } + return nil + } + + if f.Size != fn.Size { + return errors.New("Size does not match") + } + + if bytes.Compare(f.Sha1, fn.Sha1) != 0 { + return errors.New("Sha1 does not match") + } + + return nil +} + +func CreateFileObject(f string) (FileObject, error) { + var ( + fo FileObject + fi os.FileInfo + file *os.File + e error + ) + + fi, e = os.Lstat(f) + if e != nil { + return fo, e + } + + fo = FileObject{ + FileMode: fi.Mode(), + Size: fi.Size(), + } + + if fo.IsLink() { + fo.Ref, e = os.Readlink(f) + if e != nil { + return fo, e + } + return fo, nil + + } + + file, e = os.Open(f) + if e != nil { + return fo, e + } + defer file.Close() + + io.Copy(sha1Hash, file) + fo.Sha1 = sha1Hash.Sum(nil) + + return fo, nil +} + +func (f FileObject) ToBytes() ([]byte, error) { + var ( + encoder *gob.Encoder + buf bytes.Buffer + e error + ) + + encoder = gob.NewEncoder(&buf) + e = encoder.Encode(f) + return buf.Bytes(), e +} + +func FromBytes(v []byte) (FileObject, error) { + var ( + buf *bytes.Buffer + decoder *gob.Decoder + fo FileObject + e error + ) + + buf = bytes.NewBuffer(v) + + decoder = gob.NewDecoder(buf) + + e = decoder.Decode(&fo) + if e != nil { + return fo, e + } + + return fo, nil +} diff --git a/Client/Filesystem/FilesystemDiff.go b/Client/Filesystem/FilesystemDiff.go new file mode 100644 index 0000000..9f49260 --- /dev/null +++ b/Client/Filesystem/FilesystemDiff.go @@ -0,0 +1,171 @@ +package Filesystem + +import ( + "PackageManager/Client/Database" + "PackageManager/Variables" + "fmt" + "log" + "os" + "path/filepath" + + bolt "go.etcd.io/bbolt" +) + +type FilesystemStatus struct { + NewFiles []string + PickedFiles []string + ModifiedFiles []string + MissingFiles []string +} + +func ShowFilesystemDiff(root string) error { + var ( + fsStatus FilesystemStatus + //f string + e error + ) + + fsStatus, e = GetFilesystemDiff(root) + if e != nil { + return e + } + + if len(fsStatus.NewFiles) > 0 { + fmt.Printf("New files: %d\n", len(fsStatus.NewFiles)) + /* + fmt.Println("New files:") + for _, f = range fsStatus.NewFiles { + fmt.Printf("\t%s\n", Color.Green(f)) + } + */ + } + + if len(fsStatus.PickedFiles) > 0 { + fmt.Printf("Picked files: %d\n", len(fsStatus.PickedFiles)) + /* + fmt.Println("Added files:") + for _, f = range fsStatus.PickedFiles { + fmt.Printf("\t%s\n", Color.Green(f)) + } + */ + } + + if len(fsStatus.ModifiedFiles) > 0 { + fmt.Printf("Modified files: %d\n", len(fsStatus.ModifiedFiles)) + /* + fmt.Println("Modified files:") + for _, f = range fsStatus.ModifiedFiles { + fmt.Printf("\t%s\n", Color.Green(f)) + } + */ + } + + if len(fsStatus.MissingFiles) > 0 { + fmt.Printf("Modified files: %d\n", len(fsStatus.MissingFiles)) + /* + fmt.Println("Deleted files:") + for _, f = range fsStatus.MissingFiles { + fmt.Printf("\t%s\n", Color.Green(f)) + } + */ + } + + return nil +} + +func GetFilesystemDiff(root string) (FilesystemStatus, error) { + var ( + fsStatus FilesystemStatus = FilesystemStatus{} + + rootStat os.FileInfo + + picksBucket *bolt.Bucket + indexBucket *bolt.Bucket + + pick, known []byte + + newFileObject FileObject + knownFileObject FileObject + + e error + ) + + rootStat, e = os.Stat(root) + if e != nil { + return fsStatus, e + } + + if rootStat.IsDir() && root[len(root)-1:] != "/" { + root = root + "/" + } + + e = Database.FsDB.View(func(tx *bolt.Tx) error { + + picksBucket = tx.Bucket(Variables.FsHashPicksBucket) + indexBucket = tx.Bucket(Variables.FsHashPicksBucket) + + filepath.Walk(root, func(p string, i os.FileInfo, _ error) error { + + // Ignore path in Variables.PruneRegexPaths + if i.IsDir() && matchAny(p, PruneRegex) { + log.Println("Prune", p) + return filepath.SkipDir + } + + // Ignore path in Variables.IgnoreRegexPaths + if matchAny(p, IgnoreRegex) { + return nil + } + + if !i.Mode().IsRegular() && (i.Mode()&os.ModeSymlink == 0) { + return nil + } + + pick = picksBucket.Get([]byte(p)) + known = indexBucket.Get([]byte(p)) + + if pick != nil { + fsStatus.PickedFiles = append(fsStatus.PickedFiles, p) + return nil + } + + if known != nil { + newFileObject, e = CreateFileObject(p) + if os.IsNotExist(e) { + return nil + } + if e != nil { + return e + } + + knownFileObject, e = FromBytes(known) + if e != nil { + return e + } + + e = newFileObject.IsDifferent(knownFileObject) + if e != nil { + fsStatus.ModifiedFiles = append(fsStatus.ModifiedFiles, p) + } + + return nil + } + + fsStatus.NewFiles = append(fsStatus.NewFiles, p) + + return nil + }) + + indexBucket.ForEach(func(k, v []byte) error { + _, e = os.Lstat(string(k)) + if os.IsNotExist(e) { + fsStatus.MissingFiles = append(fsStatus.MissingFiles, string(k)) + } + return nil + }) + + return nil + }) + + return fsStatus, e +} diff --git a/Client/Filesystem/HashFilesystem.go b/Client/Filesystem/HashFilesystem.go deleted file mode 100644 index 2a00261..0000000 --- a/Client/Filesystem/HashFilesystem.go +++ /dev/null @@ -1,170 +0,0 @@ -package Filesystem - -import ( - "PackageManager/Client/Database" - "PackageManager/Variables" - "crypto/md5" - "fmt" - "io/ioutil" - "os" - "path/filepath" - "strings" - "time" -) - -func HashFile(path string) (string, error) { - var ( - Md5Hash string - hashBytes [16]byte - file *os.File - e error - ) - - file, e = os.Open(path) - if e != nil { - panic(e) - } - - defer file.Close() - - body, e := ioutil.ReadAll(file) - if e != nil { - panic(e) - } - - //Get the 16 bytes hash - hashBytes = md5.Sum(body) - - //Convert the bytes to a string - Md5Hash = fmt.Sprintf("%x", hashBytes) - - return Md5Hash, nil -} - -func UpdateFilesystemHash() error { - var ( - fileHash string - rows []Database.FilesystemHashRow - dir string - e error - ) - - for _, dir = range Variables.InstallDirs { - _, e = os.Stat(dir) - if os.IsNotExist(e) { - continue - } - e = filepath.Walk(dir, func(path string, info os.FileInfo, e error) error { - if e != nil { - return e - } - - // Ignore hidden files - if strings.HasPrefix(info.Name(), ".") || strings.HasPrefix(path, ".") { - return nil - } - - // Ignore directories - if info.IsDir() { - return nil - } - - fileHash, e = HashFile(path) - - if e != nil { - return e - } - - rows = append(rows, Database.FilesystemHashRow{ - Path: path, - Hash: fileHash, - UpdatedAt: info.ModTime(), - }) - - // TODO: If len(rows) > x, update the db and clear out rows, and continue - - return nil - }) - - if e != nil { - panic(e) - } - } - - return Database.FindOrCreateFileHash(rows) -} - -func GetFilesystemDiff() (map[int]string, map[int]string, error) { - var ( - fileHash string - hashes []string = []string{} - lastUpdatedAt time.Time - dirtyFiles map[int]string - newFiles map[int]string = make(map[int]string) - newFilesTmp map[string]string = make(map[string]string) - counter int - ok bool - e error - ) - - lastUpdatedAt, e = Database.GetMostRecentTimestamp() - if e != nil { - return dirtyFiles, newFiles, e - } - - e = filepath.Walk(".", func(path string, info os.FileInfo, e error) error { - if e != nil { - return e - } - - // Ignore hidden files - if strings.HasPrefix(info.Name(), ".") || strings.HasPrefix(path, ".") { - return nil - } - - // Ignore directories - if info.IsDir() { - return nil - } - - fileHash, e = HashFile(path) - - if e != nil { - return e - } - - hashes = append(hashes, fileHash) - - if info.ModTime().After(lastUpdatedAt) { - newFilesTmp[path] = path - } - - // TODO: If len(rows) > x, update the db and clear out rows, and continue - - return nil - }) - if e != nil { - return dirtyFiles, newFiles, e - } - - dirtyFiles, e = Database.FindModifiedFiles(hashes) - if e != nil { - return dirtyFiles, newFiles, e - } - - for _, file := range dirtyFiles { - _, ok = newFilesTmp[file] - if ok { - delete(newFilesTmp, file) - } - } - - counter = len(dirtyFiles) - - for _, file := range newFilesTmp { - newFiles[counter] = file - counter++ - } - - return dirtyFiles, newFiles, e -} diff --git a/Client/Filesystem/ManageFileBucket.go b/Client/Filesystem/ManageFileBucket.go new file mode 100644 index 0000000..8cd907e --- /dev/null +++ b/Client/Filesystem/ManageFileBucket.go @@ -0,0 +1,31 @@ +package Filesystem + +import ( + "os" + + bolt "go.etcd.io/bbolt" +) + +func AddFileToBucket(bucket *bolt.Bucket, filePath string) error { + var ( + fileObject FileObject + fileObjectBytes []byte + e error + ) + fileObject, e = CreateFileObject(filePath) + if os.IsNotExist(e) { + return nil + } + if e != nil { + return e + } + fileObjectBytes, e = fileObject.ToBytes() + if e != nil { + return e + } + return bucket.Put([]byte(filePath), fileObjectBytes) +} + +func RemoveFileFromBucket(bucket *bolt.Bucket, filePath string) error { + return bucket.Delete([]byte(filePath)) +} diff --git a/Client/Filesystem/PickFiles.go b/Client/Filesystem/PickFiles.go new file mode 100644 index 0000000..1c31779 --- /dev/null +++ b/Client/Filesystem/PickFiles.go @@ -0,0 +1,88 @@ +package Filesystem + +import ( + "PackageManager/Client/Database" + "PackageManager/Variables" + + bolt "go.etcd.io/bbolt" +) + +func PickFiles(rootPath string) error { + var ( + fsStatus FilesystemStatus + picksBucket *bolt.Bucket + f string + e error + ) + + fsStatus, e = GetFilesystemDiff(rootPath) + if e != nil { + return e + } + + e = Database.FsDB.Batch(func(tx *bolt.Tx) error { + picksBucket = tx.Bucket(Variables.FsHashPicksBucket) + + if len(fsStatus.NewFiles) > 0 { + for _, f = range fsStatus.NewFiles { + e = AddFileToBucket(picksBucket, f) + if e != nil { + return e + } + } + } + + if len(fsStatus.ModifiedFiles) > 0 { + for _, f = range fsStatus.NewFiles { + e = AddFileToBucket(picksBucket, f) + if e != nil { + return e + } + } + } + + if len(fsStatus.MissingFiles) > 0 { + for _, f = range fsStatus.NewFiles { + e = RemoveFileFromBucket(picksBucket, f) + if e != nil { + return e + } + } + } + + return nil + }) + + return e +} + +func ResetAllPickedFiles() error { + var ( + fsStatus FilesystemStatus + picksBucket *bolt.Bucket + f string + e error + ) + + fsStatus, e = GetFilesystemDiff(Variables.RootDir) + if e != nil { + return e + } + + e = Database.FsDB.Batch(func(tx *bolt.Tx) error { + picksBucket = tx.Bucket(Variables.FsHashPicksBucket) + + if len(fsStatus.PickedFiles) > 0 { + for _, f = range fsStatus.PickedFiles { + e = RemoveFileFromBucket(picksBucket, f) + if e != nil { + return e + } + } + } + + return nil + }) + + return e +} diff --git a/Client/Package/CreatePackage.go b/Client/Package/CreatePackage.go index 26e49b3..6da7a3a 100644 --- a/Client/Package/CreatePackage.go +++ b/Client/Package/CreatePackage.go @@ -1,145 +1,134 @@ package Package -import ( - "errors" - "fmt" - "io/ioutil" - "os" - "path/filepath" - "strconv" - "strings" - - "PackageManager/Archive" - "PackageManager/Client/Filesystem" - "PackageManager/Color" - "PackageManager/Helper" -) - func CreatePackage() error { - var ( - dirtyFiles map[int]string - newFiles map[int]string - pkgFiles map[int]string = make(map[int]string) - choices string - choicesSplit []string - filePath string - pkgName string - pkgVersion string - pkgNameVersion string - tmpDir string - index int - ok bool - e error - ) - - fmt.Println("Initialising package creation...") - - dirtyFiles, newFiles, e = Filesystem.GetFilesystemDiff() - if e != nil { - return e - } - - fmt.Println("\nModified files...") - for i, file := range dirtyFiles { - fmt.Printf( - "\t%d - %s\n", - i, - Color.Red(file), - ) - } + return nil - fmt.Println("\nNew files...") - for i, file := range newFiles { - fmt.Printf( - "\t%d - %s\n", - i, - Color.Red(file), + /* + var ( + dirtyFiles map[int]string + newFiles map[int]string + pkgFiles map[int]string = make(map[int]string) + choices string + choicesSplit []string + filePath string + pkgName string + pkgVersion string + pkgNameVersion string + tmpDir string + index int + ok bool + e error ) - } - fmt.Println("Please select the files you would like to use to create the package. Leave empty for all.") - choices = Helper.Input() + fmt.Println("Initialising package creation...") + + dirtyFiles, newFiles, e = Filesystem.GetFilesystemDiff() + if e != nil { + return e + } - if choices == "" { + fmt.Println("\nModified files...") for i, file := range dirtyFiles { - pkgFiles[i] = file + fmt.Printf( + "\t%d - %s\n", + i, + Color.Red(file), + ) } + + fmt.Println("\nNew files...") for i, file := range newFiles { - pkgFiles[i] = file + fmt.Printf( + "\t%d - %s\n", + i, + Color.Red(file), + ) } - } else { - choicesSplit = strings.Split(choices, ",") + fmt.Println("Please select the files you would like to use to create the package. Leave empty for all.") + choices = Helper.Input() - for _, i := range choicesSplit { - index, e = strconv.Atoi(i) - if e != nil { - // TODO: Handle this error - panic(e) + if choices == "" { + for i, file := range dirtyFiles { + pkgFiles[i] = file + } + for i, file := range newFiles { + pkgFiles[i] = file } - filePath, ok = dirtyFiles[index] - if !ok { - filePath, ok = newFiles[index] + } else { + + choicesSplit = strings.Split(choices, ",") + + for _, i := range choicesSplit { + index, e = strconv.Atoi(i) + if e != nil { + // TODO: Handle this error + panic(e) + } + filePath, ok = dirtyFiles[index] if !ok { - return errors.New("Invalid package selection") + filePath, ok = newFiles[index] + if !ok { + return errors.New("Invalid package selection") + } } + pkgFiles[index] = filePath } - pkgFiles[index] = filePath } - } - fmt.Println("Please enter the package name:") - pkgName = Helper.Input() - if pkgName == "" { - return errors.New("Invalid package name") - } + fmt.Println("Please enter the package name:") + pkgName = Helper.Input() + if pkgName == "" { + return errors.New("Invalid package name") + } - fmt.Println("Please enter the package version:") - pkgVersion = Helper.Input() - if pkgVersion == "" { - return errors.New("Invalid package name") - } + fmt.Println("Please enter the package version:") + pkgVersion = Helper.Input() + if pkgVersion == "" { + return errors.New("Invalid package name") + } - fmt.Printf("Package Name: %s\n", pkgName) - fmt.Printf("Package Version: %s\n", pkgVersion) + fmt.Printf("Package Name: %s\n", pkgName) + fmt.Printf("Package Version: %s\n", pkgVersion) - fmt.Println("Files to be added") - for i, file := range pkgFiles { - fmt.Printf( - "\t%d - %s\n", - i, - Color.Green(file), - ) - } + fmt.Println("Files to be added") + for i, file := range pkgFiles { + fmt.Printf( + "\t%d - %s\n", + i, + Color.Green(file), + ) + } - fmt.Println("Is this correct? [y/N]") - if strings.ToLower(Helper.Input()) != "y" { - return errors.New("User aborted") - } + fmt.Println("Is this correct? [y/N]") + if strings.ToLower(Helper.Input()) != "y" { + return errors.New("User aborted") + } - pkgNameVersion = fmt.Sprintf("%s-%s", pkgName, pkgVersion) + pkgNameVersion = fmt.Sprintf("%s-%s", pkgName, pkgVersion) - tmpDir, e = ioutil.TempDir("/tmp", pkgNameVersion) - if e != nil { - return e - } - defer os.RemoveAll(tmpDir) + tmpDir, e = ioutil.TempDir("/tmp", pkgNameVersion) + if e != nil { + return e + } + defer os.RemoveAll(tmpDir) - for _, file := range pkgFiles { - Filesystem.CopyFile(file, filepath.Join(tmpDir, file)) - } + for _, file := range pkgFiles { + Filesystem.CopyFile(file, filepath.Join(tmpDir, file)) + } - // TODO: Add dependancy management here + // TODO: Add dependancy management here - e = Archive.TarGzip(tmpDir, pkgNameVersion+".tar.gz") - if e != nil { - return e - } + e = Archive.TarGzip(tmpDir, pkgNameVersion+".tar.gz") + if e != nil { + return e + } - fmt.Printf( - Color.Green("\nSuccessfully created package %s\n"), - pkgNameVersion, - ) + fmt.Printf( + Color.Green("\nSuccessfully created package %s\n"), + pkgNameVersion, + ) - return nil + return nil + */ } diff --git a/Client/Package/InstallPackage.go b/Client/Package/InstallPackage.go index f514594..24c4fda 100644 --- a/Client/Package/InstallPackage.go +++ b/Client/Package/InstallPackage.go @@ -27,7 +27,7 @@ func InstallPackage(pkgs []string) error { "Installing %s...\n", pkg, ) - e = Archive.UntarGzip(pkg, Variables.DestDir) + e = Archive.UntarGzip(pkg, Variables.RootDir) fmt.Printf( "%s successfully installed\n", pkg, diff --git a/Client/main.go b/Client/main.go index b875ccc..967d4fb 100644 --- a/Client/main.go +++ b/Client/main.go @@ -6,19 +6,21 @@ import ( "PackageManager/Client/Database" "PackageManager/Client/Filesystem" - "PackageManager/Client/Package" "PackageManager/Color" "PackageManager/Helper" + "PackageManager/Variables" ) func main() { var ( - updateFilesytemFlag bool - updateFilesytemFlagLong bool - createPackageFlag bool - createPackageFlagLong bool - installLocalPackageFlag bool - installLocalPackageFlagLong bool + getFilesystemDiffFlag bool + getFilesystemDiffFlagLong bool + addFileDiffFlag bool + addFileDiffFlagLong bool + commitAddedFilesFlag bool + commitAddedFilesFlagLong bool + resetAddedFilesFlag bool + resetAddedFilesFlagLong bool e error ) @@ -34,20 +36,38 @@ func main() { panic(e) } + // TODO: Rework usage function + // Initialise flags - flag.BoolVar(&updateFilesytemFlag, "Uf", false, "Update filesystem database") - flag.BoolVar(&updateFilesytemFlagLong, "-update-filesystem", false, "Update filesystem database") - flag.BoolVar(&createPackageFlag, "Cp", false, "Create package") - flag.BoolVar(&createPackageFlagLong, "-create-package", false, "Create Package") + flag.BoolVar(&getFilesystemDiffFlag, "Fd", false, "Filesystem diff") + flag.BoolVar(&getFilesystemDiffFlagLong, "fs-diff", false, "Filesystem diff") + + flag.BoolVar(&addFileDiffFlag, "Af", false, "Add files") + flag.BoolVar(&addFileDiffFlagLong, "add-files", false, "Add files") + + flag.BoolVar(&commitAddedFilesFlag, "Cf", false, "Add files") + flag.BoolVar(&commitAddedFilesFlagLong, "commit", false, "Add files") - flag.BoolVar(&installLocalPackageFlag, "Il", false, "Install local package") - flag.BoolVar(&installLocalPackageFlagLong, "-install-local-package", false, "Install local Package") + flag.BoolVar(&resetAddedFilesFlag, "Rf", false, "Reset added files") + flag.BoolVar(&resetAddedFilesFlagLong, "reset", false, "Reset added files") flag.Parse() - if updateFilesytemFlag || updateFilesytemFlagLong { - e = Filesystem.UpdateFilesystemHash() + if getFilesystemDiffFlag || getFilesystemDiffFlagLong { + var rootPath string = Variables.RootDir + if len(flag.Args()) > 1 { + // TODO: Fix this msg + fmt.Println(Color.Fatal("Option takes one optional argument")) + flag.Usage() + return + } + + if len(flag.Args()) == 1 { + rootPath = flag.Arg(0) + } + + e = Filesystem.ShowFilesystemDiff(rootPath) if e != nil { panic(e) } @@ -55,22 +75,53 @@ func main() { return } - if createPackageFlag || createPackageFlagLong { - e = Package.CreatePackage() + if addFileDiffFlag || addFileDiffFlagLong { + if len(flag.Args()) > 1 && len(flag.Args()) < 1 { + fmt.Println(Color.Fatal("Must supply one argument")) + flag.Usage() + return + } + e = Filesystem.PickFiles(flag.Arg(0)) + if e != nil { + panic(e) + } + return + } + + if commitAddedFilesFlag || commitAddedFilesFlagLong { + e = Filesystem.CommitFiles() if e != nil { panic(e) } return } - if installLocalPackageFlag || installLocalPackageFlagLong { - e = Package.InstallPackage(flag.Args()) + if resetAddedFilesFlag || resetAddedFilesFlagLong { + e = Filesystem.ResetAllPickedFiles() if e != nil { panic(e) } return } + /* + if createPackageFlag || createPackageFlagLong { + e = Package.CreatePackage() + if e != nil { + panic(e) + } + return + } + + if installLocalPackageFlag || installLocalPackageFlagLong { + e = Package.InstallPackage(flag.Args()) + if e != nil { + panic(e) + } + return + } + */ + flag.Usage() fmt.Println(Color.Fatal("Nothing to do")) diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..1d594c0 --- /dev/null +++ b/Makefile @@ -0,0 +1,14 @@ +CLIENT_MAIN=Client/main.go + +BUILD=go build -o +BUILD32=env GOARCH=386 go build + +LINUX_LDFLAGS=--ldflags "-s -w -extldflags=-static" + +LINUX_ENV=env GOOS=linux GOARCH=amd64 +LINUX_ENV32=env GOOS=linux GOARCH=386 + +LINUX_OUT=tjpkg + +build: + ${LINUX_ENV} ${BUILD} ${LINUX_OUT} ${CLIENT_MAIN} diff --git a/Variables/Variables.go b/Variables/Variables.go index f0959cf..fd12a60 100644 --- a/Variables/Variables.go +++ b/Variables/Variables.go @@ -5,20 +5,45 @@ import ( ) const ( - DatabaseName string = "package_manager.db" + DatabaseName string = "package_manager.db" + FsHashDatabaseName string = "fs_hash.db" ) var ( - DestDir string = "/" - InstallDirs []string = []string{"/bin", "/etc", "/lib", "/lib64", "/sbin", "/usr"} + RootDir string = "/" + FsHashPicksBucket []byte = []byte("FilesystemPicks") + FsHashIndexBucket []byte = []byte("FilesystemIndex") + + PruneRegexPaths []string = []string{ + "^/\\.git$", + "^/dist$", + "^/boot/grub$", + "^/proc$", + "^/dev$", + "^/mnt$", + "^/sys$", + "^/src$", + "^/root$", + "^/home$", + "^/build$", + "^/tools$", + "/lost\\+found$", + } + + IgnoreRegexPaths []string = []string{ + "^/etc/passwd$", + "^/etc/passwd-$", + "^/etc/group$", + "^/etc/group-$", + } ) func init() { var ( - destDir string + rootDir string ) - destDir = os.Getenv("DESTDIR") - if destDir != "" { - DestDir = destDir + rootDir = os.Getenv("ROOTDIR") + if rootDir != "" { + RootDir = rootDir } } From c6ac8394184f2ed16a3ec978baa5344cb41e2355 Mon Sep 17 00:00:00 2001 From: Tovi Jaeschke-Rogers Date: Sun, 18 Jul 2021 22:00:01 +0930 Subject: [PATCH 10/12] Add progress bar and increase ignored files --- Client/Filesystem/CommitFiles.go | 18 +++--- Client/Filesystem/FilesystemDiff.go | 86 +++++++++++++++++------------ Client/Filesystem/PickFiles.go | 42 +++++++------- Client/Filesystem/Print.go | 24 ++++++++ Client/ProgressBar/Bar.go | 41 ++++++++++++++ Variables/Variables.go | 19 +++++++ 6 files changed, 164 insertions(+), 66 deletions(-) create mode 100644 Client/Filesystem/Print.go create mode 100644 Client/ProgressBar/Bar.go diff --git a/Client/Filesystem/CommitFiles.go b/Client/Filesystem/CommitFiles.go index 5886fce..0b47f19 100644 --- a/Client/Filesystem/CommitFiles.go +++ b/Client/Filesystem/CommitFiles.go @@ -2,9 +2,10 @@ package Filesystem import ( "PackageManager/Client/Database" + "PackageManager/Client/ProgressBar" "PackageManager/Variables" - "fmt" + "github.com/vbauerster/mpb" bolt "go.etcd.io/bbolt" ) @@ -12,6 +13,7 @@ func CommitFiles() error { var ( fsStatus FilesystemStatus indexBucket *bolt.Bucket + bar *mpb.Bar f string e error ) @@ -25,20 +27,20 @@ func CommitFiles() error { indexBucket = tx.Bucket(Variables.FsHashIndexBucket) if len(fsStatus.PickedFiles) > 0 { + bar = ProgressBar.InitBar("Commiting...", len(fsStatus.PickedFiles)) for _, f = range fsStatus.PickedFiles { - fmt.Println(f) e = AddFileToBucket(indexBucket, f) + bar.Increment() if e != nil { return e } } - /* - e = tx.DeleteBucket(Variables.FsHashPicksBucket) - if e != nil { - return e - } - */ + e = tx.DeleteBucket(Variables.FsHashPicksBucket) + if e != nil { + return e + } + ProgressBar.CloseBar(bar) } return nil diff --git a/Client/Filesystem/FilesystemDiff.go b/Client/Filesystem/FilesystemDiff.go index 9f49260..5d023b1 100644 --- a/Client/Filesystem/FilesystemDiff.go +++ b/Client/Filesystem/FilesystemDiff.go @@ -2,6 +2,7 @@ package Filesystem import ( "PackageManager/Client/Database" + "PackageManager/Color" "PackageManager/Variables" "fmt" "log" @@ -30,47 +31,60 @@ func ShowFilesystemDiff(root string) error { return e } - if len(fsStatus.NewFiles) > 0 { - fmt.Printf("New files: %d\n", len(fsStatus.NewFiles)) - /* - fmt.Println("New files:") - for _, f = range fsStatus.NewFiles { - fmt.Printf("\t%s\n", Color.Green(f)) - } - */ - } + fmt.Println("New files:") + PrintFilesOrLength(fsStatus.NewFiles, Color.Green) - if len(fsStatus.PickedFiles) > 0 { - fmt.Printf("Picked files: %d\n", len(fsStatus.PickedFiles)) - /* - fmt.Println("Added files:") - for _, f = range fsStatus.PickedFiles { - fmt.Printf("\t%s\n", Color.Green(f)) - } - */ - } + fmt.Println("Added files:") + PrintFilesOrLength(fsStatus.PickedFiles, Color.Green) - if len(fsStatus.ModifiedFiles) > 0 { - fmt.Printf("Modified files: %d\n", len(fsStatus.ModifiedFiles)) - /* - fmt.Println("Modified files:") - for _, f = range fsStatus.ModifiedFiles { - fmt.Printf("\t%s\n", Color.Green(f)) - } - */ + fmt.Println("Modified files:") + PrintFilesOrLength(fsStatus.ModifiedFiles, Color.Warning) + + fmt.Println("Deleted files:") + PrintFilesOrLength(fsStatus.MissingFiles, Color.Fatal) + + return nil +} + +func GetFilesystemLength(root string) (int, error) { + var ( + rootStat os.FileInfo + fsCount int = 0 + e error + ) + + rootStat, e = os.Stat(root) + if e != nil { + return fsCount, e } - if len(fsStatus.MissingFiles) > 0 { - fmt.Printf("Modified files: %d\n", len(fsStatus.MissingFiles)) - /* - fmt.Println("Deleted files:") - for _, f = range fsStatus.MissingFiles { - fmt.Printf("\t%s\n", Color.Green(f)) - } - */ + if rootStat.IsDir() && root[len(root)-1:] != "/" { + root = root + "/" } - return nil + filepath.Walk(root, func(p string, i os.FileInfo, _ error) error { + + // Ignore path in Variables.PruneRegexPaths + if i.IsDir() && matchAny(p, PruneRegex) { + log.Println("Prune", p) + return filepath.SkipDir + } + + // Ignore path in Variables.IgnoreRegexPaths + if matchAny(p, IgnoreRegex) { + return nil + } + + if !i.Mode().IsRegular() && (i.Mode()&os.ModeSymlink == 0) { + return nil + } + + fsCount++ + + return nil + }) + + return fsCount, e } func GetFilesystemDiff(root string) (FilesystemStatus, error) { @@ -102,7 +116,7 @@ func GetFilesystemDiff(root string) (FilesystemStatus, error) { e = Database.FsDB.View(func(tx *bolt.Tx) error { picksBucket = tx.Bucket(Variables.FsHashPicksBucket) - indexBucket = tx.Bucket(Variables.FsHashPicksBucket) + indexBucket = tx.Bucket(Variables.FsHashIndexBucket) filepath.Walk(root, func(p string, i os.FileInfo, _ error) error { diff --git a/Client/Filesystem/PickFiles.go b/Client/Filesystem/PickFiles.go index 1c31779..fd908a0 100644 --- a/Client/Filesystem/PickFiles.go +++ b/Client/Filesystem/PickFiles.go @@ -2,8 +2,10 @@ package Filesystem import ( "PackageManager/Client/Database" + "PackageManager/Client/ProgressBar" "PackageManager/Variables" + "github.com/vbauerster/mpb" bolt "go.etcd.io/bbolt" ) @@ -11,6 +13,8 @@ func PickFiles(rootPath string) error { var ( fsStatus FilesystemStatus picksBucket *bolt.Bucket + totalLen int + bar *mpb.Bar f string e error ) @@ -20,12 +24,21 @@ func PickFiles(rootPath string) error { return e } + totalLen = len(fsStatus.NewFiles) + len(fsStatus.ModifiedFiles) + len(fsStatus.MissingFiles) + + if totalLen == 0 { + return nil + } + + bar = ProgressBar.InitBar("Adding...", totalLen) + e = Database.FsDB.Batch(func(tx *bolt.Tx) error { picksBucket = tx.Bucket(Variables.FsHashPicksBucket) if len(fsStatus.NewFiles) > 0 { for _, f = range fsStatus.NewFiles { e = AddFileToBucket(picksBucket, f) + bar.Increment() if e != nil { return e } @@ -33,8 +46,9 @@ func PickFiles(rootPath string) error { } if len(fsStatus.ModifiedFiles) > 0 { - for _, f = range fsStatus.NewFiles { + for _, f = range fsStatus.ModifiedFiles { e = AddFileToBucket(picksBucket, f) + bar.Increment() if e != nil { return e } @@ -44,6 +58,7 @@ func PickFiles(rootPath string) error { if len(fsStatus.MissingFiles) > 0 { for _, f = range fsStatus.NewFiles { e = RemoveFileFromBucket(picksBucket, f) + bar.Increment() if e != nil { return e } @@ -53,35 +68,18 @@ func PickFiles(rootPath string) error { return nil }) + ProgressBar.CloseBar(bar) + return e } func ResetAllPickedFiles() error { var ( - fsStatus FilesystemStatus - picksBucket *bolt.Bucket - f string - e error + e error ) - fsStatus, e = GetFilesystemDiff(Variables.RootDir) - if e != nil { - return e - } - e = Database.FsDB.Batch(func(tx *bolt.Tx) error { - picksBucket = tx.Bucket(Variables.FsHashPicksBucket) - - if len(fsStatus.PickedFiles) > 0 { - for _, f = range fsStatus.PickedFiles { - e = RemoveFileFromBucket(picksBucket, f) - if e != nil { - return e - } - } - } - - return nil + return tx.DeleteBucket(Variables.FsHashPicksBucket) }) return e diff --git a/Client/Filesystem/Print.go b/Client/Filesystem/Print.go new file mode 100644 index 0000000..cfeaef0 --- /dev/null +++ b/Client/Filesystem/Print.go @@ -0,0 +1,24 @@ +package Filesystem + +import ( + "fmt" +) + +func PrintFiles(files []string, color func(...interface{}) string) { + var f string + for _, f = range files { + fmt.Printf("\t%s\n", color(f)) + } +} + +func PrintFilesLength(files []string) { + fmt.Printf("\t%d files found\n", len(files)) +} + +func PrintFilesOrLength(files []string, color func(...interface{}) string) { + if len(files) < 25 && len(files) > 0 { + PrintFiles(files, color) + return + } + PrintFilesLength(files) +} diff --git a/Client/ProgressBar/Bar.go b/Client/ProgressBar/Bar.go new file mode 100644 index 0000000..2e0ade7 --- /dev/null +++ b/Client/ProgressBar/Bar.go @@ -0,0 +1,41 @@ +package ProgressBar + +import ( + "sync" + + "github.com/vbauerster/mpb" + "github.com/vbauerster/mpb/decor" +) + +var ( + BarWG sync.WaitGroup + P = mpb.New(mpb.WithWaitGroup(&BarWG)) +) + +func InitBar(name string, total int) *mpb.Bar { + var ( + bar *mpb.Bar + ) + + BarWG.Add(1) + + bar = P.AddBar(int64(total), + mpb.PrependDecorators( + decor.Name(name), + decor.Percentage(decor.WCSyncSpace), + ), + mpb.AppendDecorators( + // replace ETA decorator with "done" message, OnComplete event + decor.OnComplete( + decor.AverageETA(decor.ET_STYLE_GO, decor.WC{W: 4}), "done", + ), + ), + ) + + return bar +} + +func CloseBar(bar *mpb.Bar) { + bar.Abort(false) + BarWG.Done() +} diff --git a/Variables/Variables.go b/Variables/Variables.go index fd12a60..a52aa55 100644 --- a/Variables/Variables.go +++ b/Variables/Variables.go @@ -27,6 +27,21 @@ var ( "^/home$", "^/build$", "^/tools$", + "^/run/user$", + "^/usr/share/zoneinfo-leaps$", + "^/usr/share/zoneinfo$", + "^/usr/share/zsh$", + "^/tmp$", + "^/var/db$", + "^/var/cache$", + "^/var/log$", + "^/var/spool$", + "^/var/lib/texmf$", + "^/var/lib/postgres$", + "^/var/lib/pacman$", + "^/var/lib/NetworkManager$", + "^/var/lib/systemd$", + "^/var/lib/xkb/README.compiled$", "/lost\\+found$", } @@ -35,6 +50,10 @@ var ( "^/etc/passwd-$", "^/etc/group$", "^/etc/group-$", + "^/var/.updated$", + "^/var/lib/mlocate/mlocate.db$", + "^/var/lib/krb5kdc/kdc.conf$", + "^/var/lib/alsa/asound.state$", } ) From a12d1b6d2ecd49078304fd8f139f1da440b05c39 Mon Sep 17 00:00:00 2001 From: Tovi Jaeschke-Rogers Date: Mon, 19 Jul 2021 17:12:14 +0930 Subject: [PATCH 11/12] WIP --- Client/Filesystem/CommitFiles.go | 4 +- Client/Filesystem/Config.go | 3 - Client/Filesystem/FileObject.go | 11 ++- Client/Filesystem/FilesystemDiff.go | 106 ++++++++++++++++---------- Client/Filesystem/ManageFileBucket.go | 2 +- Client/Filesystem/PickFiles.go | 69 ++++++++++++++--- Client/Filesystem/Print.go | 3 +- Client/ProgressBar/Bar.go | 8 +- Client/main.go | 37 ++++++++- Variables/Variables.go | 5 ++ 10 files changed, 176 insertions(+), 72 deletions(-) diff --git a/Client/Filesystem/CommitFiles.go b/Client/Filesystem/CommitFiles.go index 0b47f19..8cb066b 100644 --- a/Client/Filesystem/CommitFiles.go +++ b/Client/Filesystem/CommitFiles.go @@ -29,10 +29,10 @@ func CommitFiles() error { if len(fsStatus.PickedFiles) > 0 { bar = ProgressBar.InitBar("Commiting...", len(fsStatus.PickedFiles)) for _, f = range fsStatus.PickedFiles { - e = AddFileToBucket(indexBucket, f) bar.Increment() + e = AddFileToBucket(indexBucket, f) if e != nil { - return e + return nil } } diff --git a/Client/Filesystem/Config.go b/Client/Filesystem/Config.go index fb192c2..b8fabd5 100644 --- a/Client/Filesystem/Config.go +++ b/Client/Filesystem/Config.go @@ -1,15 +1,12 @@ package Filesystem import ( - "crypto/sha1" "regexp" "PackageManager/Variables" ) var ( - sha1Hash = sha1.New() - PruneRegex []*regexp.Regexp IgnoreRegex []*regexp.Regexp ) diff --git a/Client/Filesystem/FileObject.go b/Client/Filesystem/FileObject.go index b7c498b..0cf1e3e 100644 --- a/Client/Filesystem/FileObject.go +++ b/Client/Filesystem/FileObject.go @@ -2,10 +2,12 @@ package Filesystem import ( "bytes" + "crypto/sha1" "encoding/gob" "encoding/hex" "errors" "fmt" + "hash" "io" "os" "path/filepath" @@ -141,10 +143,11 @@ func (f FileObject) IsDifferent(fn FileObject) error { func CreateFileObject(f string) (FileObject, error) { var ( - fo FileObject - fi os.FileInfo - file *os.File - e error + sha1Hash hash.Hash = sha1.New() + fo FileObject + fi os.FileInfo + file *os.File + e error ) fi, e = os.Lstat(f) diff --git a/Client/Filesystem/FilesystemDiff.go b/Client/Filesystem/FilesystemDiff.go index 5d023b1..73eb41f 100644 --- a/Client/Filesystem/FilesystemDiff.go +++ b/Client/Filesystem/FilesystemDiff.go @@ -1,15 +1,18 @@ package Filesystem import ( - "PackageManager/Client/Database" - "PackageManager/Color" - "PackageManager/Variables" "fmt" - "log" "os" "path/filepath" + "github.com/vbauerster/mpb" + "github.com/zenthangplus/goccm" bolt "go.etcd.io/bbolt" + + "PackageManager/Client/Database" + "PackageManager/Client/ProgressBar" + "PackageManager/Color" + "PackageManager/Variables" ) type FilesystemStatus struct { @@ -66,7 +69,6 @@ func GetFilesystemLength(root string) (int, error) { // Ignore path in Variables.PruneRegexPaths if i.IsDir() && matchAny(p, PruneRegex) { - log.Println("Prune", p) return filepath.SkipDir } @@ -87,6 +89,51 @@ func GetFilesystemLength(root string) (int, error) { return fsCount, e } +func (fsStatus *FilesystemStatus) parseFile(indexBucket, picksBucket *bolt.Bucket, p string, bar *mpb.Bar, c goccm.ConcurrencyManager) { + var ( + newFileObject FileObject + knownFileObject FileObject + pick, known []byte + e error + ) + + defer func() { + bar.Increment() + c.Done() + }() + + pick = picksBucket.Get([]byte(p)) + known = indexBucket.Get([]byte(p)) + + if pick != nil { + fsStatus.PickedFiles = append(fsStatus.PickedFiles, p) + return + } + + if known != nil { + newFileObject, e = CreateFileObject(p) + if e != nil { + return + } + + knownFileObject, e = FromBytes(known) + if e != nil { + return + } + + e = newFileObject.IsDifferent(knownFileObject) + if e != nil { + fsStatus.ModifiedFiles = append(fsStatus.ModifiedFiles, p) + } + + return + } + + fsStatus.NewFiles = append(fsStatus.NewFiles, p) + + return +} + func GetFilesystemDiff(root string) (FilesystemStatus, error) { var ( fsStatus FilesystemStatus = FilesystemStatus{} @@ -96,10 +143,11 @@ func GetFilesystemDiff(root string) (FilesystemStatus, error) { picksBucket *bolt.Bucket indexBucket *bolt.Bucket - pick, known []byte + bar *mpb.Bar - newFileObject FileObject - knownFileObject FileObject + fsCount int + + c goccm.ConcurrencyManager e error ) @@ -113,6 +161,13 @@ func GetFilesystemDiff(root string) (FilesystemStatus, error) { root = root + "/" } + fsCount, e = GetFilesystemLength(root) + if e != nil { + return fsStatus, e + } + + bar = ProgressBar.InitBar("Scanning...", fsCount) + e = Database.FsDB.View(func(tx *bolt.Tx) error { picksBucket = tx.Bucket(Variables.FsHashPicksBucket) @@ -122,7 +177,6 @@ func GetFilesystemDiff(root string) (FilesystemStatus, error) { // Ignore path in Variables.PruneRegexPaths if i.IsDir() && matchAny(p, PruneRegex) { - log.Println("Prune", p) return filepath.SkipDir } @@ -135,37 +189,7 @@ func GetFilesystemDiff(root string) (FilesystemStatus, error) { return nil } - pick = picksBucket.Get([]byte(p)) - known = indexBucket.Get([]byte(p)) - - if pick != nil { - fsStatus.PickedFiles = append(fsStatus.PickedFiles, p) - return nil - } - - if known != nil { - newFileObject, e = CreateFileObject(p) - if os.IsNotExist(e) { - return nil - } - if e != nil { - return e - } - - knownFileObject, e = FromBytes(known) - if e != nil { - return e - } - - e = newFileObject.IsDifferent(knownFileObject) - if e != nil { - fsStatus.ModifiedFiles = append(fsStatus.ModifiedFiles, p) - } - - return nil - } - - fsStatus.NewFiles = append(fsStatus.NewFiles, p) + go fsStatus.parseFile(indexBucket, picksBucket, p, bar, c) return nil }) @@ -178,6 +202,8 @@ func GetFilesystemDiff(root string) (FilesystemStatus, error) { return nil }) + ProgressBar.CloseBar(bar) + return nil }) diff --git a/Client/Filesystem/ManageFileBucket.go b/Client/Filesystem/ManageFileBucket.go index 8cd907e..4a0c8b2 100644 --- a/Client/Filesystem/ManageFileBucket.go +++ b/Client/Filesystem/ManageFileBucket.go @@ -17,7 +17,7 @@ func AddFileToBucket(bucket *bolt.Bucket, filePath string) error { return nil } if e != nil { - return e + return nil } fileObjectBytes, e = fileObject.ToBytes() if e != nil { diff --git a/Client/Filesystem/PickFiles.go b/Client/Filesystem/PickFiles.go index fd908a0..984b7a7 100644 --- a/Client/Filesystem/PickFiles.go +++ b/Client/Filesystem/PickFiles.go @@ -1,20 +1,42 @@ package Filesystem import ( - "PackageManager/Client/Database" - "PackageManager/Client/ProgressBar" - "PackageManager/Variables" + "os" "github.com/vbauerster/mpb" bolt "go.etcd.io/bbolt" + + "PackageManager/Client/Database" + "PackageManager/Client/ProgressBar" + "PackageManager/Variables" ) -func PickFiles(rootPath string) error { +func pickFilesSingle(rootPath string) error { + var ( + indexBucket *bolt.Bucket + picksBucket *bolt.Bucket + e error + ) + e = Database.FsDB.Batch(func(tx *bolt.Tx) error { + indexBucket = tx.Bucket(Variables.FsHashIndexBucket) + picksBucket = tx.Bucket(Variables.FsHashPicksBucket) + + e = AddFileToBucket(picksBucket, rootPath) + if e != nil { + return e + } + return RemoveFileFromBucket(indexBucket, rootPath) + }) + return e +} + +func pickFilesRecursive(rootPath string) error { var ( fsStatus FilesystemStatus + indexBucket *bolt.Bucket picksBucket *bolt.Bucket - totalLen int bar *mpb.Bar + totalLen int f string e error ) @@ -33,34 +55,39 @@ func PickFiles(rootPath string) error { bar = ProgressBar.InitBar("Adding...", totalLen) e = Database.FsDB.Batch(func(tx *bolt.Tx) error { + indexBucket = tx.Bucket(Variables.FsHashIndexBucket) picksBucket = tx.Bucket(Variables.FsHashPicksBucket) if len(fsStatus.NewFiles) > 0 { for _, f = range fsStatus.NewFiles { - e = AddFileToBucket(picksBucket, f) bar.Increment() + e = AddFileToBucket(picksBucket, f) if e != nil { - return e + return nil } } } if len(fsStatus.ModifiedFiles) > 0 { for _, f = range fsStatus.ModifiedFiles { - e = AddFileToBucket(picksBucket, f) bar.Increment() + e = AddFileToBucket(picksBucket, f) if e != nil { - return e + return nil } } } if len(fsStatus.MissingFiles) > 0 { - for _, f = range fsStatus.NewFiles { - e = RemoveFileFromBucket(picksBucket, f) + for _, f = range fsStatus.MissingFiles { bar.Increment() + e = RemoveFileFromBucket(indexBucket, f) if e != nil { - return e + return nil + } + e = RemoveFileFromBucket(picksBucket, f) + if e != nil { + return nil } } } @@ -73,6 +100,24 @@ func PickFiles(rootPath string) error { return e } +func PickFiles(rootPath string) error { + var ( + rootStat os.FileInfo + e error + ) + + rootStat, e = os.Stat(rootPath) + if e != nil { + return e + } + + if !rootStat.IsDir() { + return pickFilesSingle(rootPath) + } + + return pickFilesRecursive(rootPath) +} + func ResetAllPickedFiles() error { var ( e error diff --git a/Client/Filesystem/Print.go b/Client/Filesystem/Print.go index cfeaef0..668cb61 100644 --- a/Client/Filesystem/Print.go +++ b/Client/Filesystem/Print.go @@ -1,6 +1,7 @@ package Filesystem import ( + "PackageManager/Variables" "fmt" ) @@ -16,7 +17,7 @@ func PrintFilesLength(files []string) { } func PrintFilesOrLength(files []string, color func(...interface{}) string) { - if len(files) < 25 && len(files) > 0 { + if (Variables.VerboseOutput && len(files) != 0) || (len(files) < 25 && len(files) > 0) { PrintFiles(files, color) return } diff --git a/Client/ProgressBar/Bar.go b/Client/ProgressBar/Bar.go index 2e0ade7..e60e3cd 100644 --- a/Client/ProgressBar/Bar.go +++ b/Client/ProgressBar/Bar.go @@ -1,15 +1,12 @@ package ProgressBar import ( - "sync" - "github.com/vbauerster/mpb" "github.com/vbauerster/mpb/decor" ) var ( - BarWG sync.WaitGroup - P = mpb.New(mpb.WithWaitGroup(&BarWG)) + P = mpb.New() ) func InitBar(name string, total int) *mpb.Bar { @@ -17,8 +14,6 @@ func InitBar(name string, total int) *mpb.Bar { bar *mpb.Bar ) - BarWG.Add(1) - bar = P.AddBar(int64(total), mpb.PrependDecorators( decor.Name(name), @@ -37,5 +32,4 @@ func InitBar(name string, total int) *mpb.Bar { func CloseBar(bar *mpb.Bar) { bar.Abort(false) - BarWG.Done() } diff --git a/Client/main.go b/Client/main.go index 967d4fb..f820c15 100644 --- a/Client/main.go +++ b/Client/main.go @@ -3,6 +3,7 @@ package main import ( "flag" "fmt" + "os" "PackageManager/Client/Database" "PackageManager/Client/Filesystem" @@ -11,6 +12,29 @@ import ( "PackageManager/Variables" ) +func HelpMsg() { + var helpMsg string + helpMsg = `Usage of %s: + + -Af | -add-files + Add files + + -Cf | -commit + Add files + + -Fd | -fs-diff + Filesystem diff + + -Rf | -reset + Reset added files + + -V | -verbose + Verbose output +` + helpMsg = fmt.Sprintf(helpMsg, os.Args[0]) + fmt.Println(helpMsg) +} + func main() { var ( getFilesystemDiffFlag bool @@ -22,9 +46,14 @@ func main() { resetAddedFilesFlag bool resetAddedFilesFlagLong bool + verboseOutputFlag bool + verboseOutputFlagLong bool + e error ) + flag.Usage = HelpMsg + e = Helper.CheckRoot() if e != nil { fmt.Println(Color.Fatal(e)) @@ -39,6 +68,8 @@ func main() { // TODO: Rework usage function // Initialise flags + flag.BoolVar(&verboseOutputFlag, "V", false, "Verbose output") + flag.BoolVar(&verboseOutputFlagLong, "verbose", false, "Verbose output") flag.BoolVar(&getFilesystemDiffFlag, "Fd", false, "Filesystem diff") flag.BoolVar(&getFilesystemDiffFlagLong, "fs-diff", false, "Filesystem diff") @@ -46,14 +77,16 @@ func main() { flag.BoolVar(&addFileDiffFlag, "Af", false, "Add files") flag.BoolVar(&addFileDiffFlagLong, "add-files", false, "Add files") - flag.BoolVar(&commitAddedFilesFlag, "Cf", false, "Add files") - flag.BoolVar(&commitAddedFilesFlagLong, "commit", false, "Add files") + flag.BoolVar(&commitAddedFilesFlag, "Cf", false, "Commit files") + flag.BoolVar(&commitAddedFilesFlagLong, "commit", false, "Commit files") flag.BoolVar(&resetAddedFilesFlag, "Rf", false, "Reset added files") flag.BoolVar(&resetAddedFilesFlagLong, "reset", false, "Reset added files") flag.Parse() + Variables.VerboseOutput = verboseOutputFlag || verboseOutputFlagLong + if getFilesystemDiffFlag || getFilesystemDiffFlagLong { var rootPath string = Variables.RootDir if len(flag.Args()) > 1 { diff --git a/Variables/Variables.go b/Variables/Variables.go index a52aa55..15482d8 100644 --- a/Variables/Variables.go +++ b/Variables/Variables.go @@ -2,6 +2,7 @@ package Variables import ( "os" + "sync" ) const ( @@ -10,6 +11,9 @@ const ( ) var ( + WG sync.WaitGroup + + VerboseOutput bool = false RootDir string = "/" FsHashPicksBucket []byte = []byte("FilesystemPicks") FsHashIndexBucket []byte = []byte("FilesystemIndex") @@ -46,6 +50,7 @@ var ( } IgnoreRegexPaths []string = []string{ + "^/swapfile$", "^/etc/passwd$", "^/etc/passwd-$", "^/etc/group$", From 35075c741eacea403828fb4a446ff5402f56fa75 Mon Sep 17 00:00:00 2001 From: Tovi Jaeschke-Rogers Date: Mon, 19 Jul 2021 17:54:39 +0930 Subject: [PATCH 12/12] Make fs scan asynchronous --- Client/Filesystem/FilesystemDiff.go | 38 +++++++++++++++++------------ Client/Filesystem/PickFiles.go | 12 ++++----- Variables/Variables.go | 7 ++++-- 3 files changed, 33 insertions(+), 24 deletions(-) diff --git a/Client/Filesystem/FilesystemDiff.go b/Client/Filesystem/FilesystemDiff.go index 73eb41f..64550de 100644 --- a/Client/Filesystem/FilesystemDiff.go +++ b/Client/Filesystem/FilesystemDiff.go @@ -1,13 +1,15 @@ package Filesystem import ( + "context" "fmt" "os" "path/filepath" + "runtime" "github.com/vbauerster/mpb" - "github.com/zenthangplus/goccm" bolt "go.etcd.io/bbolt" + "golang.org/x/sync/semaphore" "PackageManager/Client/Database" "PackageManager/Client/ProgressBar" @@ -89,7 +91,7 @@ func GetFilesystemLength(root string) (int, error) { return fsCount, e } -func (fsStatus *FilesystemStatus) parseFile(indexBucket, picksBucket *bolt.Bucket, p string, bar *mpb.Bar, c goccm.ConcurrencyManager) { +func (fsStatus *FilesystemStatus) parseFile(indexBucket, picksBucket *bolt.Bucket, p string, bar *mpb.Bar) { var ( newFileObject FileObject knownFileObject FileObject @@ -99,7 +101,6 @@ func (fsStatus *FilesystemStatus) parseFile(indexBucket, picksBucket *bolt.Bucke defer func() { bar.Increment() - c.Done() }() pick = picksBucket.Get([]byte(p)) @@ -136,22 +137,20 @@ func (fsStatus *FilesystemStatus) parseFile(indexBucket, picksBucket *bolt.Bucke func GetFilesystemDiff(root string) (FilesystemStatus, error) { var ( - fsStatus FilesystemStatus = FilesystemStatus{} - - rootStat os.FileInfo - + fsStatus FilesystemStatus = FilesystemStatus{} + sem *semaphore.Weighted picksBucket *bolt.Bucket indexBucket *bolt.Bucket - - bar *mpb.Bar - - fsCount int - - c goccm.ConcurrencyManager - - e error + rootStat os.FileInfo + bar *mpb.Bar + fsCount int + poolSize int + e error ) + poolSize = runtime.NumCPU() + sem = semaphore.NewWeighted(int64(poolSize)) + rootStat, e = os.Stat(root) if e != nil { return fsStatus, e @@ -189,7 +188,13 @@ func GetFilesystemDiff(root string) (FilesystemStatus, error) { return nil } - go fsStatus.parseFile(indexBucket, picksBucket, p, bar, c) + Variables.WG.Add(1) + sem.Acquire(context.Background(), 1) + go func() { + fsStatus.parseFile(indexBucket, picksBucket, p, bar) + Variables.WG.Done() + sem.Release(1) + }() return nil }) @@ -202,6 +207,7 @@ func GetFilesystemDiff(root string) (FilesystemStatus, error) { return nil }) + Variables.WG.Wait() ProgressBar.CloseBar(bar) return nil diff --git a/Client/Filesystem/PickFiles.go b/Client/Filesystem/PickFiles.go index 984b7a7..330cf4a 100644 --- a/Client/Filesystem/PickFiles.go +++ b/Client/Filesystem/PickFiles.go @@ -63,7 +63,7 @@ func pickFilesRecursive(rootPath string) error { bar.Increment() e = AddFileToBucket(picksBucket, f) if e != nil { - return nil + return e } } } @@ -73,7 +73,7 @@ func pickFilesRecursive(rootPath string) error { bar.Increment() e = AddFileToBucket(picksBucket, f) if e != nil { - return nil + return e } } } @@ -83,20 +83,20 @@ func pickFilesRecursive(rootPath string) error { bar.Increment() e = RemoveFileFromBucket(indexBucket, f) if e != nil { - return nil + return e } e = RemoveFileFromBucket(picksBucket, f) if e != nil { - return nil + return e } } } + Variables.WG.Wait() + ProgressBar.CloseBar(bar) return nil }) - ProgressBar.CloseBar(bar) - return e } diff --git a/Variables/Variables.go b/Variables/Variables.go index 15482d8..8ab00a0 100644 --- a/Variables/Variables.go +++ b/Variables/Variables.go @@ -31,10 +31,12 @@ var ( "^/home$", "^/build$", "^/tools$", + "^/opt$", "^/run/user$", - "^/usr/share/zoneinfo-leaps$", - "^/usr/share/zoneinfo$", "^/usr/share/zsh$", + "^/usr/share/texmf-dist$", + "^/usr/share/zoneinfo$", + "^/usr/share/zoneinfo-leaps$", "^/tmp$", "^/var/db$", "^/var/cache$", @@ -59,6 +61,7 @@ var ( "^/var/lib/mlocate/mlocate.db$", "^/var/lib/krb5kdc/kdc.conf$", "^/var/lib/alsa/asound.state$", + "^/run/systemd/journal/kernel-seqnum$", } )