From 8154034d5aabcedac227ab5eb75a6b08cca4f4a7 Mon Sep 17 00:00:00 2001 From: Tovi Jaeschke-Rogers Date: Thu, 8 Jul 2021 08:45:47 +0930 Subject: [PATCH] Get filesystem diff --- Client/Database/Filesystem.go | 143 ++++++++++++++++++++++++++++ Client/Database/Init.go | 26 ++++- Client/Filesystem/HashFilesystem.go | 128 +++++++++++++++++++++++++ Client/main.go | 28 +++++- Variables/Variables.go | 5 + package_manager.db | Bin 0 -> 12288 bytes 6 files changed, 321 insertions(+), 9 deletions(-) create mode 100644 Client/Database/Filesystem.go create mode 100644 Client/Filesystem/HashFilesystem.go create mode 100644 Variables/Variables.go create mode 100644 package_manager.db diff --git a/Client/Database/Filesystem.go b/Client/Database/Filesystem.go new file mode 100644 index 0000000..489879d --- /dev/null +++ b/Client/Database/Filesystem.go @@ -0,0 +1,143 @@ +package Database + +import ( + "database/sql" + "fmt" + "time" +) + +type FilesystemHashRow struct { + Path string + Hash string + UpdatedAt time.Time +} + +func FindOrCreateFileHash(rows []FilesystemHashRow) error { + var ( + stmtString string + stmt *sql.Stmt + values []interface{} = []interface{}{} + e error + ) + + stmtString = "INSERT OR REPLACE INTO filesystem_hash (id, path, hash, updated_at) VALUES " + + for _, row := range rows { + stmtString += `( + (SELECT id FROM filesystem_hash WHERE path = ?), + ?, + ?, + ? + ),` + values = append(values, row.Path, row.Path, row.Hash, row.UpdatedAt.String()) + } + + stmtString = stmtString[0 : len(stmtString)-1] + + stmt, e = DB.Prepare(stmtString) + if e != nil { + return e + } + + _, e = stmt.Exec(values...) + + return e +} + +func FindModifiedFiles(hashes []string) ([]string, error) { + var ( + stmtString string + stmt *sql.Stmt + values []interface{} = []interface{}{} + result *sql.Rows + dirtyFiles []string = []string{} + e error + ) + + stmtString = "SELECT id, path FROM filesystem_hash WHERE hash NOT IN (" + + for _, row := range hashes { + stmtString += "?," + values = append(values, row) + } + + stmtString = stmtString[0:len(stmtString)-1] + ")" + + stmt, e = DB.Prepare(stmtString) + if e != nil { + return dirtyFiles, e + } + + result, e = stmt.Query(values...) + if e != nil { + return dirtyFiles, e + } + + defer result.Close() + for result.Next() { + var id string + var hash string + e = result.Scan(&id, &hash) + if e != nil { + return dirtyFiles, e + } + fmt.Println(id, hash) + dirtyFiles = append(dirtyFiles, hash) + } + + e = result.Err() + return dirtyFiles, e +} + +/* +func FindNewFiles(files []string) ([]string, error) { + var ( + stmtString string + stmt *sql.Stmt + values []interface{} = []interface{}{} + result *sql.Rows + dirtyFiles []string = []string{} + e error + ) + + stmtString = `select * +from ( + values (4),(5),(6) +) as v(id) +where not exists (select * + from images i + where i.id = v.id);` + + for _, row := range hashes { + stmtString += "?," + values = append(values, row) + } + + stmtString = stmtString[0:len(stmtString)-1] + ")" + + stmt, e = DB.Prepare(stmtString) + if e != nil { + return dirtyFiles, e + } + + result, e = stmt.Query(values...) + if e != nil { + return dirtyFiles, e + } + + defer result.Close() + for result.Next() { + var id string + var hash string + e = result.Scan(&id, &hash) + if e != nil { + return dirtyFiles, e + } + fmt.Println(id, hash) + dirtyFiles = append(dirtyFiles, hash) + } + + e = result.Err() + return dirtyFiles, e +} +*/ diff --git a/Client/Database/Init.go b/Client/Database/Init.go index 9410e25..1aa7d2c 100644 --- a/Client/Database/Init.go +++ b/Client/Database/Init.go @@ -2,9 +2,10 @@ package Database import ( "database/sql" - "log" _ "github.com/mattn/go-sqlite3" + + "PackageManager/Variables" ) var ( @@ -13,12 +14,29 @@ var ( func init() { var e error - DB, e = sql.Open("sqlite3", "./foo.db") + DB, e = sql.Open("sqlite3", Variables.DatabaseName) if e != nil { panic(e) } } -func InitDB() { - log.Println("Initialising Database...") +func InitDB() error { + var ( + sqlStmt string + e error + ) + sqlStmt = ` + CREATE TABLE IF NOT EXISTS filesystem_hash ( + id INTEGER NOT NULL PRIMARY KEY, + path VARCHAR(256), + hash VARCHAR(64), + created_at DATE DEFAULT CURRENT_TIMESTAMP, + updated_at DATE + ) + ` + _, e = DB.Exec(sqlStmt) + if e != nil { + return e + } + return nil } diff --git a/Client/Filesystem/HashFilesystem.go b/Client/Filesystem/HashFilesystem.go new file mode 100644 index 0000000..262e7c6 --- /dev/null +++ b/Client/Filesystem/HashFilesystem.go @@ -0,0 +1,128 @@ +package Filesystem + +import ( + "PackageManager/Client/Database" + "crypto/md5" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" +) + +func HashFile(path string) (string, error) { + var ( + Md5Hash string + hashBytes [16]byte + file *os.File + e error + ) + + file, e = os.Open(path) + if e != nil { + panic(e) + } + + defer file.Close() + + body, e := ioutil.ReadAll(file) + if e != nil { + panic(e) + } + + //Get the 16 bytes hash + hashBytes = md5.Sum(body) + + //Convert the bytes to a string + Md5Hash = fmt.Sprintf("%x", hashBytes) + + return Md5Hash, nil +} + +func UpdateFilesystemHash() error { + var ( + fileHash string + rows []Database.FilesystemHashRow + e error + ) + + e = filepath.Walk(".", func(path string, info os.FileInfo, e error) error { + if e != nil { + return e + } + + // Ignore hidden files + if strings.HasPrefix(info.Name(), ".") || strings.HasPrefix(path, ".") { + return nil + } + + // Ignore directories + if info.IsDir() { + return nil + } + + fileHash, e = HashFile(path) + + if e != nil { + return e + } + + rows = append(rows, Database.FilesystemHashRow{ + Path: path, + Hash: fileHash, + UpdatedAt: info.ModTime(), + }) + + // TODO: If len(rows) > x, update the db and clear out rows, and continue + + return nil + }) + + if e != nil { + panic(e) + } + + return Database.FindOrCreateFileHash(rows) +} + +func GetFilesystemDiff() error { + var ( + fileHash string + hashes []string = []string{} + e error + ) + + e = filepath.Walk(".", func(path string, info os.FileInfo, e error) error { + if e != nil { + return e + } + + // Ignore hidden files + if strings.HasPrefix(info.Name(), ".") || strings.HasPrefix(path, ".") { + return nil + } + + // Ignore directories + if info.IsDir() { + return nil + } + + fileHash, e = HashFile(path) + + if e != nil { + return e + } + + hashes = append(hashes, fileHash) + + // TODO: If len(rows) > x, update the db and clear out rows, and continue + + return nil + }) + + dirty, e := Database.FindModifiedFiles(hashes) + + fmt.Println(dirty) + + return e +} diff --git a/Client/main.go b/Client/main.go index 4675230..e68248b 100644 --- a/Client/main.go +++ b/Client/main.go @@ -1,13 +1,31 @@ package main import ( - "fmt" - - "PackageManager/Archive" + "PackageManager/Client/Database" + "PackageManager/Client/Filesystem" ) func main() { + var e error + + e = Database.InitDB() + if e != nil { + panic(e) + } + + /* + e = Filesystem.UpdateFilesystemHash() + if e != nil { + panic(e) + } + */ + + e = Filesystem.GetFilesystemDiff() + if e != nil { + panic(e) + } + //e := Archive.TarGzip("/tmp/test", "/tmp/test.tar.gz") - e := Archive.UntarGzip("/tmp/test.tar.gz", "/tmp/test") - fmt.Println(e) + //e := Archive.UntarGzip("/tmp/test.tar.gz", "/tmp/test") + //fmt.Println(e) } diff --git a/Variables/Variables.go b/Variables/Variables.go new file mode 100644 index 0000000..8592aed --- /dev/null +++ b/Variables/Variables.go @@ -0,0 +1,5 @@ +package Variables + +const ( + DatabaseName string = "package_manager.db" +) diff --git a/package_manager.db b/package_manager.db new file mode 100644 index 0000000000000000000000000000000000000000..01f184cd7c6f7a42f1eaab2744c85aa26073fca8 GIT binary patch literal 12288 zcmeH~O>f&q5Qa%frlTOX8lXUcoQNI*8+NJv{!lJeWF;{wyQw6n$f0nT zNe=;%Yme=p=s)YN*PeH070x7U7N`|QTA;J|?d zkN^@u0!RP}AOR$R1dsp{00Q%CQ9$?jw`VT<-aYk)CvFPUEXmW%Na_lfI+-e`thJOe ztuvkxo~4C$j4=8+(Qw6#b>Jd;;W9DN9SJU#aI1`_#+rDGSWe<@Z!~_s8U+?aAG_J4 z>elA%g_S%#HJ%s3i9)GNNbRZ4U8c381!q)qnJ-j6Doi^XPtj6a||g+&iB5 zY1uw;lPL&NCS^_y;SvHACimV_s(r4w%W{{Js|6ubmS_VzCAD@b)7OoaC>UKR-d}W^ z?T63+UMwLD;1SUsVun7Q=U-=m1S)1d7dlEbxPBeyMJ}!TqJ~7CoYv*(tlQq0tx!vS#~t}!M6{l z?wti`OJ}Jdne#*lZM|Z-^Nv$*v`?wG>WZ{w2Y^$WK%hb?a;ebf;k)1JT=wr7L^Drq zCWWTFfUH3qirZNwwImuuFaDtqSGVkNV674*xR6Gv0VIF~ skN^@u0!RP}AOR$R1dsp{Kmz|`0xR`KGYD5!YmG1rYxPDHHky_37geoMK>z>% literal 0 HcmV?d00001