Merge pull request 'feature/add-filesystem-db' (#1) from feature/add-filesystem-db into develop

Reviewed-on: #1

.gitignore

@@ -0,0 +1,4 @@
+*.db
+*.tar
+*.tar.gz
+tjpkg

Archive/Archive.go

@@ -44,7 +44,6 @@ func Tar(source, target string) error {
 	var (
 		tarfile, file *os.File
 		tarball       *tar.Writer
-		info          os.FileInfo
 		header        *tar.Header
 		e             error
 	)
@@ -58,16 +57,6 @@ func Tar(source, target string) error {
 	tarball = tar.NewWriter(tarfile)
 	defer tarball.Close()
 
-	info, e = os.Stat(source)
-	if e != nil {
-		return e
-	}
-
-	var baseDir string
-	if info.IsDir() {
-		baseDir = filepath.Base(source)
-	}
-
 	return filepath.Walk(source,
 		func(path string, info os.FileInfo, e error) error {
 			if e != nil {
@@ -79,14 +68,11 @@ func Tar(source, target string) error {
 				return e
 			}
 
-			if baseDir != "" {
-				header.Name = filepath.Join(
-					baseDir,
-					strings.TrimPrefix(
-						strings.Join(strings.Split(path, "/")[1:], "/"),
-						source,
-					),
-				)
+			// TODO change "/" to work cross platform
+			header.Name = strings.TrimPrefix(strings.TrimPrefix(path, source), "/")
+
+			if header.Name == "" {
+				return nil
 			}
 
 			e = tarball.WriteHeader(header)

Archive/Unarchive.go

@@ -69,7 +69,7 @@ func Untar(tarball, target string) error {
 			return e
 		}
 
-		path = "/" + strings.Join(strings.Split(header.Name, "/")[1:], "/")
+		path = filepath.Join(target, header.Name)
 
 		info = header.FileInfo()
 		if info.IsDir() {
@@ -107,5 +107,10 @@ func UntarGzip(source, target string) error {
 		return e
 	}
 
-	return Untar(tarPath, target)
+	e = Untar(tarPath, target)
+	if e != nil {
+		return e
+	}
+
+	return os.Remove(tarPath)
 }

Client/Database/Init.go

@@ -2,23 +2,59 @@ package Database
 
 import (
 	"database/sql"
-	"log"
+	"time"
 
 	_ "github.com/mattn/go-sqlite3"
+	bolt "go.etcd.io/bbolt"
+
+	"PackageManager/Variables"
 )
 
 var (
-	DB *sql.DB
+	DB   *sql.DB
+	FsDB *bolt.DB
 )
 
 func init() {
 	var e error
-	DB, e = sql.Open("sqlite3", "./foo.db")
+	// Initialise sqlite3 database for package versioning
+	DB, e = sql.Open("sqlite3", Variables.DatabaseName)
+	if e != nil {
+		panic(e)
+	}
+
+	// Initialise bolt db for filesystem hashing
+	FsDB, e = bolt.Open(Variables.FsHashDatabaseName, 0600, &bolt.Options{
+		Timeout: 5 * time.Second,
+	})
 	if e != nil {
 		panic(e)
 	}
 }
 
-func InitDB() {
-	log.Println("Initialising Database...")
+func InitDB() error {
+	var (
+		tx *bolt.Tx
+		e  error
+	)
+
+	tx, e = FsDB.Begin(true)
+	if e != nil {
+		return e
+	}
+	defer tx.Rollback()
+
+	_, e = tx.CreateBucketIfNotExists(Variables.FsHashIndexBucket)
+	if e != nil {
+		return e
+	}
+
+	_, e = tx.CreateBucketIfNotExists(Variables.FsHashPicksBucket)
+	if e != nil {
+		return e
+	}
+
+	e = tx.Commit()
+
+	return e
 }

Client/Filesystem/CommitFiles.go

@@ -0,0 +1,50 @@
+package Filesystem
+
+import (
+	"PackageManager/Client/Database"
+	"PackageManager/Client/ProgressBar"
+	"PackageManager/Variables"
+
+	"github.com/vbauerster/mpb"
+	bolt "go.etcd.io/bbolt"
+)
+
+func CommitFiles() error {
+	var (
+		fsStatus    FilesystemStatus
+		indexBucket *bolt.Bucket
+		bar         *mpb.Bar
+		f           string
+		e           error
+	)
+
+	fsStatus, e = GetFilesystemDiff(Variables.RootDir)
+	if e != nil {
+		return e
+	}
+
+	e = Database.FsDB.Batch(func(tx *bolt.Tx) error {
+		indexBucket = tx.Bucket(Variables.FsHashIndexBucket)
+
+		if len(fsStatus.PickedFiles) > 0 {
+			bar = ProgressBar.InitBar("Commiting...", len(fsStatus.PickedFiles))
+			for _, f = range fsStatus.PickedFiles {
+				bar.Increment()
+				e = AddFileToBucket(indexBucket, f)
+				if e != nil {
+					return nil
+				}
+			}
+
+			e = tx.DeleteBucket(Variables.FsHashPicksBucket)
+			if e != nil {
+				return e
+			}
+			ProgressBar.CloseBar(bar)
+		}
+
+		return nil
+	})
+
+	return e
+}

Client/Filesystem/Config.go

@@ -0,0 +1,74 @@
+package Filesystem
+
+import (
+	"regexp"
+
+	"PackageManager/Variables"
+)
+
+var (
+	PruneRegex  []*regexp.Regexp
+	IgnoreRegex []*regexp.Regexp
+)
+
+func init() {
+	var e error
+
+	e = InitPruneRegex()
+	if e != nil {
+		panic(e)
+	}
+
+	e = InitIgnoreRegex()
+	if e != nil {
+		panic(e)
+	}
+}
+
+func InitPruneRegex() error {
+	var (
+		r *regexp.Regexp
+		s string
+		e error
+	)
+
+	for _, s = range Variables.PruneRegexPaths {
+		r, e = regexp.Compile(s)
+		if e != nil {
+			return e
+		}
+		PruneRegex = append(PruneRegex, r)
+	}
+	return nil
+}
+
+func InitIgnoreRegex() error {
+	var (
+		r *regexp.Regexp
+		s string
+		e error
+	)
+
+	for _, s = range Variables.IgnoreRegexPaths {
+		r, e = regexp.Compile(s)
+		if e != nil {
+			return e
+		}
+		IgnoreRegex = append(IgnoreRegex, r)
+	}
+	return nil
+}
+
+func matchAny(p string, a []*regexp.Regexp) bool {
+	var (
+		regex *regexp.Regexp
+		match bool
+	)
+	for _, regex = range a {
+		match = regex.MatchString(p)
+		if match == true {
+			return true
+		}
+	}
+	return false
+}

Client/Filesystem/CopyFile.go

@@ -0,0 +1,50 @@
+package Filesystem
+
+import (
+	"fmt"
+	"io/fs"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+)
+
+func CopyFile(src, dest string) error {
+	var (
+		input        []byte
+		fileInfo     fs.FileInfo
+		srcBasePath  string
+		destBasePath string
+		e            error
+	)
+
+	srcBasePath = filepath.Dir(src)
+	destBasePath = filepath.Dir(dest)
+
+	fileInfo, e = os.Stat(srcBasePath)
+	if e != nil {
+		return e
+	}
+
+	e = os.MkdirAll(destBasePath, fileInfo.Mode())
+	if e != nil {
+		return e
+	}
+
+	fileInfo, e = os.Stat(src)
+	if e != nil {
+		return e
+	}
+
+	input, e = ioutil.ReadFile(src)
+	if e != nil {
+		return e
+	}
+
+	e = ioutil.WriteFile(dest, input, fileInfo.Mode())
+	if e != nil {
+		fmt.Println(e)
+		return e
+	}
+
+	return nil
+}

Client/Filesystem/FileObject.go

@@ -0,0 +1,214 @@
+package Filesystem
+
+import (
+	"bytes"
+	"crypto/sha1"
+	"encoding/gob"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"hash"
+	"io"
+	"os"
+	"path/filepath"
+)
+
+var (
+	// TODO: Where do I put this
+	Data string
+)
+
+type Package struct {
+	Name    string
+	Version string
+}
+
+type FileObject struct {
+	FileMode os.FileMode
+	Size     int64
+	Package  Package
+	Ref      string
+	Sha1     []byte
+}
+
+type ByName []Package
+
+func (f FileObject) IsLink() bool {
+	return f.FileMode&os.ModeSymlink != 0
+}
+
+func (f FileObject) objFile() string {
+	return filepath.Join(f.objDir(), hex.EncodeToString(f.Sha1))
+}
+
+func (f FileObject) objDir() string {
+	return filepath.Join(Data, hex.EncodeToString(f.Sha1[:2]))
+}
+
+func (f FileObject) Reset(dst string) error {
+	var e error
+
+	if f.IsLink() {
+		_, e = os.Lstat(dst)
+		if !os.IsNotExist(e) {
+			e = os.Remove(dst)
+			if e != nil {
+				return e
+			}
+		}
+
+		e = os.Symlink(f.Ref, dst)
+		if e != nil {
+			return e
+		}
+
+		return nil
+	}
+
+	f.cp(f.objFile(), dst)
+	e = os.Chmod(dst, f.FileMode)
+	if e != nil {
+		return e
+	}
+
+	return nil
+}
+
+func (f FileObject) Stov(src string) error {
+	var e error
+
+	if f.IsLink() {
+		return nil
+	}
+
+	e = os.MkdirAll(f.objDir(), 0744)
+	if e != nil {
+		return e
+	}
+
+	f.cp(src, f.objFile())
+	return nil
+}
+
+func (f FileObject) cp(src string, dst string) error {
+	var (
+		srcFile, dstFile *os.File
+		e                error
+	)
+
+	fmt.Println("cp ", src, dst)
+
+	srcFile, e = os.Open(src)
+	if e != nil {
+		return e
+	}
+	defer srcFile.Close()
+
+	dstFile, e = os.Create(dst)
+	if e != nil {
+		return e
+	}
+	defer dstFile.Close()
+
+	_, e = io.Copy(dstFile, srcFile)
+	if e != nil {
+		return e
+	}
+
+	return dstFile.Sync()
+}
+
+func (f FileObject) IsDifferent(fn FileObject) error {
+	if f.FileMode != fn.FileMode {
+		return errors.New("Mode does not match")
+	}
+
+	if f.IsLink() {
+		if f.Ref != fn.Ref {
+			return errors.New("Ref does not match")
+		}
+		return nil
+	}
+
+	if f.Size != fn.Size {
+		return errors.New("Size does not match")
+	}
+
+	if bytes.Compare(f.Sha1, fn.Sha1) != 0 {
+		return errors.New("Sha1 does not match")
+	}
+
+	return nil
+}
+
+func CreateFileObject(f string) (FileObject, error) {
+	var (
+		sha1Hash hash.Hash = sha1.New()
+		fo       FileObject
+		fi       os.FileInfo
+		file     *os.File
+		e        error
+	)
+
+	fi, e = os.Lstat(f)
+	if e != nil {
+		return fo, e
+	}
+
+	fo = FileObject{
+		FileMode: fi.Mode(),
+		Size:     fi.Size(),
+	}
+
+	if fo.IsLink() {
+		fo.Ref, e = os.Readlink(f)
+		if e != nil {
+			return fo, e
+		}
+		return fo, nil
+
+	}
+
+	file, e = os.Open(f)
+	if e != nil {
+		return fo, e
+	}
+	defer file.Close()
+
+	io.Copy(sha1Hash, file)
+	fo.Sha1 = sha1Hash.Sum(nil)
+
+	return fo, nil
+}
+
+func (f FileObject) ToBytes() ([]byte, error) {
+	var (
+		encoder *gob.Encoder
+		buf     bytes.Buffer
+		e       error
+	)
+
+	encoder = gob.NewEncoder(&buf)
+	e = encoder.Encode(f)
+	return buf.Bytes(), e
+}
+
+func FromBytes(v []byte) (FileObject, error) {
+	var (
+		buf     *bytes.Buffer
+		decoder *gob.Decoder
+		fo      FileObject
+		e       error
+	)
+
+	buf = bytes.NewBuffer(v)
+
+	decoder = gob.NewDecoder(buf)
+
+	e = decoder.Decode(&fo)
+	if e != nil {
+		return fo, e
+	}
+
+	return fo, nil
+}

Client/Filesystem/FilesystemDiff.go

@@ -0,0 +1,217 @@
+package Filesystem
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"runtime"
+
+	"github.com/vbauerster/mpb"
+	bolt "go.etcd.io/bbolt"
+	"golang.org/x/sync/semaphore"
+
+	"PackageManager/Client/Database"
+	"PackageManager/Client/ProgressBar"
+	"PackageManager/Color"
+	"PackageManager/Variables"
+)
+
+type FilesystemStatus struct {
+	NewFiles      []string
+	PickedFiles   []string
+	ModifiedFiles []string
+	MissingFiles  []string
+}
+
+func ShowFilesystemDiff(root string) error {
+	var (
+		fsStatus FilesystemStatus
+		//f        string
+		e error
+	)
+
+	fsStatus, e = GetFilesystemDiff(root)
+	if e != nil {
+		return e
+	}
+
+	fmt.Println("New files:")
+	PrintFilesOrLength(fsStatus.NewFiles, Color.Green)
+
+	fmt.Println("Added files:")
+	PrintFilesOrLength(fsStatus.PickedFiles, Color.Green)
+
+	fmt.Println("Modified files:")
+	PrintFilesOrLength(fsStatus.ModifiedFiles, Color.Warning)
+
+	fmt.Println("Deleted files:")
+	PrintFilesOrLength(fsStatus.MissingFiles, Color.Fatal)
+
+	return nil
+}
+
+func GetFilesystemLength(root string) (int, error) {
+	var (
+		rootStat os.FileInfo
+		fsCount  int = 0
+		e        error
+	)
+
+	rootStat, e = os.Stat(root)
+	if e != nil {
+		return fsCount, e
+	}
+
+	if rootStat.IsDir() && root[len(root)-1:] != "/" {
+		root = root + "/"
+	}
+
+	filepath.Walk(root, func(p string, i os.FileInfo, _ error) error {
+
+		// Ignore path in Variables.PruneRegexPaths
+		if i.IsDir() && matchAny(p, PruneRegex) {
+			return filepath.SkipDir
+		}
+
+		// Ignore path in Variables.IgnoreRegexPaths
+		if matchAny(p, IgnoreRegex) {
+			return nil
+		}
+
+		if !i.Mode().IsRegular() && (i.Mode()&os.ModeSymlink == 0) {
+			return nil
+		}
+
+		fsCount++
+
+		return nil
+	})
+
+	return fsCount, e
+}
+
+func (fsStatus *FilesystemStatus) parseFile(indexBucket, picksBucket *bolt.Bucket, p string, bar *mpb.Bar) {
+	var (
+		newFileObject   FileObject
+		knownFileObject FileObject
+		pick, known     []byte
+		e               error
+	)
+
+	defer func() {
+		bar.Increment()
+	}()
+
+	pick = picksBucket.Get([]byte(p))
+	known = indexBucket.Get([]byte(p))
+
+	if pick != nil {
+		fsStatus.PickedFiles = append(fsStatus.PickedFiles, p)
+		return
+	}
+
+	if known != nil {
+		newFileObject, e = CreateFileObject(p)
+		if e != nil {
+			return
+		}
+
+		knownFileObject, e = FromBytes(known)
+		if e != nil {
+			return
+		}
+
+		e = newFileObject.IsDifferent(knownFileObject)
+		if e != nil {
+			fsStatus.ModifiedFiles = append(fsStatus.ModifiedFiles, p)
+		}
+
+		return
+	}
+
+	fsStatus.NewFiles = append(fsStatus.NewFiles, p)
+
+	return
+}
+
+func GetFilesystemDiff(root string) (FilesystemStatus, error) {
+	var (
+		fsStatus    FilesystemStatus = FilesystemStatus{}
+		sem         *semaphore.Weighted
+		picksBucket *bolt.Bucket
+		indexBucket *bolt.Bucket
+		rootStat    os.FileInfo
+		bar         *mpb.Bar
+		fsCount     int
+		poolSize    int
+		e           error
+	)
+
+	poolSize = runtime.NumCPU()
+	sem = semaphore.NewWeighted(int64(poolSize))
+
+	rootStat, e = os.Stat(root)
+	if e != nil {
+		return fsStatus, e
+	}
+
+	if rootStat.IsDir() && root[len(root)-1:] != "/" {
+		root = root + "/"
+	}
+
+	fsCount, e = GetFilesystemLength(root)
+	if e != nil {
+		return fsStatus, e
+	}
+
+	bar = ProgressBar.InitBar("Scanning...", fsCount)
+
+	e = Database.FsDB.View(func(tx *bolt.Tx) error {
+
+		picksBucket = tx.Bucket(Variables.FsHashPicksBucket)
+		indexBucket = tx.Bucket(Variables.FsHashIndexBucket)
+
+		filepath.Walk(root, func(p string, i os.FileInfo, _ error) error {
+
+			// Ignore path in Variables.PruneRegexPaths
+			if i.IsDir() && matchAny(p, PruneRegex) {
+				return filepath.SkipDir
+			}
+
+			// Ignore path in Variables.IgnoreRegexPaths
+			if matchAny(p, IgnoreRegex) {
+				return nil
+			}
+
+			if !i.Mode().IsRegular() && (i.Mode()&os.ModeSymlink == 0) {
+				return nil
+			}
+
+			Variables.WG.Add(1)
+			sem.Acquire(context.Background(), 1)
+			go func() {
+				fsStatus.parseFile(indexBucket, picksBucket, p, bar)
+				Variables.WG.Done()
+				sem.Release(1)
+			}()
+
+			return nil
+		})
+
+		indexBucket.ForEach(func(k, v []byte) error {
+			_, e = os.Lstat(string(k))
+			if os.IsNotExist(e) {
+				fsStatus.MissingFiles = append(fsStatus.MissingFiles, string(k))
+			}
+			return nil
+		})
+
+		Variables.WG.Wait()
+		ProgressBar.CloseBar(bar)
+
+		return nil
+	})
+
+	return fsStatus, e
+}

Client/Filesystem/ManageFileBucket.go

@@ -0,0 +1,31 @@
+package Filesystem
+
+import (
+	"os"
+
+	bolt "go.etcd.io/bbolt"
+)
+
+func AddFileToBucket(bucket *bolt.Bucket, filePath string) error {
+	var (
+		fileObject      FileObject
+		fileObjectBytes []byte
+		e               error
+	)
+	fileObject, e = CreateFileObject(filePath)
+	if os.IsNotExist(e) {
+		return nil
+	}
+	if e != nil {
+		return nil
+	}
+	fileObjectBytes, e = fileObject.ToBytes()
+	if e != nil {
+		return e
+	}
+	return bucket.Put([]byte(filePath), fileObjectBytes)
+}
+
+func RemoveFileFromBucket(bucket *bolt.Bucket, filePath string) error {
+	return bucket.Delete([]byte(filePath))
+}

Client/Filesystem/PickFiles.go

@@ -0,0 +1,131 @@
+package Filesystem
+
+import (
+	"os"
+
+	"github.com/vbauerster/mpb"
+	bolt "go.etcd.io/bbolt"
+
+	"PackageManager/Client/Database"
+	"PackageManager/Client/ProgressBar"
+	"PackageManager/Variables"
+)
+
+func pickFilesSingle(rootPath string) error {
+	var (
+		indexBucket *bolt.Bucket
+		picksBucket *bolt.Bucket
+		e           error
+	)
+	e = Database.FsDB.Batch(func(tx *bolt.Tx) error {
+		indexBucket = tx.Bucket(Variables.FsHashIndexBucket)
+		picksBucket = tx.Bucket(Variables.FsHashPicksBucket)
+
+		e = AddFileToBucket(picksBucket, rootPath)
+		if e != nil {
+			return e
+		}
+		return RemoveFileFromBucket(indexBucket, rootPath)
+	})
+	return e
+}
+
+func pickFilesRecursive(rootPath string) error {
+	var (
+		fsStatus    FilesystemStatus
+		indexBucket *bolt.Bucket
+		picksBucket *bolt.Bucket
+		bar         *mpb.Bar
+		totalLen    int
+		f           string
+		e           error
+	)
+
+	fsStatus, e = GetFilesystemDiff(rootPath)
+	if e != nil {
+		return e
+	}
+
+	totalLen = len(fsStatus.NewFiles) + len(fsStatus.ModifiedFiles) + len(fsStatus.MissingFiles)
+
+	if totalLen == 0 {
+		return nil
+	}
+
+	bar = ProgressBar.InitBar("Adding...", totalLen)
+
+	e = Database.FsDB.Batch(func(tx *bolt.Tx) error {
+		indexBucket = tx.Bucket(Variables.FsHashIndexBucket)
+		picksBucket = tx.Bucket(Variables.FsHashPicksBucket)
+
+		if len(fsStatus.NewFiles) > 0 {
+			for _, f = range fsStatus.NewFiles {
+				bar.Increment()
+				e = AddFileToBucket(picksBucket, f)
+				if e != nil {
+					return e
+				}
+			}
+		}
+
+		if len(fsStatus.ModifiedFiles) > 0 {
+			for _, f = range fsStatus.ModifiedFiles {
+				bar.Increment()
+				e = AddFileToBucket(picksBucket, f)
+				if e != nil {
+					return e
+				}
+			}
+		}
+
+		if len(fsStatus.MissingFiles) > 0 {
+			for _, f = range fsStatus.MissingFiles {
+				bar.Increment()
+				e = RemoveFileFromBucket(indexBucket, f)
+				if e != nil {
+					return e
+				}
+				e = RemoveFileFromBucket(picksBucket, f)
+				if e != nil {
+					return e
+				}
+			}
+		}
+
+		Variables.WG.Wait()
+		ProgressBar.CloseBar(bar)
+		return nil
+	})
+
+	return e
+}
+
+func PickFiles(rootPath string) error {
+	var (
+		rootStat os.FileInfo
+		e        error
+	)
+
+	rootStat, e = os.Stat(rootPath)
+	if e != nil {
+		return e
+	}
+
+	if !rootStat.IsDir() {
+		return pickFilesSingle(rootPath)
+	}
+
+	return pickFilesRecursive(rootPath)
+}
+
+func ResetAllPickedFiles() error {
+	var (
+		e error
+	)
+
+	e = Database.FsDB.Batch(func(tx *bolt.Tx) error {
+		return tx.DeleteBucket(Variables.FsHashPicksBucket)
+	})
+
+	return e
+}

Client/Filesystem/Print.go

@@ -0,0 +1,25 @@
+package Filesystem
+
+import (
+	"PackageManager/Variables"
+	"fmt"
+)
+
+func PrintFiles(files []string, color func(...interface{}) string) {
+	var f string
+	for _, f = range files {
+		fmt.Printf("\t%s\n", color(f))
+	}
+}
+
+func PrintFilesLength(files []string) {
+	fmt.Printf("\t%d files found\n", len(files))
+}
+
+func PrintFilesOrLength(files []string, color func(...interface{}) string) {
+	if (Variables.VerboseOutput && len(files) != 0) || (len(files) < 25 && len(files) > 0) {
+		PrintFiles(files, color)
+		return
+	}
+	PrintFilesLength(files)
+}

Client/Package/CreatePackage.go

@@ -0,0 +1,134 @@
+package Package
+
+func CreatePackage() error {
+	return nil
+
+	/*
+		var (
+			dirtyFiles     map[int]string
+			newFiles       map[int]string
+			pkgFiles       map[int]string = make(map[int]string)
+			choices        string
+			choicesSplit   []string
+			filePath       string
+			pkgName        string
+			pkgVersion     string
+			pkgNameVersion string
+			tmpDir         string
+			index          int
+			ok             bool
+			e              error
+		)
+
+		fmt.Println("Initialising package creation...")
+
+		dirtyFiles, newFiles, e = Filesystem.GetFilesystemDiff()
+		if e != nil {
+			return e
+		}
+
+		fmt.Println("\nModified files...")
+		for i, file := range dirtyFiles {
+			fmt.Printf(
+				"\t%d - %s\n",
+				i,
+				Color.Red(file),
+			)
+		}
+
+		fmt.Println("\nNew files...")
+		for i, file := range newFiles {
+			fmt.Printf(
+				"\t%d - %s\n",
+				i,
+				Color.Red(file),
+			)
+		}
+
+		fmt.Println("Please select the files you would like to use to create the package. Leave empty for all.")
+		choices = Helper.Input()
+
+		if choices == "" {
+			for i, file := range dirtyFiles {
+				pkgFiles[i] = file
+			}
+			for i, file := range newFiles {
+				pkgFiles[i] = file
+			}
+		} else {
+
+			choicesSplit = strings.Split(choices, ",")
+
+			for _, i := range choicesSplit {
+				index, e = strconv.Atoi(i)
+				if e != nil {
+					// TODO: Handle this error
+					panic(e)
+				}
+				filePath, ok = dirtyFiles[index]
+				if !ok {
+					filePath, ok = newFiles[index]
+					if !ok {
+						return errors.New("Invalid package selection")
+					}
+				}
+				pkgFiles[index] = filePath
+			}
+		}
+
+		fmt.Println("Please enter the package name:")
+		pkgName = Helper.Input()
+		if pkgName == "" {
+			return errors.New("Invalid package name")
+		}
+
+		fmt.Println("Please enter the package version:")
+		pkgVersion = Helper.Input()
+		if pkgVersion == "" {
+			return errors.New("Invalid package name")
+		}
+
+		fmt.Printf("Package Name: %s\n", pkgName)
+		fmt.Printf("Package Version: %s\n", pkgVersion)
+
+		fmt.Println("Files to be added")
+		for i, file := range pkgFiles {
+			fmt.Printf(
+				"\t%d - %s\n",
+				i,
+				Color.Green(file),
+			)
+		}
+
+		fmt.Println("Is this correct? [y/N]")
+		if strings.ToLower(Helper.Input()) != "y" {
+			return errors.New("User aborted")
+		}
+
+		pkgNameVersion = fmt.Sprintf("%s-%s", pkgName, pkgVersion)
+
+		tmpDir, e = ioutil.TempDir("/tmp", pkgNameVersion)
+		if e != nil {
+			return e
+		}
+		defer os.RemoveAll(tmpDir)
+
+		for _, file := range pkgFiles {
+			Filesystem.CopyFile(file, filepath.Join(tmpDir, file))
+		}
+
+		// TODO: Add dependancy management here
+
+		e = Archive.TarGzip(tmpDir, pkgNameVersion+".tar.gz")
+		if e != nil {
+			return e
+		}
+
+		fmt.Printf(
+			Color.Green("\nSuccessfully created package %s\n"),
+			pkgNameVersion,
+		)
+
+		return nil
+	*/
+}

Client/Package/InstallPackage.go

@@ -0,0 +1,38 @@
+package Package
+
+import (
+	"errors"
+	"fmt"
+	"os"
+
+	"PackageManager/Archive"
+	"PackageManager/Variables"
+)
+
+func InstallPackage(pkgs []string) error {
+	var (
+		pkg string
+		e   error
+	)
+
+	for _, pkg = range pkgs {
+		_, e = os.Stat(pkg)
+		if os.IsNotExist(e) {
+			return errors.New(fmt.Sprintf("Invalid package %s", pkg))
+		}
+	}
+
+	for _, pkg = range pkgs {
+		fmt.Printf(
+			"Installing %s...\n",
+			pkg,
+		)
+		e = Archive.UntarGzip(pkg, Variables.RootDir)
+		fmt.Printf(
+			"%s successfully installed\n",
+			pkg,
+		)
+	}
+
+	return nil
+}

Client/ProgressBar/Bar.go

@@ -0,0 +1,35 @@
+package ProgressBar
+
+import (
+	"github.com/vbauerster/mpb"
+	"github.com/vbauerster/mpb/decor"
+)
+
+var (
+	P = mpb.New()
+)
+
+func InitBar(name string, total int) *mpb.Bar {
+	var (
+		bar *mpb.Bar
+	)
+
+	bar = P.AddBar(int64(total),
+		mpb.PrependDecorators(
+			decor.Name(name),
+			decor.Percentage(decor.WCSyncSpace),
+		),
+		mpb.AppendDecorators(
+			// replace ETA decorator with "done" message, OnComplete event
+			decor.OnComplete(
+				decor.AverageETA(decor.ET_STYLE_GO, decor.WC{W: 4}), "done",
+			),
+		),
+	)
+
+	return bar
+}
+
+func CloseBar(bar *mpb.Bar) {
+	bar.Abort(false)
+}

Client/main.go

@@ -1,13 +1,164 @@
 package main
 
 import (
+	"flag"
 	"fmt"
+	"os"
 
-	"PackageManager/Archive"
+	"PackageManager/Client/Database"
+	"PackageManager/Client/Filesystem"
+	"PackageManager/Color"
+	"PackageManager/Helper"
+	"PackageManager/Variables"
 )
 
+func HelpMsg() {
+	var helpMsg string
+	helpMsg = `Usage of %s:
+
+	-Af | -add-files
+		Add files
+
+	-Cf | -commit
+		Add files
+
+	-Fd | -fs-diff
+		Filesystem diff
+
+	-Rf | -reset
+		Reset added files
+
+	-V | -verbose
+		Verbose output
+`
+	helpMsg = fmt.Sprintf(helpMsg, os.Args[0])
+	fmt.Println(helpMsg)
+}
+
 func main() {
+	var (
+		getFilesystemDiffFlag     bool
+		getFilesystemDiffFlagLong bool
+		addFileDiffFlag           bool
+		addFileDiffFlagLong       bool
+		commitAddedFilesFlag      bool
+		commitAddedFilesFlagLong  bool
+		resetAddedFilesFlag       bool
+		resetAddedFilesFlagLong   bool
+
+		verboseOutputFlag     bool
+		verboseOutputFlagLong bool
+
+		e error
+	)
+
+	flag.Usage = HelpMsg
+
+	e = Helper.CheckRoot()
+	if e != nil {
+		fmt.Println(Color.Fatal(e))
+		return
+	}
+
+	e = Database.InitDB()
+	if e != nil {
+		panic(e)
+	}
+
+	// TODO: Rework usage function
+
+	// Initialise flags
+	flag.BoolVar(&verboseOutputFlag, "V", false, "Verbose output")
+	flag.BoolVar(&verboseOutputFlagLong, "verbose", false, "Verbose output")
+
+	flag.BoolVar(&getFilesystemDiffFlag, "Fd", false, "Filesystem diff")
+	flag.BoolVar(&getFilesystemDiffFlagLong, "fs-diff", false, "Filesystem diff")
+
+	flag.BoolVar(&addFileDiffFlag, "Af", false, "Add files")
+	flag.BoolVar(&addFileDiffFlagLong, "add-files", false, "Add files")
+
+	flag.BoolVar(&commitAddedFilesFlag, "Cf", false, "Commit files")
+	flag.BoolVar(&commitAddedFilesFlagLong, "commit", false, "Commit files")
+
+	flag.BoolVar(&resetAddedFilesFlag, "Rf", false, "Reset added files")
+	flag.BoolVar(&resetAddedFilesFlagLong, "reset", false, "Reset added files")
+
+	flag.Parse()
+
+	Variables.VerboseOutput = verboseOutputFlag || verboseOutputFlagLong
+
+	if getFilesystemDiffFlag || getFilesystemDiffFlagLong {
+		var rootPath string = Variables.RootDir
+		if len(flag.Args()) > 1 {
+			// TODO: Fix this msg
+			fmt.Println(Color.Fatal("Option takes one optional argument"))
+			flag.Usage()
+			return
+		}
+
+		if len(flag.Args()) == 1 {
+			rootPath = flag.Arg(0)
+		}
+
+		e = Filesystem.ShowFilesystemDiff(rootPath)
+		if e != nil {
+			panic(e)
+		}
+
+		return
+	}
+
+	if addFileDiffFlag || addFileDiffFlagLong {
+		if len(flag.Args()) > 1 && len(flag.Args()) < 1 {
+			fmt.Println(Color.Fatal("Must supply one argument"))
+			flag.Usage()
+			return
+		}
+		e = Filesystem.PickFiles(flag.Arg(0))
+		if e != nil {
+			panic(e)
+		}
+		return
+	}
+
+	if commitAddedFilesFlag || commitAddedFilesFlagLong {
+		e = Filesystem.CommitFiles()
+		if e != nil {
+			panic(e)
+		}
+		return
+	}
+
+	if resetAddedFilesFlag || resetAddedFilesFlagLong {
+		e = Filesystem.ResetAllPickedFiles()
+		if e != nil {
+			panic(e)
+		}
+		return
+	}
+
+	/*
+		if createPackageFlag || createPackageFlagLong {
+			e = Package.CreatePackage()
+			if e != nil {
+				panic(e)
+			}
+			return
+		}
+
+		if installLocalPackageFlag || installLocalPackageFlagLong {
+			e = Package.InstallPackage(flag.Args())
+			if e != nil {
+				panic(e)
+			}
+			return
+		}
+	*/
+
+	flag.Usage()
+	fmt.Println(Color.Fatal("Nothing to do"))
+
 	//e := Archive.TarGzip("/tmp/test", "/tmp/test.tar.gz")
-	e := Archive.UntarGzip("/tmp/test.tar.gz", "/tmp/test")
-	fmt.Println(e)
+	//e := Archive.UntarGzip("/tmp/test.tar.gz", "/tmp/test")
+	//fmt.Println(e)
 }

Color/Color.go

@@ -0,0 +1,58 @@
+package Color
+
+import (
+	"fmt"
+	"regexp"
+	"runtime"
+)
+
+var (
+	Success = Green
+	Info    = Teal
+	Warning = Yellow
+	Fatal   = Red
+)
+
+var (
+	Black   = Color("\033[1;30m%s\033[0m")
+	Red     = Color("\033[1;31m%s\033[0m")
+	Green   = Color("\033[1;32m%s\033[0m")
+	Yellow  = Color("\033[1;33m%s\033[0m")
+	Purple  = Color("\033[1;34m%s\033[0m")
+	Magenta = Color("\033[1;35m%s\033[0m")
+	Teal    = Color("\033[1;36m%s\033[0m")
+	White   = Color("\033[1;37m%s\033[0m")
+)
+
+func init() {
+	if runtime.GOOS != "windows" {
+		return
+	}
+
+	Black = Color("%s")
+	Red = Color("%s")
+	Green = Color("%s")
+	Yellow = Color("%s")
+	Purple = Color("%s")
+	Magenta = Color("%s")
+	Teal = Color("%s")
+	White = Color("%s")
+}
+
+func Color(colorString string) func(...interface{}) string {
+	sprint := func(args ...interface{}) string {
+		return fmt.Sprintf(colorString,
+			fmt.Sprint(args...))
+	}
+	return sprint
+}
+
+func Strip(s string) string {
+	var (
+		reg *regexp.Regexp
+		res string
+	)
+	reg = regexp.MustCompile("\\033\\[.{1,4}m")
+	res = reg.ReplaceAllString(s, "${1}")
+	return res
+}

Helper/CheckRoot.go

@@ -0,0 +1,42 @@
+package Helper
+
+import (
+	"errors"
+	"os/exec"
+	"strconv"
+)
+
+func CheckRoot() error {
+	var (
+		cmd    *exec.Cmd
+		output []byte
+		i      int
+		e      error
+	)
+
+	// TODO Make cross platform
+
+	cmd = exec.Command("id", "-u")
+	output, e = cmd.Output()
+
+	if e != nil {
+		return e
+	}
+
+	// output has trailing \n
+	// need to remove the \n
+	// otherwise it will cause error for strconv.Atoi
+	// log.Println(output[:len(output)-1])
+
+	// 0 = root, 501 = non-root user
+	i, e = strconv.Atoi(string(output[:len(output)-1]))
+	if e != nil {
+		return e
+	}
+
+	if i != 0 {
+		return errors.New("Please run as root")
+	}
+
+	return nil
+}

Helper/Input.go

@@ -0,0 +1,24 @@
+package Helper
+
+import (
+	"bufio"
+	"os"
+	"strings"
+)
+
+var (
+	reader *bufio.Reader = bufio.NewReader(os.Stdin)
+)
+
+func Input() string {
+	var (
+		text string
+		e    error
+	)
+	text, e = reader.ReadString('\n')
+	if e != nil {
+		panic(e)
+	}
+	// convert CRLF to LF
+	return strings.Replace(text, "\n", "", -1)
+}

Makefile

@@ -0,0 +1,14 @@
+CLIENT_MAIN=Client/main.go
+
+BUILD=go build -o
+BUILD32=env GOARCH=386 go build
+
+LINUX_LDFLAGS=--ldflags "-s -w -extldflags=-static"
+
+LINUX_ENV=env GOOS=linux GOARCH=amd64
+LINUX_ENV32=env GOOS=linux GOARCH=386
+
+LINUX_OUT=tjpkg
+
+build:
+	${LINUX_ENV} ${BUILD} ${LINUX_OUT} ${CLIENT_MAIN}

Variables/Variables.go

@@ -0,0 +1,76 @@
+package Variables
+
+import (
+	"os"
+	"sync"
+)
+
+const (
+	DatabaseName       string = "package_manager.db"
+	FsHashDatabaseName string = "fs_hash.db"
+)
+
+var (
+	WG sync.WaitGroup
+
+	VerboseOutput     bool   = false
+	RootDir           string = "/"
+	FsHashPicksBucket []byte = []byte("FilesystemPicks")
+	FsHashIndexBucket []byte = []byte("FilesystemIndex")
+
+	PruneRegexPaths []string = []string{
+		"^/\\.git$",
+		"^/dist$",
+		"^/boot/grub$",
+		"^/proc$",
+		"^/dev$",
+		"^/mnt$",
+		"^/sys$",
+		"^/src$",
+		"^/root$",
+		"^/home$",
+		"^/build$",
+		"^/tools$",
+		"^/opt$",
+		"^/run/user$",
+		"^/usr/share/zsh$",
+		"^/usr/share/texmf-dist$",
+		"^/usr/share/zoneinfo$",
+		"^/usr/share/zoneinfo-leaps$",
+		"^/tmp$",
+		"^/var/db$",
+		"^/var/cache$",
+		"^/var/log$",
+		"^/var/spool$",
+		"^/var/lib/texmf$",
+		"^/var/lib/postgres$",
+		"^/var/lib/pacman$",
+		"^/var/lib/NetworkManager$",
+		"^/var/lib/systemd$",
+		"^/var/lib/xkb/README.compiled$",
+		"/lost\\+found$",
+	}
+
+	IgnoreRegexPaths []string = []string{
+		"^/swapfile$",
+		"^/etc/passwd$",
+		"^/etc/passwd-$",
+		"^/etc/group$",
+		"^/etc/group-$",
+		"^/var/.updated$",
+		"^/var/lib/mlocate/mlocate.db$",
+		"^/var/lib/krb5kdc/kdc.conf$",
+		"^/var/lib/alsa/asound.state$",
+		"^/run/systemd/journal/kernel-seqnum$",
+	}
+)
+
+func init() {
+	var (
+		rootDir string
+	)
+	rootDir = os.Getenv("ROOTDIR")
+	if rootDir != "" {
+		RootDir = rootDir
+	}
+}

go.mod

@@ -0,0 +1,5 @@
+module PackageManager
+
+go 1.16
+
+require github.com/mattn/go-sqlite3 v1.14.7

go.sum

@@ -0,0 +1,2 @@
+github.com/mattn/go-sqlite3 v1.14.7 h1:fxWBnXkxfM6sRiuH3bqJ4CfzZojMOLVc0UTsTglEghA=
+github.com/mattn/go-sqlite3 v1.14.7/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=