From 35075c741eacea403828fb4a446ff5402f56fa75 Mon Sep 17 00:00:00 2001
From: Tovi Jaeschke-Rogers <tovi.jaeschke-rogers@four.io>
Date: Mon, 19 Jul 2021 17:54:39 +0930
Subject: [PATCH] Make fs scan asynchronous

---
 Client/Filesystem/FilesystemDiff.go | 38 +++++++++++++++++------------
 Client/Filesystem/PickFiles.go      | 12 ++++-----
 Variables/Variables.go              |  7 ++++--
 3 files changed, 33 insertions(+), 24 deletions(-)

diff --git a/Client/Filesystem/FilesystemDiff.go b/Client/Filesystem/FilesystemDiff.go
index 73eb41f..64550de 100644
--- a/Client/Filesystem/FilesystemDiff.go
+++ b/Client/Filesystem/FilesystemDiff.go
@@ -1,13 +1,15 @@
 package Filesystem
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"path/filepath"
+	"runtime"
 
 	"github.com/vbauerster/mpb"
-	"github.com/zenthangplus/goccm"
 	bolt "go.etcd.io/bbolt"
+	"golang.org/x/sync/semaphore"
 
 	"PackageManager/Client/Database"
 	"PackageManager/Client/ProgressBar"
@@ -89,7 +91,7 @@ func GetFilesystemLength(root string) (int, error) {
 	return fsCount, e
 }
 
-func (fsStatus *FilesystemStatus) parseFile(indexBucket, picksBucket *bolt.Bucket, p string, bar *mpb.Bar, c goccm.ConcurrencyManager) {
+func (fsStatus *FilesystemStatus) parseFile(indexBucket, picksBucket *bolt.Bucket, p string, bar *mpb.Bar) {
 	var (
 		newFileObject   FileObject
 		knownFileObject FileObject
@@ -99,7 +101,6 @@ func (fsStatus *FilesystemStatus) parseFile(indexBucket, picksBucket *bolt.Bucke
 
 	defer func() {
 		bar.Increment()
-		c.Done()
 	}()
 
 	pick = picksBucket.Get([]byte(p))
@@ -136,22 +137,20 @@ func (fsStatus *FilesystemStatus) parseFile(indexBucket, picksBucket *bolt.Bucke
 
 func GetFilesystemDiff(root string) (FilesystemStatus, error) {
 	var (
-		fsStatus FilesystemStatus = FilesystemStatus{}
-
-		rootStat os.FileInfo
-
+		fsStatus    FilesystemStatus = FilesystemStatus{}
+		sem         *semaphore.Weighted
 		picksBucket *bolt.Bucket
 		indexBucket *bolt.Bucket
-
-		bar *mpb.Bar
-
-		fsCount int
-
-		c goccm.ConcurrencyManager
-
-		e error
+		rootStat    os.FileInfo
+		bar         *mpb.Bar
+		fsCount     int
+		poolSize    int
+		e           error
 	)
 
+	poolSize = runtime.NumCPU()
+	sem = semaphore.NewWeighted(int64(poolSize))
+
 	rootStat, e = os.Stat(root)
 	if e != nil {
 		return fsStatus, e
@@ -189,7 +188,13 @@ func GetFilesystemDiff(root string) (FilesystemStatus, error) {
 				return nil
 			}
 
-			go fsStatus.parseFile(indexBucket, picksBucket, p, bar, c)
+			Variables.WG.Add(1)
+			sem.Acquire(context.Background(), 1)
+			go func() {
+				fsStatus.parseFile(indexBucket, picksBucket, p, bar)
+				Variables.WG.Done()
+				sem.Release(1)
+			}()
 
 			return nil
 		})
@@ -202,6 +207,7 @@ func GetFilesystemDiff(root string) (FilesystemStatus, error) {
 			return nil
 		})
 
+		Variables.WG.Wait()
 		ProgressBar.CloseBar(bar)
 
 		return nil
diff --git a/Client/Filesystem/PickFiles.go b/Client/Filesystem/PickFiles.go
index 984b7a7..330cf4a 100644
--- a/Client/Filesystem/PickFiles.go
+++ b/Client/Filesystem/PickFiles.go
@@ -63,7 +63,7 @@ func pickFilesRecursive(rootPath string) error {
 				bar.Increment()
 				e = AddFileToBucket(picksBucket, f)
 				if e != nil {
-					return nil
+					return e
 				}
 			}
 		}
@@ -73,7 +73,7 @@ func pickFilesRecursive(rootPath string) error {
 				bar.Increment()
 				e = AddFileToBucket(picksBucket, f)
 				if e != nil {
-					return nil
+					return e
 				}
 			}
 		}
@@ -83,20 +83,20 @@ func pickFilesRecursive(rootPath string) error {
 				bar.Increment()
 				e = RemoveFileFromBucket(indexBucket, f)
 				if e != nil {
-					return nil
+					return e
 				}
 				e = RemoveFileFromBucket(picksBucket, f)
 				if e != nil {
-					return nil
+					return e
 				}
 			}
 		}
 
+		Variables.WG.Wait()
+		ProgressBar.CloseBar(bar)
 		return nil
 	})
 
-	ProgressBar.CloseBar(bar)
-
 	return e
 }
 
diff --git a/Variables/Variables.go b/Variables/Variables.go
index 15482d8..8ab00a0 100644
--- a/Variables/Variables.go
+++ b/Variables/Variables.go
@@ -31,10 +31,12 @@ var (
 		"^/home$",
 		"^/build$",
 		"^/tools$",
+		"^/opt$",
 		"^/run/user$",
-		"^/usr/share/zoneinfo-leaps$",
-		"^/usr/share/zoneinfo$",
 		"^/usr/share/zsh$",
+		"^/usr/share/texmf-dist$",
+		"^/usr/share/zoneinfo$",
+		"^/usr/share/zoneinfo-leaps$",
 		"^/tmp$",
 		"^/var/db$",
 		"^/var/cache$",
@@ -59,6 +61,7 @@ var (
 		"^/var/lib/mlocate/mlocate.db$",
 		"^/var/lib/krb5kdc/kdc.conf$",
 		"^/var/lib/alsa/asound.state$",
+		"^/run/systemd/journal/kernel-seqnum$",
 	}
 )