PackageManager just because
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

236 lines
4.5 KiB

3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
  1. package Filesystem
  2. import (
  3. "context"
  4. "fmt"
  5. "os"
  6. "path/filepath"
  7. "runtime"
  8. "sync"
  9. "github.com/vbauerster/mpb"
  10. bolt "go.etcd.io/bbolt"
  11. "golang.org/x/sync/semaphore"
  12. "PackageManager/Client/Database"
  13. "PackageManager/Client/ProgressBar"
  14. "PackageManager/Color"
  15. "PackageManager/Variables"
  16. )
  17. var (
  18. fsStatusWG sync.WaitGroup
  19. FsWalkWG sync.WaitGroup
  20. )
  21. type FilesystemStatus struct {
  22. NewFiles []string
  23. PickedFiles []string
  24. ModifiedFiles []string
  25. MissingFiles []string
  26. }
  27. func ShowFilesystemDiff(root string) error {
  28. var (
  29. fsStatus FilesystemStatus
  30. picksBucket *bolt.Bucket
  31. pickedFiles []string
  32. e error
  33. )
  34. fsStatus, e = GetFilesystemDiff(root)
  35. if e != nil {
  36. return e
  37. }
  38. e = Database.FsDB.View(func(tx *bolt.Tx) error {
  39. picksBucket = tx.Bucket(Variables.FsHashPicksBucket)
  40. return picksBucket.ForEach(func(key, _ []byte) error {
  41. pickedFiles = append(pickedFiles, string(key))
  42. return nil
  43. })
  44. })
  45. fmt.Println("New files:")
  46. PrintFilesOrLength(fsStatus.NewFiles, Color.Green)
  47. fmt.Println("Added files:")
  48. PrintFilesOrLength(pickedFiles, Color.Green)
  49. fmt.Println("Modified files:")
  50. PrintFilesOrLength(fsStatus.ModifiedFiles, Color.Warning)
  51. fmt.Println("Deleted files:")
  52. PrintFilesOrLength(fsStatus.MissingFiles, Color.Fatal)
  53. return nil
  54. }
  55. func GetFilesystemLength(root string) (int, error) {
  56. var (
  57. rootStat os.FileInfo
  58. fsCount int = 0
  59. e error
  60. )
  61. rootStat, e = os.Stat(root)
  62. if e != nil {
  63. return fsCount, e
  64. }
  65. if rootStat.IsDir() && root[len(root)-1:] != "/" {
  66. root = root + "/"
  67. }
  68. filepath.Walk(root, func(p string, i os.FileInfo, _ error) error {
  69. // Ignore path in Variables.PruneRegexPaths
  70. if i.IsDir() && matchAny(p, PruneRegex) {
  71. return filepath.SkipDir
  72. }
  73. // Ignore path in Variables.IgnoreRegexPaths
  74. if matchAny(p, IgnoreRegex) {
  75. return nil
  76. }
  77. if !i.Mode().IsRegular() && (i.Mode()&os.ModeSymlink == 0) {
  78. return nil
  79. }
  80. fsCount++
  81. return nil
  82. })
  83. return fsCount, e
  84. }
  85. func (fsStatus *FilesystemStatus) parseFile(indexBucket, picksBucket *bolt.Bucket, p string, bar *mpb.Bar) {
  86. var (
  87. newFileObject FileObject
  88. knownFileObject FileObject
  89. pick, known []byte
  90. e error
  91. )
  92. defer func() {
  93. bar.Increment()
  94. fsStatusWG.Done()
  95. }()
  96. fsStatusWG.Wait()
  97. fsStatusWG.Add(1)
  98. pick = picksBucket.Get([]byte(p))
  99. known = indexBucket.Get([]byte(p))
  100. if pick != nil {
  101. fsStatus.PickedFiles = append(fsStatus.PickedFiles, p)
  102. return
  103. }
  104. if known != nil {
  105. newFileObject, e = CreateFileObject(p)
  106. if e != nil {
  107. return
  108. }
  109. knownFileObject, e = FromBytes(known)
  110. if e != nil {
  111. return
  112. }
  113. e = newFileObject.IsDifferent(knownFileObject)
  114. if e != nil {
  115. fsStatus.ModifiedFiles = append(fsStatus.ModifiedFiles, p)
  116. }
  117. return
  118. }
  119. fsStatusWG.Wait()
  120. fsStatusWG.Add(1)
  121. fsStatus.NewFiles = append(fsStatus.NewFiles, p)
  122. return
  123. }
  124. func GetFilesystemDiff(root string) (FilesystemStatus, error) {
  125. var (
  126. fsStatus FilesystemStatus = FilesystemStatus{}
  127. sem *semaphore.Weighted
  128. picksBucket *bolt.Bucket
  129. indexBucket *bolt.Bucket
  130. rootStat os.FileInfo
  131. bar *mpb.Bar
  132. fsCount int
  133. poolSize int
  134. e error
  135. )
  136. poolSize = runtime.NumCPU()
  137. sem = semaphore.NewWeighted(int64(poolSize))
  138. rootStat, e = os.Stat(root)
  139. if e != nil {
  140. return fsStatus, e
  141. }
  142. if rootStat.IsDir() && root[len(root)-1:] != "/" {
  143. root = root + "/"
  144. }
  145. fsCount, e = GetFilesystemLength(root)
  146. if e != nil {
  147. return fsStatus, e
  148. }
  149. bar = ProgressBar.InitBar("Scanning...", fsCount)
  150. e = Database.FsDB.View(func(tx *bolt.Tx) error {
  151. picksBucket = tx.Bucket(Variables.FsHashPicksBucket)
  152. indexBucket = tx.Bucket(Variables.FsHashIndexBucket)
  153. filepath.Walk(root, func(p string, i os.FileInfo, _ error) error {
  154. // Ignore path in Variables.PruneRegexPaths
  155. if i.IsDir() && matchAny(p, PruneRegex) {
  156. return filepath.SkipDir
  157. }
  158. // Ignore path in Variables.IgnoreRegexPaths
  159. if matchAny(p, IgnoreRegex) {
  160. return nil
  161. }
  162. if !i.Mode().IsRegular() && (i.Mode()&os.ModeSymlink == 0) {
  163. return nil
  164. }
  165. FsWalkWG.Add(1)
  166. sem.Acquire(context.Background(), 1)
  167. go func() {
  168. fsStatus.parseFile(indexBucket, picksBucket, p, bar)
  169. sem.Release(1)
  170. FsWalkWG.Done()
  171. }()
  172. FsWalkWG.Wait()
  173. return nil
  174. })
  175. indexBucket.ForEach(func(k, v []byte) error {
  176. _, e = os.Lstat(string(k))
  177. if os.IsNotExist(e) {
  178. fsStatus.MissingFiles = append(fsStatus.MissingFiles, string(k))
  179. }
  180. return nil
  181. })
  182. return nil
  183. })
  184. return fsStatus, e
  185. }