Implement persistent SQLite scan database
All checks were successful
check / check (push) Successful in 3s
All checks were successful
check / check (push) Successful in 3s
scan now synchronizes a database that survives between runs (SFDUPES_DATABASE, default /var/lib/sfdupes/db.sqlite) instead of emitting a stream: operands are resolved to absolute paths, unchanged files (same size, mtime not newer than recorded) are never re-read, new and changed files are hashed, and records under the scanned operands that were not verified this run are deleted; records outside the operands are untouched. All changes commit in a single transaction, and WAL journaling with a busy timeout keeps a report run during a cron scan safe. report and trees read the database (no positional arguments); the NUL-terminated stream format, its parser, and the malformed-record handling are gone. The driver is modernc.org/sqlite (pure Go), so builds keep cgo disabled.
This commit is contained in:
32
main.go
32
main.go
@@ -2,13 +2,15 @@
|
||||
// very large filesystems without reading full file contents. Files are
|
||||
// considered duplicates when they have identical size, identical SHA-256
|
||||
// of their first 1024 bytes, and identical SHA-256 of their last 1024
|
||||
// bytes.
|
||||
// bytes. scan maintains a persistent SQLite database of file signatures
|
||||
// (SFDUPES_DATABASE, default /var/lib/sfdupes/db.sqlite) that the
|
||||
// reporting subcommands read.
|
||||
//
|
||||
// Usage:
|
||||
//
|
||||
// sfdupes scan [--workers N] [-x] PATH... > files.dat
|
||||
// sfdupes report [files.dat|-] > dupes.tsv
|
||||
// sfdupes trees [files.dat|-] > dupetrees.tsv
|
||||
// sfdupes scan [--workers N] [-x] PATH...
|
||||
// sfdupes report > dupes.tsv
|
||||
// sfdupes trees > dupetrees.tsv
|
||||
//
|
||||
// See README.md for the complete specification.
|
||||
package main
|
||||
@@ -60,7 +62,7 @@ func main() {
|
||||
|
||||
scanCmd := &cobra.Command{
|
||||
Use: "scan [--workers N] [-x] PATH...",
|
||||
Short: "Walk trees and emit one record per regular file on stdout",
|
||||
Short: "Walk trees and synchronize the scan database",
|
||||
Args: cobra.MinimumNArgs(1),
|
||||
Run: func(_ *cobra.Command, args []string) {
|
||||
runScan(args, scanWorkers, scanOneFS)
|
||||
@@ -72,20 +74,20 @@ func main() {
|
||||
"do not cross filesystem boundaries")
|
||||
|
||||
reportCmd := &cobra.Command{
|
||||
Use: "report [files.dat|-]",
|
||||
Short: "Read a scan stream and print the file-level duplicates report",
|
||||
Args: cobra.MaximumNArgs(1),
|
||||
Run: func(_ *cobra.Command, args []string) {
|
||||
runReport(args)
|
||||
Use: "report",
|
||||
Short: "Read the scan database and print the file-level duplicates report",
|
||||
Args: cobra.NoArgs,
|
||||
Run: func(_ *cobra.Command, _ []string) {
|
||||
runReport()
|
||||
},
|
||||
}
|
||||
|
||||
treesCmd := &cobra.Command{
|
||||
Use: "trees [files.dat|-]",
|
||||
Short: "Read a scan stream and print the duplicate-tree report",
|
||||
Args: cobra.MaximumNArgs(1),
|
||||
Run: func(_ *cobra.Command, args []string) {
|
||||
runTrees(args)
|
||||
Use: "trees",
|
||||
Short: "Read the scan database and print the duplicate-tree report",
|
||||
Args: cobra.NoArgs,
|
||||
Run: func(_ *cobra.Command, _ []string) {
|
||||
runTrees()
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user