Implement persistent SQLite scan database
All checks were successful
check / check (push) Successful in 3s

scan now synchronizes a database that survives between runs
(SFDUPES_DATABASE, default /var/lib/sfdupes/db.sqlite) instead of
emitting a stream: operands are resolved to absolute paths, unchanged
files (same size, mtime not newer than recorded) are never re-read, new
and changed files are hashed, and records under the scanned operands
that were not verified this run are deleted; records outside the
operands are untouched. All changes commit in a single transaction, and
WAL journaling with a busy timeout keeps a report run during a cron
scan safe.

report and trees read the database (no positional arguments); the
NUL-terminated stream format, its parser, and the malformed-record
handling are gone. The driver is modernc.org/sqlite (pure Go), so
builds keep cgo disabled.
This commit is contained in:
2026-07-24 03:08:49 +07:00
parent e0d578a707
commit d8fbcb32c2
12 changed files with 1229 additions and 379 deletions

32
main.go
View File

@@ -2,13 +2,15 @@
// very large filesystems without reading full file contents. Files are
// considered duplicates when they have identical size, identical SHA-256
// of their first 1024 bytes, and identical SHA-256 of their last 1024
// bytes.
// bytes. scan maintains a persistent SQLite database of file signatures
// (SFDUPES_DATABASE, default /var/lib/sfdupes/db.sqlite) that the
// reporting subcommands read.
//
// Usage:
//
// sfdupes scan [--workers N] [-x] PATH... > files.dat
// sfdupes report [files.dat|-] > dupes.tsv
// sfdupes trees [files.dat|-] > dupetrees.tsv
// sfdupes scan [--workers N] [-x] PATH...
// sfdupes report > dupes.tsv
// sfdupes trees > dupetrees.tsv
//
// See README.md for the complete specification.
package main
@@ -60,7 +62,7 @@ func main() {
scanCmd := &cobra.Command{
Use: "scan [--workers N] [-x] PATH...",
Short: "Walk trees and emit one record per regular file on stdout",
Short: "Walk trees and synchronize the scan database",
Args: cobra.MinimumNArgs(1),
Run: func(_ *cobra.Command, args []string) {
runScan(args, scanWorkers, scanOneFS)
@@ -72,20 +74,20 @@ func main() {
"do not cross filesystem boundaries")
reportCmd := &cobra.Command{
Use: "report [files.dat|-]",
Short: "Read a scan stream and print the file-level duplicates report",
Args: cobra.MaximumNArgs(1),
Run: func(_ *cobra.Command, args []string) {
runReport(args)
Use: "report",
Short: "Read the scan database and print the file-level duplicates report",
Args: cobra.NoArgs,
Run: func(_ *cobra.Command, _ []string) {
runReport()
},
}
treesCmd := &cobra.Command{
Use: "trees [files.dat|-]",
Short: "Read a scan stream and print the duplicate-tree report",
Args: cobra.MaximumNArgs(1),
Run: func(_ *cobra.Command, args []string) {
runTrees(args)
Use: "trees",
Short: "Read the scan database and print the duplicate-tree report",
Args: cobra.NoArgs,
Run: func(_ *cobra.Command, _ []string) {
runTrees()
},
}