Add 64 KiB head/tail and content-hash duplicate ladder (closes #61) (#62)

db.go

@@ -25,8 +25,11 @@ const defaultDatabasePath = "/var/lib/sfdupes/db.sqlite"
 const databaseEnv = "SFDUPES_DATABASE"
 
 // schemaVersion is the database schema version this build reads and
-// writes, stored in PRAGMA user_version.
-const schemaVersion = 1
+// writes, stored in PRAGMA user_version. Version 2 adds the content
+// column and the head/tail/content signature (replacing the version 1
+// 1 KiB end windows), so a version 1 database is rejected and must be
+// rescanned.
+const schemaVersion = 2
 
 // dbDirPerm is the mode for a database parent directory created by
 // scan.
@@ -36,22 +39,24 @@ const dbDirPerm = 0o755
 // BLOBs because Unix paths are raw bytes, not guaranteed UTF-8.
 const createTableSQL = `
 CREATE TABLE files (
-    path  BLOB PRIMARY KEY,
-    size  INTEGER NOT NULL,
-    mtime INTEGER NOT NULL,
-    head  TEXT NOT NULL,
-    tail  TEXT NOT NULL
+    path    BLOB PRIMARY KEY,
+    size    INTEGER NOT NULL,
+    mtime   INTEGER NOT NULL,
+    head    TEXT NOT NULL,
+    tail    TEXT NOT NULL,
+    content TEXT NOT NULL
 ) WITHOUT ROWID
 `
 
 // upsertSQL inserts one file record, replacing any existing record for
 // the same path.
 const upsertSQL = `
-INSERT INTO files (path, size, mtime, head, tail)
-VALUES (?, ?, ?, ?, ?)
+INSERT INTO files (path, size, mtime, head, tail, content)
+VALUES (?, ?, ?, ?, ?, ?)
 ON CONFLICT (path) DO UPDATE SET
     size = excluded.size, mtime = excluded.mtime,
-    head = excluded.head, tail = excluded.tail
+    head = excluded.head, tail = excluded.tail,
+    content = excluded.content
 `
 
 // errNoDatabase reports a missing database file for report/trees.
@@ -205,7 +210,7 @@ func userVersion(ctx context.Context, db *sql.DB) (int, error) {
 // loadFileRows reads every record from the files table.
 func loadFileRows(ctx context.Context, db *sql.DB) ([]scanRec, error) {
 	rows, err := db.QueryContext(ctx,
-		"SELECT path, size, mtime, head, tail FROM files")
+		"SELECT path, size, mtime, head, tail, content FROM files")
 	if err != nil {
 		return nil, fmt.Errorf("read records: %w", err)
 	}
@@ -220,7 +225,8 @@ func loadFileRows(ctx context.Context, db *sql.DB) ([]scanRec, error) {
 			r    scanRec
 		)
 
-		err = rows.Scan(&path, &r.size, &r.mtime, &r.head, &r.tail)
+		err = rows.Scan(&path, &r.size, &r.mtime, &r.head, &r.tail,
+			&r.content)
 		if err != nil {
 			return nil, fmt.Errorf("read record: %w", err)
 		}
@@ -348,7 +354,7 @@ func execUpserts(ctx context.Context, tx *sql.Tx, upserts []scanRec,
 
 	for _, r := range upserts {
 		_, err = st.ExecContext(ctx,
-			[]byte(r.path), r.size, r.mtime, r.head, r.tail)
+			[]byte(r.path), r.size, r.mtime, r.head, r.tail, r.content)
 		if err != nil {
 			return fmt.Errorf("upsert %s: %w", r.path, err)
 		}