Store each hash as its raw 32 bytes in a BLOB column (closes #134)
check / check (push) Waiting to run

The head, tail and content columns of the files table are BLOB NOT
NULL and hold the raw 32-byte SHA-256, half the size of the hex text,
in the table and in the files_signature index. The hash phase, the
records, report and trees carry each hash as a [sha256.Size]byte and
compare it with ==; nothing is hex-encoded. Reading a record whose
hash column is not 32 bytes is an error naming its path.

In the tests, testHash is a stand-in hash: a string's bytes padded
with zeros, so fixtures keep the sort order their strings give.

Judgement call: a hash of the wrong length is refused on reading,
not copied silently.

Model: opus-5-5
This commit is contained in:
clawbot
2026-10-26 01:53:25 +00:00
parent f7a746f912
commit a77f07249e
12 changed files with 335 additions and 157 deletions
+46 -13
View File
@@ -2,6 +2,7 @@ package main
import (
"context"
"crypto/sha256"
"database/sql"
"errors"
"fmt"
@@ -48,16 +49,17 @@ const lockFilePerm = 0o600
// createTableSQL is the schema applied to a fresh database. Paths are
// BLOBs because Unix paths are raw bytes, not guaranteed UTF-8. mtime
// holds whole Unix seconds and mtime_nsec the nanoseconds within that
// second.
// second. head, tail and content each hold the raw 32 bytes of a
// SHA-256.
const createTableSQL = `
CREATE TABLE files (
path BLOB PRIMARY KEY,
size INTEGER NOT NULL,
mtime INTEGER NOT NULL,
mtime_nsec INTEGER NOT NULL,
head TEXT NOT NULL,
tail TEXT NOT NULL,
content TEXT NOT NULL
head BLOB NOT NULL,
tail BLOB NOT NULL,
content BLOB NOT NULL
) WITHOUT ROWID
`
@@ -92,6 +94,10 @@ var errSchemaVersion = errors.New("unsupported database schema version")
// database.
var errScanRunning = errors.New("another scan is running")
// errHashSize reports a record whose head, tail or content is not the
// 32 bytes of a SHA-256.
var errHashSize = errors.New("hash is not 32 bytes")
// databasePath resolves the database location: SFDUPES_DATABASE when
// set and non-empty, the compiled-in default otherwise. It returns the
// path, with the working directory, its symlinks resolved, put in front
@@ -492,19 +498,24 @@ func loadFileRows(ctx context.Context, db *sql.DB, total func(n int64),
for rows.Next() {
var (
path []byte
sec, nsec int64
r scanRec
path, head, tail, content []byte
sec, nsec int64
r scanRec
)
err = rows.Scan(&path, &r.size, &sec, &nsec, &r.head, &r.tail,
&r.content)
err = rows.Scan(&path, &r.size, &sec, &nsec, &head, &tail, &content)
if err != nil {
return fmt.Errorf("read record: %w", err)
}
r.path = string(path)
r.mtime = time.Unix(sec, nsec)
err = r.setHashes(head, tail, content)
if err != nil {
return fmt.Errorf("read record %s: %w", escapePath(r.path), err)
}
fn(r)
}
@@ -561,17 +572,22 @@ func loadSignatureRows(ctx context.Context, db *sql.DB,
for rows.Next() {
var (
path []byte
r scanRec
path, head, tail, content []byte
r scanRec
)
err = rows.Scan(&r.size, &r.head, &r.tail, &r.content, &path)
err = rows.Scan(&r.size, &head, &tail, &content, &path)
if err != nil {
return fmt.Errorf("read record: %w", err)
}
r.path = string(path)
err = r.setHashes(head, tail, content)
if err != nil {
return fmt.Errorf("read record %s: %w", escapePath(r.path), err)
}
err = fn(r)
if err != nil {
return err
@@ -586,6 +602,22 @@ func loadSignatureRows(ctx context.Context, db *sql.DB,
return nil
}
// setHashes sets r's head, tail and content to the values read from
// those columns. Each must be the raw 32 bytes of a SHA-256.
func (r *scanRec) setHashes(head, tail, content []byte) error {
for _, b := range [][]byte{head, tail, content} {
if len(b) != sha256.Size {
return fmt.Errorf("%w: found %d bytes", errHashSize, len(b))
}
}
r.head = [sha256.Size]byte(head)
r.tail = [sha256.Size]byte(tail)
r.content = [sha256.Size]byte(content)
return nil
}
// countRecords returns the number of records in the database.
func countRecords(ctx context.Context, db *sql.DB) (int64, error) {
var n int64
@@ -756,7 +788,8 @@ func execUpserts(ctx context.Context, tx *sql.Tx, upserts []scanRec,
for _, r := range upserts {
_, err = st.ExecContext(ctx, []byte(r.path), r.size,
r.mtime.Unix(), r.mtime.Nanosecond(), r.head, r.tail, r.content)
r.mtime.Unix(), r.mtime.Nanosecond(),
r.head[:], r.tail[:], r.content[:])
if err != nil {
return fmt.Errorf("upsert %s: %w", r.path, err)
}