Store each hash as its raw 32 bytes in a BLOB column (closes #134)
check / check (push) Waiting to run
check / check (push) Waiting to run
The head, tail and content columns of the files table are BLOB NOT NULL and hold the raw 32-byte SHA-256, half the size of the hex text, in the table and in the files_signature index. The hash phase, the records, report and trees carry each hash as a [sha256.Size]byte and compare it with ==; nothing is hex-encoded. Reading a record whose hash column is not 32 bytes is an error naming its path. In the tests, testHash is a stand-in hash: a string's bytes padded with zeros, so fixtures keep the sort order their strings give. Judgement call: a hash of the wrong length is refused on reading, not copied silently. Model: opus-5-5
This commit is contained in:
@@ -2,6 +2,7 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -48,16 +49,17 @@ const lockFilePerm = 0o600
|
||||
// createTableSQL is the schema applied to a fresh database. Paths are
|
||||
// BLOBs because Unix paths are raw bytes, not guaranteed UTF-8. mtime
|
||||
// holds whole Unix seconds and mtime_nsec the nanoseconds within that
|
||||
// second.
|
||||
// second. head, tail and content each hold the raw 32 bytes of a
|
||||
// SHA-256.
|
||||
const createTableSQL = `
|
||||
CREATE TABLE files (
|
||||
path BLOB PRIMARY KEY,
|
||||
size INTEGER NOT NULL,
|
||||
mtime INTEGER NOT NULL,
|
||||
mtime_nsec INTEGER NOT NULL,
|
||||
head TEXT NOT NULL,
|
||||
tail TEXT NOT NULL,
|
||||
content TEXT NOT NULL
|
||||
head BLOB NOT NULL,
|
||||
tail BLOB NOT NULL,
|
||||
content BLOB NOT NULL
|
||||
) WITHOUT ROWID
|
||||
`
|
||||
|
||||
@@ -92,6 +94,10 @@ var errSchemaVersion = errors.New("unsupported database schema version")
|
||||
// database.
|
||||
var errScanRunning = errors.New("another scan is running")
|
||||
|
||||
// errHashSize reports a record whose head, tail or content is not the
|
||||
// 32 bytes of a SHA-256.
|
||||
var errHashSize = errors.New("hash is not 32 bytes")
|
||||
|
||||
// databasePath resolves the database location: SFDUPES_DATABASE when
|
||||
// set and non-empty, the compiled-in default otherwise. It returns the
|
||||
// path, with the working directory, its symlinks resolved, put in front
|
||||
@@ -492,19 +498,24 @@ func loadFileRows(ctx context.Context, db *sql.DB, total func(n int64),
|
||||
|
||||
for rows.Next() {
|
||||
var (
|
||||
path []byte
|
||||
sec, nsec int64
|
||||
r scanRec
|
||||
path, head, tail, content []byte
|
||||
sec, nsec int64
|
||||
r scanRec
|
||||
)
|
||||
|
||||
err = rows.Scan(&path, &r.size, &sec, &nsec, &r.head, &r.tail,
|
||||
&r.content)
|
||||
err = rows.Scan(&path, &r.size, &sec, &nsec, &head, &tail, &content)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read record: %w", err)
|
||||
}
|
||||
|
||||
r.path = string(path)
|
||||
r.mtime = time.Unix(sec, nsec)
|
||||
|
||||
err = r.setHashes(head, tail, content)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read record %s: %w", escapePath(r.path), err)
|
||||
}
|
||||
|
||||
fn(r)
|
||||
}
|
||||
|
||||
@@ -561,17 +572,22 @@ func loadSignatureRows(ctx context.Context, db *sql.DB,
|
||||
|
||||
for rows.Next() {
|
||||
var (
|
||||
path []byte
|
||||
r scanRec
|
||||
path, head, tail, content []byte
|
||||
r scanRec
|
||||
)
|
||||
|
||||
err = rows.Scan(&r.size, &r.head, &r.tail, &r.content, &path)
|
||||
err = rows.Scan(&r.size, &head, &tail, &content, &path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read record: %w", err)
|
||||
}
|
||||
|
||||
r.path = string(path)
|
||||
|
||||
err = r.setHashes(head, tail, content)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read record %s: %w", escapePath(r.path), err)
|
||||
}
|
||||
|
||||
err = fn(r)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -586,6 +602,22 @@ func loadSignatureRows(ctx context.Context, db *sql.DB,
|
||||
return nil
|
||||
}
|
||||
|
||||
// setHashes sets r's head, tail and content to the values read from
|
||||
// those columns. Each must be the raw 32 bytes of a SHA-256.
|
||||
func (r *scanRec) setHashes(head, tail, content []byte) error {
|
||||
for _, b := range [][]byte{head, tail, content} {
|
||||
if len(b) != sha256.Size {
|
||||
return fmt.Errorf("%w: found %d bytes", errHashSize, len(b))
|
||||
}
|
||||
}
|
||||
|
||||
r.head = [sha256.Size]byte(head)
|
||||
r.tail = [sha256.Size]byte(tail)
|
||||
r.content = [sha256.Size]byte(content)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// countRecords returns the number of records in the database.
|
||||
func countRecords(ctx context.Context, db *sql.DB) (int64, error) {
|
||||
var n int64
|
||||
@@ -756,7 +788,8 @@ func execUpserts(ctx context.Context, tx *sql.Tx, upserts []scanRec,
|
||||
|
||||
for _, r := range upserts {
|
||||
_, err = st.ExecContext(ctx, []byte(r.path), r.size,
|
||||
r.mtime.Unix(), r.mtime.Nanosecond(), r.head, r.tail, r.content)
|
||||
r.mtime.Unix(), r.mtime.Nanosecond(),
|
||||
r.head[:], r.tail[:], r.content[:])
|
||||
if err != nil {
|
||||
return fmt.Errorf("upsert %s: %w", r.path, err)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user