Stream report and trees instead of loading every record (closes #14)
check / check (push) Successful in 2m41s
check / check (push) Successful in 2m41s
report now has SQLite group the records and put the rows in report order, helped by a new files_signature index on (size, head, tail, content), and writes each row as it reads it. trees reads the records in path order, where all the paths under a directory come together, so it computes each directory's digest as soon as the stream leaves it and keeps only its path, parent, digest and totals. Output is unchanged. The tests that called the removed in-memory grouping functions now group records stored in a database. New tests check that both commands give the same output whatever order the records were inserted in, and that a stdout failure partway through a long report is reported as one. Model: opus-5-5
This commit is contained in:
@@ -12,39 +12,48 @@ import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
// fileSig is a file's duplicate signature; mtime is excluded.
|
||||
type fileSig struct {
|
||||
size int64
|
||||
head string
|
||||
tail string
|
||||
content string
|
||||
}
|
||||
|
||||
// treeNode is one directory reconstructed from the scan stream.
|
||||
// treeNode is one directory reconstructed from the record paths.
|
||||
type treeNode struct {
|
||||
path string
|
||||
parent *treeNode
|
||||
dirs map[string]*treeNode
|
||||
files map[string]fileSig
|
||||
path string
|
||||
parent *treeNode
|
||||
// entries holds the serialized child entries until the digest is
|
||||
// computed from them, and is then dropped.
|
||||
entries []string
|
||||
digest [sha256.Size]byte
|
||||
fileCount int64
|
||||
totalSize int64
|
||||
}
|
||||
|
||||
// runTrees implements the trees subcommand: it reads every record from
|
||||
// the database, reconstructs the directory hierarchy from the record
|
||||
// paths, computes a Merkle-style digest per directory, and prints
|
||||
// maximal duplicate-tree groups as TSV on stdout. It never touches the
|
||||
// scanned filesystem; its only I/O is the database, stdout, and
|
||||
// stderr.
|
||||
// the database in path order, reconstructs the directory hierarchy from
|
||||
// the record paths, computes a Merkle-style digest per directory, and
|
||||
// prints maximal duplicate-tree groups as TSV on stdout. It never
|
||||
// touches the scanned filesystem; its only I/O is the database, stdout,
|
||||
// and stderr. Any database problem, including a missing database, is
|
||||
// fatal.
|
||||
func runTrees(ctx context.Context, stdout io.Writer) error {
|
||||
recs, err := loadRecords(ctx)
|
||||
dbPath := databasePath()
|
||||
|
||||
db, err := openReportDatabase(ctx, dbPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
super, allDirs := buildHierarchy(recs)
|
||||
super.compute()
|
||||
defer func() { _ = db.Close() }()
|
||||
|
||||
records := 0
|
||||
tree := newTreeBuilder()
|
||||
|
||||
err = loadFileRows(ctx, db, func(r scanRec) {
|
||||
records++
|
||||
|
||||
tree.add(r)
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("database %s: %w", dbPath, err)
|
||||
}
|
||||
|
||||
super, allDirs := tree.finish()
|
||||
|
||||
dupes := collectTreeGroups(allDirs, super)
|
||||
|
||||
@@ -82,73 +91,128 @@ func runTrees(ctx context.Context, stdout io.Writer) error {
|
||||
fmt.Fprintf(os.Stderr,
|
||||
"trees: %d records read, %d duplicate tree groups, %d dupe trees, "+
|
||||
"%s reclaimable\n",
|
||||
len(recs), len(dupes), dupeTrees, humanBytes(reclaimable))
|
||||
records, len(dupes), dupeTrees, humanBytes(reclaimable))
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// buildHierarchy reconstructs the directory hierarchy from the record
|
||||
// paths under a synthetic super-root. Paths are split on "/"; for
|
||||
// absolute paths the first component is empty, which becomes the
|
||||
// top-level node with path "/". It returns the super-root and every
|
||||
// directory node created.
|
||||
func buildHierarchy(recs []scanRec) (*treeNode, []*treeNode) {
|
||||
// treeBuilder reconstructs the directory hierarchy from records added
|
||||
// in path order, under a synthetic super-root. Paths are split on "/";
|
||||
// for absolute paths the first component is empty, which becomes the
|
||||
// top-level directory with path "/". In path order all the paths under
|
||||
// one directory come together, so a directory is complete once a path
|
||||
// outside it is added: its digest is computed then and its entries are
|
||||
// dropped. Only the directories holding the latest path keep entries.
|
||||
type treeBuilder struct {
|
||||
super *treeNode
|
||||
// open lists the directories holding the latest path, outermost
|
||||
// first, starting with the super-root; names[i] is open[i]'s name.
|
||||
open []*treeNode
|
||||
names []string
|
||||
// dirs lists every completed directory.
|
||||
dirs []*treeNode
|
||||
}
|
||||
|
||||
func newTreeBuilder() *treeBuilder {
|
||||
super := &treeNode{}
|
||||
|
||||
var allDirs []*treeNode
|
||||
return &treeBuilder{
|
||||
super: super,
|
||||
open: []*treeNode{super},
|
||||
names: []string{""},
|
||||
}
|
||||
}
|
||||
|
||||
for _, r := range recs {
|
||||
comps := strings.Split(r.path, "/")
|
||||
// add adds one record. Each record must come after the previous one in
|
||||
// path order (byte order); otherwise a completed directory would be
|
||||
// started again as a second directory with the same path.
|
||||
func (b *treeBuilder) add(r scanRec) {
|
||||
comps := strings.Split(r.path, "/")
|
||||
dirNames, name := comps[:len(comps)-1], comps[len(comps)-1]
|
||||
|
||||
node := super
|
||||
for _, c := range comps[:len(comps)-1] {
|
||||
child := node.dirs[c]
|
||||
if child == nil {
|
||||
childPath := node.path + "/" + c
|
||||
|
||||
// The root directory's path is "/", not empty, and its
|
||||
// children's paths start with one slash, not two.
|
||||
switch {
|
||||
case node == super && c == "":
|
||||
childPath = "/"
|
||||
case node == super:
|
||||
childPath = c
|
||||
case node.path == "/":
|
||||
childPath = "/" + c
|
||||
}
|
||||
|
||||
child = &treeNode{path: childPath, parent: node}
|
||||
if node.dirs == nil {
|
||||
node.dirs = make(map[string]*treeNode)
|
||||
}
|
||||
|
||||
node.dirs[c] = child
|
||||
allDirs = append(allDirs, child)
|
||||
}
|
||||
|
||||
node = child
|
||||
}
|
||||
|
||||
if node.files == nil {
|
||||
node.files = make(map[string]fileSig)
|
||||
}
|
||||
|
||||
sig := fileSig{
|
||||
size: r.size, head: r.head, tail: r.tail, content: r.content,
|
||||
}
|
||||
|
||||
// A record without a content hash has unknown content (README
|
||||
// "Database"): give it a signature no other file can share, so
|
||||
// trees containing it never compare equal. Real hashes are
|
||||
// hex, so the NUL-prefixed form cannot collide.
|
||||
if sig.content == "" {
|
||||
sig.content = "unhashed\x00" + r.path
|
||||
}
|
||||
|
||||
node.files[comps[len(comps)-1]] = sig
|
||||
// Keep the open directories that hold this path; complete the rest.
|
||||
depth := 1
|
||||
for depth < len(b.open) && depth <= len(dirNames) &&
|
||||
b.names[depth] == dirNames[depth-1] {
|
||||
depth++
|
||||
}
|
||||
|
||||
return super, allDirs
|
||||
b.closeTo(depth)
|
||||
|
||||
for _, c := range dirNames[depth-1:] {
|
||||
b.openDir(c)
|
||||
}
|
||||
|
||||
dir := b.open[len(b.open)-1]
|
||||
dir.entries = append(dir.entries, fileEntry(name, r))
|
||||
dir.fileCount++
|
||||
dir.totalSize += r.size
|
||||
}
|
||||
|
||||
// openDir starts the directory called name inside the innermost open
|
||||
// one.
|
||||
func (b *treeBuilder) openDir(name string) {
|
||||
parent := b.open[len(b.open)-1]
|
||||
path := parent.path + "/" + name
|
||||
|
||||
// The root directory's path is "/", not empty, and its children's
|
||||
// paths start with one slash, not two.
|
||||
switch {
|
||||
case parent == b.super && name == "":
|
||||
path = "/"
|
||||
case parent == b.super:
|
||||
path = name
|
||||
case parent.path == "/":
|
||||
path = "/" + name
|
||||
}
|
||||
|
||||
b.open = append(b.open, &treeNode{path: path, parent: parent})
|
||||
b.names = append(b.names, name)
|
||||
}
|
||||
|
||||
// closeTo completes the open directories after the first n, innermost
|
||||
// first: each one's digest is computed and entered in its parent along
|
||||
// with its totals.
|
||||
func (b *treeBuilder) closeTo(n int) {
|
||||
for len(b.open) > n {
|
||||
last := len(b.open) - 1
|
||||
dir, name := b.open[last], b.names[last]
|
||||
b.open, b.names = b.open[:last], b.names[:last]
|
||||
|
||||
dir.computeDigest()
|
||||
|
||||
dir.parent.entries = append(dir.parent.entries,
|
||||
"d\x00"+name+"\x00"+string(dir.digest[:]))
|
||||
dir.parent.fileCount += dir.fileCount
|
||||
dir.parent.totalSize += dir.totalSize
|
||||
b.dirs = append(b.dirs, dir)
|
||||
}
|
||||
}
|
||||
|
||||
// finish completes every open directory and returns the super-root and
|
||||
// every directory.
|
||||
func (b *treeBuilder) finish() (*treeNode, []*treeNode) {
|
||||
b.closeTo(1)
|
||||
|
||||
return b.super, b.dirs
|
||||
}
|
||||
|
||||
// fileEntry serializes a file child for its directory's digest: its
|
||||
// name and its signature (size, head, tail, content); mtime is
|
||||
// excluded.
|
||||
func fileEntry(name string, r scanRec) string {
|
||||
content := r.content
|
||||
|
||||
// A record without a content hash has unknown content (README
|
||||
// "Database"): give it a signature no other file can share, so
|
||||
// trees containing it never compare equal. Real hashes are hex, so
|
||||
// the NUL-prefixed form cannot collide.
|
||||
if content == "" {
|
||||
content = "unhashed\x00" + r.path
|
||||
}
|
||||
|
||||
return "f\x00" + name + "\x00" + strconv.FormatInt(r.size, 10) +
|
||||
"\x00" + r.head + "\x00" + r.tail + "\x00" + content
|
||||
}
|
||||
|
||||
// collectTreeGroups groups directories by digest and returns every
|
||||
@@ -190,38 +254,22 @@ func collectTreeGroups(allDirs []*treeNode, super *treeNode) [][]*treeNode {
|
||||
return dupes
|
||||
}
|
||||
|
||||
// compute fills in digest, fileCount, and totalSize for n and all of
|
||||
// its descendants. A directory's digest is the SHA-256 of its child
|
||||
// entries — files serialized with name and signature, subdirectories
|
||||
// with name and recursive digest — sorted byte-lexicographically.
|
||||
// Filenames cannot contain NUL or "/", so NUL delimiters are
|
||||
// unambiguous.
|
||||
func (n *treeNode) compute() {
|
||||
entries := make([]string, 0, len(n.dirs)+len(n.files))
|
||||
for name, sig := range n.files {
|
||||
entries = append(entries,
|
||||
"f\x00"+name+"\x00"+strconv.FormatInt(sig.size, 10)+
|
||||
"\x00"+sig.head+"\x00"+sig.tail+"\x00"+sig.content)
|
||||
n.fileCount++
|
||||
n.totalSize += sig.size
|
||||
}
|
||||
|
||||
for name, child := range n.dirs {
|
||||
child.compute()
|
||||
entries = append(entries, "d\x00"+name+"\x00"+string(child.digest[:]))
|
||||
n.fileCount += child.fileCount
|
||||
n.totalSize += child.totalSize
|
||||
}
|
||||
|
||||
slices.Sort(entries)
|
||||
// computeDigest sets n's digest and drops its entries. A directory's
|
||||
// digest is the SHA-256 of its child entries — files serialized with
|
||||
// name and signature, subdirectories with name and recursive digest —
|
||||
// sorted byte-lexicographically. Filenames cannot contain NUL or "/",
|
||||
// so NUL delimiters are unambiguous.
|
||||
func (n *treeNode) computeDigest() {
|
||||
slices.Sort(n.entries)
|
||||
|
||||
h := sha256.New()
|
||||
for _, e := range entries {
|
||||
for _, e := range n.entries {
|
||||
h.Write([]byte(e))
|
||||
h.Write([]byte{0})
|
||||
}
|
||||
|
||||
copy(n.digest[:], h.Sum(nil))
|
||||
n.entries = nil
|
||||
}
|
||||
|
||||
// suppressed reports whether a duplicate-tree group is non-maximal: its
|
||||
|
||||
Reference in New Issue
Block a user