package main import ( "bytes" "database/sql" "errors" "fmt" "io" "os" "path/filepath" "slices" "strconv" "strings" "testing" "time" ) // awkwardDir is a directory name holding every byte the reports escape. const awkwardDir = "/d/\tone\ntwo\rthree\\four" // awkwardPairRecs is a duplicate pair in sibling directories /d/A and // awkwardDir. A raw tab sorts before "A" but its escaped form `\t` // sorts after it, so awkwardDir coming first shows that sorting uses // the raw path. func awkwardPairRecs() []scanRec { return []scanRec{ {size: 5, head: "h", tail: "t", content: "c", path: "/d/A/f"}, {size: 5, head: "h", tail: "t", content: "c", path: awkwardDir + "/f"}, } } // seedDatabase writes recs into a fresh database and returns its path. func seedDatabase(t *testing.T, recs []scanRec) string { t.Helper() path := testDBPath(t) db, err := openScanDatabase(t.Context(), path) if err != nil { t.Fatal(err) } err = applyChanges(t.Context(), db, recs, nil, nil) if err != nil { t.Fatal(err) } err = db.Close() if err != nil { t.Fatal(err) } return path } // dupeGroup is one duplicate group as report reads it: the size, and // the paths in report order, first path first. type dupeGroup struct { size int64 paths []string } // dupeGroups returns the duplicate groups report reads from db, in // report order, from the rows it writes: a group's rows come together, // each naming the group's first path. func dupeGroups(t *testing.T, db *sql.DB) []dupeGroup { t.Helper() var out bytes.Buffer _, err := readReport(t.Context(), db, "", &out, nil) if err != nil { t.Fatal(err) } var groups []dupeGroup rows := strings.Split(strings.TrimSuffix(out.String(), "\n"), "\n") for _, row := range rows[1:] { // after the header fields := strings.Split(row, "\t") if len(fields) != 3 { t.Fatalf("row %q, want first, dupe and size", row) } size, err := strconv.ParseInt(fields[2], 10, 64) if err != nil { t.Fatal(err) } if len(groups) == 0 || groups[len(groups)-1].paths[0] != fields[0] { groups = append(groups, dupeGroup{size: size, paths: []string{fields[0]}}) } g := &groups[len(groups)-1] g.paths = append(g.paths, fields[1]) } return groups } // dupeGroupsOf writes recs into a fresh database and returns the // duplicate groups report reads from it. func dupeGroupsOf(t *testing.T, recs []scanRec) []dupeGroup { t.Helper() db := openTestDB(t) err := applyChanges(t.Context(), db, recs, nil, nil) if err != nil { t.Fatal(err) } return dupeGroups(t, db) } func TestRunReportEscapesPaths(t *testing.T) { t.Setenv(databaseEnv, seedDatabase(t, awkwardPairRecs())) var stdout, stderr bytes.Buffer code := run([]string{cmdReport}, &stdout, &stderr) if code != exitOK { t.Fatalf("run(report) = %d, want %d; stderr: %s", code, exitOK, stderr.String()) } want := "first\tdupe\tsize\n" + `/d/\tone\ntwo\rthree\\four/f` + "\t/d/A/f\t5\n" if got := stdout.String(); got != want { t.Errorf("stdout = %q, want %q", got, want) } } func TestReportStdoutFailsWhileReading(t *testing.T) { t.Parallel() // Every two records are a duplicate pair of a size of their own, and // a pair's row is written once a smaller size is read. Each row // holds two paths longer than dir, so the rows come to more than // twice the stdout buffer, and stdout fails while records are still // being read, not once every record is. dir := "/" + strings.Repeat("d", 4096) recs := make([]scanRec, 2*ioBufSize/len(dir)) for i := range recs { recs[i] = scanRec{ size: int64(i/2) + 1, head: "h", tail: "t", content: "c", path: fmt.Sprintf("%s/%d", dir, i), } } db := openTestDB(t) err := applyChanges(t.Context(), db, recs, nil, nil) if err != nil { t.Fatal(err) } // Counts the records read, and draws nothing. prog := &progress{} _, err = readReport(t.Context(), db, "", failingWriter{}, prog) if !errors.Is(err, errWriteFailed) || !strings.HasPrefix(err.Error(), "write stdout: ") { t.Errorf("error = %v, want write stdout: %v", err, errWriteFailed) } if prog.count >= int64(len(recs)) { t.Errorf("stdout failed after all %d records were read, want it "+ "to stop the read", prog.count) } } // reportOpenLine is the plain line of report's open phase when it opens // the database at once; it prints it as the phase starts and as it ends. const reportOpenLine = "[1/2] open: opening the database, elapsed 0s, " + "eta unknown" // smokeReport is report's stdout for the smoke-test records. const smokeReport = "first\tdupe\tsize\n" + "/d/t1/f1\t/d/t2/f1\t3000\n" + "/d/t1/f1\t/d/t3/f1\t3000\n" + "/d/t1/sub/f2\t/d/t2/sub/f2\t100\n" + "/d/t1/sub/f2\t/d/t3/sub/f2renamed\t100\n" // TestRunReportProgressNamesEachPhase runs report on the smoke-test // records with stderr not a terminal, and checks every line it prints // there. After the line naming the database, each phase prints a line // naming it with its place among the two as it starts and another as // it ends; read counts the 6 records before it reads them. The summary // comes last, and stdout holds the report alone. func TestRunReportProgressNamesEachPhase(t *testing.T) { path := seedDatabase(t, smokeTreeRecs()) t.Setenv(databaseEnv, path) stderr := captureStderr(t) var stdout bytes.Buffer code := run([]string{cmdReport}, &stdout, os.Stderr) if code != exitOK { t.Fatalf("run(report) = %d, want %d; stderr: %s", code, exitOK, stderr()) } if got := stdout.String(); got != smokeReport { t.Errorf("stdout = %q, want %q", got, smokeReport) } // How each line starts; the rate at the end of a phase varies. want := []string{ "report: database " + path + " (from " + databaseEnv + ")", reportOpenLine, reportOpenLine, "[2/2] read: counting the records, elapsed 0s, eta unknown", "[2/2] read: [6/6] 100%, ", "report: 6 records read, 2 duplicate groups, 4 dupe files, " + "6.1 KiB reclaimable", } lines := strings.Split(strings.TrimSuffix(stderr(), "\n"), "\n") if len(lines) != len(want) { t.Fatalf("stderr lines = %q, want %d lines starting %q", lines, len(want), want) } for i := range want { if !strings.HasPrefix(lines[i], want[i]) { t.Errorf("stderr line %d = %q, want it to start %q", i+1, lines[i], want[i]) } } } // TestReadReportOnTerminal writes the smoke-test report to the terminal // its read phase's display is drawn on, as a plain "sfdupes report" // does: the terminal must show the header and every row whole, each on // a line of its own. // //nolint:paralleltest // captureStderr replaces the process-wide os.Stderr func TestReadReportOnTerminal(t *testing.T) { stderr := captureStderr(t) db := openTestDB(t) err := applyChanges(t.Context(), db, smokeTreeRecs(), nil, nil) if err != nil { t.Fatal(err) } // No timed draws: the display is drawn as it starts and as it ends. p := startProgress(progressParams{ label: reportPhase("read"), unit: "records", total: -1, waitingFor: countingRecords, }, terminalWidth, time.Hour) _, err = readReport(t.Context(), db, "", os.Stderr, p) p.finish() if err != nil { t.Fatal(err) } // A terminal shows each line as the text after its last carriage // return. var shown []string for line := range strings.SplitSeq(stderr(), "\n") { shown = append(shown, line[strings.LastIndex(line, "\r")+1:]) } want := strings.Split(strings.TrimSuffix(smokeReport, "\n"), "\n") if len(shown) < len(want) || !slices.Equal(shown[:len(want)], want) { t.Errorf("terminal shows lines %q, want %q first", shown, want) } } func TestRunReportsIgnoreInsertionOrder(t *testing.T) { // README §Constraints: identical database contents give identical // output, whatever order the records were inserted in. recs := append(smokeTreeRecs(), awkwardPairRecs()...) recs = append(recs, scanRec{size: 50, head: "b", tail: "b", content: "b", path: "/y/2"}, scanRec{size: 50, head: "b", tail: "b", content: "b", path: "/y/1"}, scanRec{size: 50, head: "a", tail: "a", content: "a", path: "/x/2"}, scanRec{size: 50, head: "a", tail: "a", content: "a", path: "/x/1"}, scanRec{size: 50, path: "/x/unhashed"}, ) reversed := slices.Clone(recs) slices.Reverse(reversed) for _, name := range []string{cmdReport, cmdTrees} { t.Run(name, func(t *testing.T) { t.Setenv(databaseEnv, seedDatabase(t, recs)) forward := runStdout(t, name) t.Setenv(databaseEnv, seedDatabase(t, reversed)) backward := runStdout(t, name) if strings.Count(forward, "\n") < 3 { t.Errorf("stdout = %q, want at least two rows", forward) } if forward != backward { t.Errorf("stdout depends on insertion order: %q vs %q", forward, backward) } }) } } // runStdout runs the subcommand name and returns its stdout, failing // the test unless it succeeds. func runStdout(t *testing.T, name string) string { t.Helper() var stdout, stderr bytes.Buffer code := run([]string{name}, &stdout, &stderr) if code != exitOK { t.Fatalf("run(%s) = %d, want %d; stderr: %s", name, code, exitOK, stderr.String()) } return stdout.String() } func TestEscapePath(t *testing.T) { t.Parallel() cases := map[string]string{ "/srv/plain": "/srv/plain", "/a\tb": `/a\tb`, "/a\nb": `/a\nb`, "/a\rb": `/a\rb`, `/a\b`: `/a\\b`, `/a\tb`: `/a\\tb`, "/not-utf8\xff": "/not-utf8\xff", } for in, want := range cases { if got := escapePath(in); got != want { t.Errorf("escapePath(%q) = %q, want %q", in, got, want) } } } // TestWarnfEscapes checks that a warning naming a path that holds a // newline is still one line. // //nolint:paralleltest // replaces the process-wide os.Stderr func TestWarnfEscapes(t *testing.T) { f, err := os.Create(filepath.Join(t.TempDir(), "stderr")) if err != nil { t.Fatal(err) } saved := os.Stderr os.Stderr = f t.Cleanup(func() { os.Stderr = saved _ = f.Close() }) (&progress{}).warnf("stat %s: %s", "/d/a\nb", "gone") _, err = f.Seek(0, io.SeekStart) if err != nil { t.Fatal(err) } got, err := io.ReadAll(f) if err != nil { t.Fatal(err) } want := `stat /d/a\nb: gone` + "\n" if string(got) != want { t.Errorf("warning = %q, want %q", got, want) } } func TestDupeGroups(t *testing.T) { t.Parallel() recs := []scanRec{ {size: 100, head: "h", tail: "t", content: "c", path: "/z/b"}, {size: 100, head: "h", tail: "t", content: "c", path: "/z/a"}, {size: 100, head: "h", tail: "t", content: "c", path: "/z/c"}, {size: 4000, head: "H", tail: "T", content: "C", path: "/big/2"}, {size: 4000, head: "H", tail: "T", content: "C", path: "/big/1"}, // Same size as the /z group but a different head hash. {size: 100, head: "other", tail: "t", content: "c", path: "/z/d"}, // A singleton signature must not form a group. {size: 7, head: "u", tail: "u", content: "u", path: "/lonely"}, } groups := dupeGroupsOf(t, recs) if len(groups) != 2 { t.Fatalf("len(groups) = %d, want 2", len(groups)) } if groups[0].size != 4000 || !slices.Equal(groups[0].paths, []string{"/big/1", "/big/2"}) { t.Errorf("groups[0] = %+v, want size 4000, paths /big/1 /big/2", groups[0]) } if groups[1].size != 100 || !slices.Equal(groups[1].paths, []string{"/z/a", "/z/b", "/z/c"}) { t.Errorf("groups[1] = %+v, want size 100, paths /z/a /z/b /z/c", groups[1]) } } func TestDupeGroupsContentSeparates(t *testing.T) { t.Parallel() // Same size, head, and tail, but different content hashes: the final // rung keeps them apart, so no group forms. Matching content groups. // Records without a content hash never group, not even with each // other. recs := []scanRec{ {size: 100, head: "h", tail: "t", content: "c1", path: "/a"}, {size: 100, head: "h", tail: "t", content: "c2", path: "/b"}, {size: 100, head: "h", tail: "t", content: "c1", path: "/c"}, {size: 100, head: "h", tail: "t", path: "/d"}, {size: 100, head: "h", tail: "t", path: "/e"}, } groups := dupeGroupsOf(t, recs) if len(groups) != 1 { t.Fatalf("len(groups) = %d, want 1 (only the matching content)", len(groups)) } if !slices.Equal(groups[0].paths, []string{"/a", "/c"}) { t.Errorf("group paths = %q, want /a /c", groups[0].paths) } } func TestDupeGroupsMtimeExcluded(t *testing.T) { t.Parallel() // mtime is informational only; records differing only in mtime // still group together. recs := []scanRec{ { size: 9, mtime: time.Unix(100, 0), head: "h", tail: "t", content: "c", path: "/m/1", }, { size: 9, mtime: time.Unix(200, 0), head: "h", tail: "t", content: "c", path: "/m/2", }, } groups := dupeGroupsOf(t, recs) if len(groups) != 1 { t.Fatalf("len(groups) = %d, want 1", len(groups)) } } func TestDupeGroupsTieBreak(t *testing.T) { t.Parallel() // The hashes sort opposite to the first paths, so ordering the // groups by hash instead of by first path fails this test. recs := []scanRec{ {size: 50, head: "a", tail: "a", content: "a", path: "/beta/2"}, {size: 50, head: "a", tail: "a", content: "a", path: "/beta/1"}, {size: 50, head: "b", tail: "b", content: "b", path: "/alpha/2"}, {size: 50, head: "b", tail: "b", content: "b", path: "/alpha/1"}, } groups := dupeGroupsOf(t, recs) if len(groups) != 2 { t.Fatalf("len(groups) = %d, want 2", len(groups)) } // Equal sizes: ordered by first path ascending. if groups[0].paths[0] != "/alpha/1" || groups[1].paths[0] != "/beta/1" { t.Fatalf("tie-break order wrong: %q then %q", groups[0].paths[0], groups[1].paths[0]) } } func TestDupeGroupsOrderedByFirstPath(t *testing.T) { t.Parallel() // report reads the groups of size 50 in signature order, here /a, // /z, /k, which is neither the order of their first paths nor its // reverse, and must write them ordered by first path. The group of // size 7 comes after them all, though its first path sorts first. recs := []scanRec{ {size: 50, head: "a", tail: "a", content: "a", path: "/k/2"}, {size: 50, head: "a", tail: "a", content: "a", path: "/k/1"}, {size: 50, head: "b", tail: "b", content: "b", path: "/z/1"}, {size: 50, head: "b", tail: "b", content: "b", path: "/z/2"}, {size: 50, head: "c", tail: "c", content: "c", path: "/a/2"}, {size: 50, head: "c", tail: "c", content: "c", path: "/a/1"}, {size: 7, head: "d", tail: "d", content: "d", path: "/0/2"}, {size: 7, head: "d", tail: "d", content: "d", path: "/0/1"}, } groups := dupeGroupsOf(t, recs) got := make([]string, 0, len(groups)) for _, g := range groups { got = append(got, g.paths[0]) } want := []string{"/a/1", "/k/1", "/z/1", "/0/1"} if !slices.Equal(got, want) { t.Errorf("groups' first paths = %q, want %q", got, want) } } func TestDupeGroupsDeterministic(t *testing.T) { t.Parallel() recs := []scanRec{ {size: 1, head: "a", tail: "a", content: "a", path: "/p/1"}, {size: 1, head: "a", tail: "a", content: "a", path: "/p/2"}, {size: 2, head: "b", tail: "b", content: "b", path: "/q/1"}, {size: 2, head: "b", tail: "b", content: "b", path: "/q/2"}, } forward := dupeGroupsOf(t, recs) reversed := slices.Clone(recs) slices.Reverse(reversed) backward := dupeGroupsOf(t, reversed) if !slices.EqualFunc(forward, backward, func(a, b dupeGroup) bool { return a.size == b.size && slices.Equal(a.paths, b.paths) }) { t.Fatalf("output depends on record order: %+v vs %+v", forward, backward) } } func TestHumanBytes(t *testing.T) { t.Parallel() cases := []struct { n int64 want string }{ {0, "0 B"}, {1, "1 B"}, {1023, "1023 B"}, {1024, "1.0 KiB"}, {1536, "1.5 KiB"}, {1 << 20, "1.0 MiB"}, {5 << 30, "5.0 GiB"}, {1 << 40, "1.0 TiB"}, {1 << 50, "1.0 PiB"}, {1 << 60, "1.0 EiB"}, } for _, c := range cases { if got := humanBytes(c.n); got != c.want { t.Errorf("humanBytes(%d) = %q, want %q", c.n, got, c.want) } } }