Make code lint-clean under the canonical golangci-lint config
150 findings fixed: linter autofixes for whitespace style (wsl_v5, nlreturn, noinlineerr), named returns removed, magic numbers replaced with named constants, static errNotRegular error, explicit Close/Parse error handling, unused cobra params renamed to _, and runReport/ runTrees/hashPass split into helpers to satisfy cyclop, gocognit, and funlen. Behavior verified unchanged against the README smoke test.
This commit is contained in:
26
main.go
26
main.go
@@ -20,15 +20,23 @@ import (
|
|||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// Exit codes: 0 is success (even with per-file warnings), exitFatal is
|
||||||
|
// a fatal error, exitUsage is a usage error.
|
||||||
|
const (
|
||||||
|
exitFatal = 1
|
||||||
|
exitUsage = 2
|
||||||
|
)
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
root := &cobra.Command{
|
root := &cobra.Command{
|
||||||
Use: "sfdupes",
|
Use: "sfdupes",
|
||||||
Short: "Find candidate duplicate files by size and head/tail SHA-256",
|
Short: "Find candidate duplicate files by size and head/tail SHA-256",
|
||||||
Args: cobra.NoArgs,
|
Args: cobra.NoArgs,
|
||||||
Run: func(cmd *cobra.Command, args []string) {
|
Run: func(cmd *cobra.Command, _ []string) {
|
||||||
// A missing subcommand prints usage and exits 2.
|
// A missing subcommand prints usage and exits 2.
|
||||||
_ = cmd.Usage()
|
_ = cmd.Usage()
|
||||||
os.Exit(2)
|
|
||||||
|
os.Exit(exitUsage)
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
// Everything on stdout is machine-readable data; all human-facing
|
// Everything on stdout is machine-readable data; all human-facing
|
||||||
@@ -43,7 +51,7 @@ func main() {
|
|||||||
// The README specifies single-dash flags (-root, -workers);
|
// The README specifies single-dash flags (-root, -workers);
|
||||||
// parse them with the stdlib flag package inside runScan.
|
// parse them with the stdlib flag package inside runScan.
|
||||||
DisableFlagParsing: true,
|
DisableFlagParsing: true,
|
||||||
Run: func(cmd *cobra.Command, args []string) {
|
Run: func(_ *cobra.Command, args []string) {
|
||||||
runScan(args)
|
runScan(args)
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
@@ -52,7 +60,7 @@ func main() {
|
|||||||
Use: "report [files.dat|-]",
|
Use: "report [files.dat|-]",
|
||||||
Short: "Read a scan stream and print the file-level duplicates report",
|
Short: "Read a scan stream and print the file-level duplicates report",
|
||||||
Args: cobra.MaximumNArgs(1),
|
Args: cobra.MaximumNArgs(1),
|
||||||
Run: func(cmd *cobra.Command, args []string) {
|
Run: func(_ *cobra.Command, args []string) {
|
||||||
runReport(args)
|
runReport(args)
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
@@ -61,21 +69,23 @@ func main() {
|
|||||||
Use: "trees [files.dat|-]",
|
Use: "trees [files.dat|-]",
|
||||||
Short: "Read a scan stream and print the duplicate-tree report",
|
Short: "Read a scan stream and print the duplicate-tree report",
|
||||||
Args: cobra.MaximumNArgs(1),
|
Args: cobra.MaximumNArgs(1),
|
||||||
Run: func(cmd *cobra.Command, args []string) {
|
Run: func(_ *cobra.Command, args []string) {
|
||||||
runTrees(args)
|
runTrees(args)
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
root.AddCommand(scanCmd, reportCmd, treesCmd)
|
root.AddCommand(scanCmd, reportCmd, treesCmd)
|
||||||
if err := root.Execute(); err != nil {
|
|
||||||
|
err := root.Execute()
|
||||||
|
if err != nil {
|
||||||
// Cobra has already printed the error and usage to stderr;
|
// Cobra has already printed the error and usage to stderr;
|
||||||
// an invalid subcommand or bad arguments is a usage error.
|
// an invalid subcommand or bad arguments is a usage error.
|
||||||
os.Exit(2)
|
os.Exit(exitUsage)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// fatalf reports a fatal error and exits 1.
|
// fatalf reports a fatal error and exits 1.
|
||||||
func fatalf(format string, args ...any) {
|
func fatalf(format string, args ...any) {
|
||||||
fmt.Fprintf(os.Stderr, "sfdupes: "+format+"\n", args...)
|
fmt.Fprintf(os.Stderr, "sfdupes: "+format+"\n", args...)
|
||||||
os.Exit(1)
|
os.Exit(exitFatal)
|
||||||
}
|
}
|
||||||
|
|||||||
33
progress.go
33
progress.go
@@ -12,12 +12,23 @@ import (
|
|||||||
// is not a TTY.
|
// is not a TTY.
|
||||||
const plainInterval = 5 * time.Second
|
const plainInterval = 5 * time.Second
|
||||||
|
|
||||||
|
// barThrottle caps how often the TTY progress bar redraws.
|
||||||
|
const barThrottle = 100 * time.Millisecond
|
||||||
|
|
||||||
|
// walkSpinnerType selects the progressbar library's spinner style used
|
||||||
|
// for the walk pass, which has no known total.
|
||||||
|
const walkSpinnerType = 14
|
||||||
|
|
||||||
|
// percentScale converts a fraction to a percentage.
|
||||||
|
const percentScale = 100
|
||||||
|
|
||||||
// stderrIsTTY reports whether stderr is attached to a terminal.
|
// stderrIsTTY reports whether stderr is attached to a terminal.
|
||||||
func stderrIsTTY() bool {
|
func stderrIsTTY() bool {
|
||||||
fi, err := os.Stderr.Stat()
|
fi, err := os.Stderr.Stat()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
return fi.Mode()&os.ModeCharDevice != 0
|
return fi.Mode()&os.ModeCharDevice != 0
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -42,6 +53,7 @@ func newProgress(label string, total int64) *progress {
|
|||||||
if !stderrIsTTY() {
|
if !stderrIsTTY() {
|
||||||
return p
|
return p
|
||||||
}
|
}
|
||||||
|
|
||||||
opts := []progressbar.Option{
|
opts := []progressbar.Option{
|
||||||
progressbar.OptionSetWriter(os.Stderr),
|
progressbar.OptionSetWriter(os.Stderr),
|
||||||
progressbar.OptionSetDescription(label),
|
progressbar.OptionSetDescription(label),
|
||||||
@@ -49,7 +61,7 @@ func newProgress(label string, total int64) *progress {
|
|||||||
progressbar.OptionShowIts(),
|
progressbar.OptionShowIts(),
|
||||||
progressbar.OptionSetItsString("files"),
|
progressbar.OptionSetItsString("files"),
|
||||||
progressbar.OptionSetElapsedTime(true),
|
progressbar.OptionSetElapsedTime(true),
|
||||||
progressbar.OptionThrottle(100 * time.Millisecond),
|
progressbar.OptionThrottle(barThrottle),
|
||||||
}
|
}
|
||||||
if total >= 0 {
|
if total >= 0 {
|
||||||
opts = append(opts,
|
opts = append(opts,
|
||||||
@@ -59,10 +71,12 @@ func newProgress(label string, total int64) *progress {
|
|||||||
} else {
|
} else {
|
||||||
opts = append(opts,
|
opts = append(opts,
|
||||||
progressbar.OptionSetPredictTime(false),
|
progressbar.OptionSetPredictTime(false),
|
||||||
progressbar.OptionSpinnerType(14),
|
progressbar.OptionSpinnerType(walkSpinnerType),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
p.bar = progressbar.NewOptions64(total, opts...)
|
p.bar = progressbar.NewOptions64(total, opts...)
|
||||||
|
|
||||||
return p
|
return p
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -71,8 +85,10 @@ func (p *progress) increment() {
|
|||||||
p.count++
|
p.count++
|
||||||
if p.bar != nil {
|
if p.bar != nil {
|
||||||
_ = p.bar.Add(1)
|
_ = p.bar.Add(1)
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if time.Since(p.last) >= plainInterval {
|
if time.Since(p.last) >= plainInterval {
|
||||||
p.last = time.Now()
|
p.last = time.Now()
|
||||||
fmt.Fprintln(os.Stderr, p.plainLine())
|
fmt.Fprintln(os.Stderr, p.plainLine())
|
||||||
@@ -84,6 +100,7 @@ func (p *progress) warnf(format string, args ...any) {
|
|||||||
if p.bar != nil {
|
if p.bar != nil {
|
||||||
_ = p.bar.Clear()
|
_ = p.bar.Clear()
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Fprintf(os.Stderr, format+"\n", args...)
|
fmt.Fprintf(os.Stderr, format+"\n", args...)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -91,9 +108,12 @@ func (p *progress) warnf(format string, args ...any) {
|
|||||||
func (p *progress) finish() {
|
func (p *progress) finish() {
|
||||||
if p.bar != nil {
|
if p.bar != nil {
|
||||||
_ = p.bar.Finish()
|
_ = p.bar.Finish()
|
||||||
|
|
||||||
fmt.Fprintln(os.Stderr)
|
fmt.Fprintln(os.Stderr)
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Fprintln(os.Stderr, p.plainLine())
|
fmt.Fprintln(os.Stderr, p.plainLine())
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -104,19 +124,24 @@ func (p *progress) plainLine() string {
|
|||||||
return fmt.Sprintf("%s: %d files, elapsed %s",
|
return fmt.Sprintf("%s: %d files, elapsed %s",
|
||||||
p.label, p.count, elapsed.Round(time.Second))
|
p.label, p.count, elapsed.Round(time.Second))
|
||||||
}
|
}
|
||||||
pct := 100.0
|
|
||||||
|
pct := float64(percentScale)
|
||||||
if p.total > 0 {
|
if p.total > 0 {
|
||||||
pct = 100 * float64(p.count) / float64(p.total)
|
pct = percentScale * float64(p.count) / float64(p.total)
|
||||||
}
|
}
|
||||||
|
|
||||||
rate := 0.0
|
rate := 0.0
|
||||||
if s := elapsed.Seconds(); s > 0 {
|
if s := elapsed.Seconds(); s > 0 {
|
||||||
rate = float64(p.count) / s
|
rate = float64(p.count) / s
|
||||||
}
|
}
|
||||||
|
|
||||||
eta := "?"
|
eta := "?"
|
||||||
|
|
||||||
if rate > 0 && p.count <= p.total {
|
if rate > 0 && p.count <= p.total {
|
||||||
remaining := time.Duration(float64(p.total-p.count) / rate * float64(time.Second))
|
remaining := time.Duration(float64(p.total-p.count) / rate * float64(time.Second))
|
||||||
eta = remaining.Round(time.Second).String()
|
eta = remaining.Round(time.Second).String()
|
||||||
}
|
}
|
||||||
|
|
||||||
return fmt.Sprintf("%s: [%d/%d] %.0f%% %.0f files/s elapsed %s eta %s",
|
return fmt.Sprintf("%s: [%d/%d] %.0f%% %.0f files/s elapsed %s eta %s",
|
||||||
p.label, p.count, p.total, pct, rate,
|
p.label, p.count, p.total, pct, rate,
|
||||||
elapsed.Round(time.Second), eta)
|
elapsed.Round(time.Second), eta)
|
||||||
|
|||||||
147
report.go
147
report.go
@@ -11,6 +11,26 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// ioBufSize is the buffer size for the buffered scan-stream reader and
|
||||||
|
// the buffered stdout writers.
|
||||||
|
const ioBufSize = 1 << 20
|
||||||
|
|
||||||
|
// recordFieldCount is the number of tab-separated fields in a scan
|
||||||
|
// record: size, mtime, head hash, tail hash, path.
|
||||||
|
const recordFieldCount = 5
|
||||||
|
|
||||||
|
// scanInitBufSize and scanMaxRecordSize bound the scanner buffer used
|
||||||
|
// to read scan records; a record longer than scanMaxRecordSize is a
|
||||||
|
// fatal read error.
|
||||||
|
const (
|
||||||
|
scanInitBufSize = 64 << 10
|
||||||
|
scanMaxRecordSize = 4 << 20
|
||||||
|
)
|
||||||
|
|
||||||
|
// minGroupSize is the smallest number of members that makes a
|
||||||
|
// duplicate group.
|
||||||
|
const minGroupSize = 2
|
||||||
|
|
||||||
// scanRec is one well-formed record parsed from a scan stream. The
|
// scanRec is one well-formed record parsed from a scan stream. The
|
||||||
// signature (size, head, tail) is the duplicate key; mtime is
|
// signature (size, head, tail) is the duplicate key; mtime is
|
||||||
// informational and not retained.
|
// informational and not retained.
|
||||||
@@ -24,14 +44,16 @@ type scanRec struct {
|
|||||||
// openScanInput resolves the analysis-mode input: the file named by the
|
// openScanInput resolves the analysis-mode input: the file named by the
|
||||||
// single optional positional argument, or stdin when it is absent or
|
// single optional positional argument, or stdin when it is absent or
|
||||||
// "-". The returned closer must be called when reading is done.
|
// "-". The returned closer must be called when reading is done.
|
||||||
func openScanInput(args []string) (in io.Reader, name string, closer func()) {
|
func openScanInput(args []string) (io.Reader, string, func()) {
|
||||||
if len(args) == 1 && args[0] != "-" {
|
if len(args) == 1 && args[0] != "-" {
|
||||||
f, err := os.Open(args[0])
|
f, err := os.Open(args[0])
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fatalf("open %s: %v", args[0], err)
|
fatalf("open %s: %v", args[0], err)
|
||||||
}
|
}
|
||||||
return f, args[0], func() { f.Close() }
|
|
||||||
|
return f, args[0], func() { _ = f.Close() }
|
||||||
}
|
}
|
||||||
|
|
||||||
return os.Stdin, "stdin", func() {}
|
return os.Stdin, "stdin", func() {}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -39,23 +61,33 @@ func openScanInput(args []string) (in io.Reader, name string, closer func()) {
|
|||||||
// that does not have exactly 5 fields or whose size is non-numeric is
|
// that does not have exactly 5 fields or whose size is non-numeric is
|
||||||
// counted as malformed and skipped. Total records read is
|
// counted as malformed and skipped. Total records read is
|
||||||
// len(recs) + malformed.
|
// len(recs) + malformed.
|
||||||
func parseScanStream(in io.Reader, name string) (recs []scanRec, malformed int) {
|
func parseScanStream(in io.Reader, name string) ([]scanRec, int) {
|
||||||
sc := bufio.NewScanner(bufio.NewReaderSize(in, 1<<20))
|
sc := bufio.NewScanner(bufio.NewReaderSize(in, ioBufSize))
|
||||||
sc.Buffer(make([]byte, 0, 64<<10), 4<<20)
|
sc.Buffer(make([]byte, 0, scanInitBufSize), scanMaxRecordSize)
|
||||||
sc.Split(splitNUL)
|
sc.Split(splitNUL)
|
||||||
|
|
||||||
|
var (
|
||||||
|
recs []scanRec
|
||||||
|
malformed int
|
||||||
|
)
|
||||||
|
|
||||||
for sc.Scan() {
|
for sc.Scan() {
|
||||||
// The path is the last field and may itself contain tabs,
|
// The path is the last field and may itself contain tabs,
|
||||||
// so split into at most 5 fields.
|
// so split into at most recordFieldCount fields.
|
||||||
fields := strings.SplitN(sc.Text(), "\t", 5)
|
fields := strings.SplitN(sc.Text(), "\t", recordFieldCount)
|
||||||
if len(fields) != 5 {
|
if len(fields) != recordFieldCount {
|
||||||
malformed++
|
malformed++
|
||||||
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
size, err := strconv.ParseInt(fields[0], 10, 64)
|
size, err := strconv.ParseInt(fields[0], 10, 64)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
malformed++
|
malformed++
|
||||||
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
recs = append(recs, scanRec{
|
recs = append(recs, scanRec{
|
||||||
size: size,
|
size: size,
|
||||||
head: fields[2],
|
head: fields[2],
|
||||||
@@ -63,9 +95,12 @@ func parseScanStream(in io.Reader, name string) (recs []scanRec, malformed int)
|
|||||||
path: fields[4],
|
path: fields[4],
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
if err := sc.Err(); err != nil {
|
|
||||||
|
err := sc.Err()
|
||||||
|
if err != nil {
|
||||||
fatalf("read %s: %v", name, err)
|
fatalf("read %s: %v", name, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return recs, malformed
|
return recs, malformed
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -86,56 +121,37 @@ type dupeGroup struct {
|
|||||||
func runReport(args []string) {
|
func runReport(args []string) {
|
||||||
in, name, closer := openScanInput(args)
|
in, name, closer := openScanInput(args)
|
||||||
defer closer()
|
defer closer()
|
||||||
|
|
||||||
recs, malformed := parseScanStream(in, name)
|
recs, malformed := parseScanStream(in, name)
|
||||||
records := len(recs) + malformed
|
records := len(recs) + malformed
|
||||||
|
dupes := collectDupeGroups(recs)
|
||||||
|
|
||||||
type dupeKey struct {
|
out := bufio.NewWriterSize(os.Stdout, ioBufSize)
|
||||||
size int64
|
|
||||||
head string
|
|
||||||
tail string
|
|
||||||
}
|
|
||||||
groups := make(map[dupeKey][]string)
|
|
||||||
for _, r := range recs {
|
|
||||||
k := dupeKey{size: r.size, head: r.head, tail: r.tail}
|
|
||||||
groups[k] = append(groups[k], r.path)
|
|
||||||
}
|
|
||||||
|
|
||||||
var dupes []dupeGroup
|
_, err := fmt.Fprintln(out, "first\tdupe\tsize")
|
||||||
for k, paths := range groups {
|
if err != nil {
|
||||||
if len(paths) < 2 {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
slices.Sort(paths)
|
|
||||||
dupes = append(dupes, dupeGroup{size: k.size, paths: paths})
|
|
||||||
}
|
|
||||||
// Biggest reclaimable space first; ties broken by first path.
|
|
||||||
slices.SortFunc(dupes, func(a, b dupeGroup) int {
|
|
||||||
if a.size != b.size {
|
|
||||||
if a.size > b.size {
|
|
||||||
return -1
|
|
||||||
}
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
return strings.Compare(a.paths[0], b.paths[0])
|
|
||||||
})
|
|
||||||
|
|
||||||
out := bufio.NewWriterSize(os.Stdout, 1<<20)
|
|
||||||
if _, err := fmt.Fprintln(out, "first\tdupe\tsize"); err != nil {
|
|
||||||
fatalf("write stdout: %v", err)
|
fatalf("write stdout: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
dupeFiles := 0
|
dupeFiles := 0
|
||||||
|
|
||||||
var reclaimable int64
|
var reclaimable int64
|
||||||
|
|
||||||
for _, g := range dupes {
|
for _, g := range dupes {
|
||||||
for _, p := range g.paths[1:] {
|
for _, p := range g.paths[1:] {
|
||||||
if _, err := fmt.Fprintf(out, "%s\t%s\t%d\n",
|
_, err = fmt.Fprintf(out, "%s\t%s\t%d\n",
|
||||||
g.paths[0], p, g.size); err != nil {
|
g.paths[0], p, g.size)
|
||||||
|
if err != nil {
|
||||||
fatalf("write stdout: %v", err)
|
fatalf("write stdout: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
dupeFiles++
|
dupeFiles++
|
||||||
reclaimable += g.size
|
reclaimable += g.size
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if err := out.Flush(); err != nil {
|
|
||||||
|
err = out.Flush()
|
||||||
|
if err != nil {
|
||||||
fatalf("write stdout: %v", err)
|
fatalf("write stdout: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -145,24 +161,65 @@ func runReport(args []string) {
|
|||||||
humanBytes(reclaimable))
|
humanBytes(reclaimable))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// collectDupeGroups groups records by signature and returns every group
|
||||||
|
// with two or more paths, each group's paths sorted lexicographically,
|
||||||
|
// groups ordered by size descending then by first path ascending.
|
||||||
|
func collectDupeGroups(recs []scanRec) []dupeGroup {
|
||||||
|
groups := make(map[fileSig][]string)
|
||||||
|
|
||||||
|
for _, r := range recs {
|
||||||
|
k := fileSig{size: r.size, head: r.head, tail: r.tail}
|
||||||
|
groups[k] = append(groups[k], r.path)
|
||||||
|
}
|
||||||
|
|
||||||
|
var dupes []dupeGroup
|
||||||
|
|
||||||
|
for k, paths := range groups {
|
||||||
|
if len(paths) < minGroupSize {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
slices.Sort(paths)
|
||||||
|
dupes = append(dupes, dupeGroup{size: k.size, paths: paths})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Biggest reclaimable space first; ties broken by first path.
|
||||||
|
slices.SortFunc(dupes, func(a, b dupeGroup) int {
|
||||||
|
if a.size != b.size {
|
||||||
|
if a.size > b.size {
|
||||||
|
return -1
|
||||||
|
}
|
||||||
|
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
return strings.Compare(a.paths[0], b.paths[0])
|
||||||
|
})
|
||||||
|
|
||||||
|
return dupes
|
||||||
|
}
|
||||||
|
|
||||||
// malformedNote formats the optional malformed-record note for the
|
// malformedNote formats the optional malformed-record note for the
|
||||||
// stderr summaries.
|
// stderr summaries.
|
||||||
func malformedNote(malformed int) string {
|
func malformedNote(malformed int) string {
|
||||||
if malformed == 0 {
|
if malformed == 0 {
|
||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
|
|
||||||
return fmt.Sprintf(" (%d malformed, skipped)", malformed)
|
return fmt.Sprintf(" (%d malformed, skipped)", malformed)
|
||||||
}
|
}
|
||||||
|
|
||||||
// splitNUL is a bufio.SplitFunc for NUL-terminated records. Trailing
|
// splitNUL is a bufio.SplitFunc for NUL-terminated records. Trailing
|
||||||
// data without a terminator at EOF is returned as a final record.
|
// data without a terminator at EOF is returned as a final record.
|
||||||
func splitNUL(data []byte, atEOF bool) (advance int, token []byte, err error) {
|
func splitNUL(data []byte, atEOF bool) (int, []byte, error) {
|
||||||
if i := bytes.IndexByte(data, 0); i >= 0 {
|
if i := bytes.IndexByte(data, 0); i >= 0 {
|
||||||
return i + 1, data[:i], nil
|
return i + 1, data[:i], nil
|
||||||
}
|
}
|
||||||
|
|
||||||
if atEOF && len(data) > 0 {
|
if atEOF && len(data) > 0 {
|
||||||
return len(data), data, nil
|
return len(data), data, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
return 0, nil, nil
|
return 0, nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -172,10 +229,12 @@ func humanBytes(n int64) string {
|
|||||||
if n < unit {
|
if n < unit {
|
||||||
return fmt.Sprintf("%d B", n)
|
return fmt.Sprintf("%d B", n)
|
||||||
}
|
}
|
||||||
|
|
||||||
div, exp := int64(unit), 0
|
div, exp := int64(unit), 0
|
||||||
for m := n / unit; m >= unit; m /= unit {
|
for m := n / unit; m >= unit; m /= unit {
|
||||||
div *= unit
|
div *= unit
|
||||||
exp++
|
exp++
|
||||||
}
|
}
|
||||||
|
|
||||||
return fmt.Sprintf("%.1f %ciB", float64(n)/float64(div), "KMGTPE"[exp])
|
return fmt.Sprintf("%.1f %ciB", float64(n)/float64(div), "KMGTPE"[exp])
|
||||||
}
|
}
|
||||||
|
|||||||
160
scan.go
160
scan.go
@@ -4,6 +4,7 @@ import (
|
|||||||
"bufio"
|
"bufio"
|
||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
|
"errors"
|
||||||
"flag"
|
"flag"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io/fs"
|
"io/fs"
|
||||||
@@ -15,6 +16,14 @@ import (
|
|||||||
// chunk is the number of bytes hashed from each end of a file.
|
// chunk is the number of bytes hashed from each end of a file.
|
||||||
const chunk = 1024
|
const chunk = 1024
|
||||||
|
|
||||||
|
// workQueueDepth bounds the job and result channels feeding the stat
|
||||||
|
// and hash worker pools.
|
||||||
|
const workQueueDepth = 1024
|
||||||
|
|
||||||
|
// errNotRegular reports a path that stopped being a regular file
|
||||||
|
// between the walk and stat passes.
|
||||||
|
var errNotRegular = errors.New("no longer a regular file")
|
||||||
|
|
||||||
// fileRec carries one file between the stat and hash passes.
|
// fileRec carries one file between the stat and hash passes.
|
||||||
type fileRec struct {
|
type fileRec struct {
|
||||||
path string
|
path string
|
||||||
@@ -30,11 +39,18 @@ func runScan(args []string) {
|
|||||||
root := fl.String("root", "/srv", "directory tree to scan")
|
root := fl.String("root", "/srv", "directory tree to scan")
|
||||||
workers := fl.Int("workers", runtime.NumCPU(),
|
workers := fl.Int("workers", runtime.NumCPU(),
|
||||||
"concurrent workers for the stat and hash passes")
|
"concurrent workers for the stat and hash passes")
|
||||||
fl.Parse(args)
|
// The flag set uses ExitOnError, so Parse cannot return a non-nil
|
||||||
|
// error; the check keeps the error handled explicitly.
|
||||||
|
err := fl.Parse(args)
|
||||||
|
if err != nil {
|
||||||
|
os.Exit(exitUsage)
|
||||||
|
}
|
||||||
|
|
||||||
if fl.NArg() > 0 {
|
if fl.NArg() > 0 {
|
||||||
fmt.Fprintf(os.Stderr, "scan: unexpected argument %q\n", fl.Arg(0))
|
fmt.Fprintf(os.Stderr, "scan: unexpected argument %q\n", fl.Arg(0))
|
||||||
os.Exit(2)
|
os.Exit(exitUsage)
|
||||||
}
|
}
|
||||||
|
|
||||||
if *workers < 1 {
|
if *workers < 1 {
|
||||||
*workers = 1
|
*workers = 1
|
||||||
}
|
}
|
||||||
@@ -43,6 +59,7 @@ func runScan(args []string) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
fatalf("root %s: %v", *root, err)
|
fatalf("root %s: %v", *root, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if !st.IsDir() {
|
if !st.IsDir() {
|
||||||
fatalf("root %s: not a directory", *root)
|
fatalf("root %s: not a directory", *root)
|
||||||
}
|
}
|
||||||
@@ -59,23 +76,33 @@ func runScan(args []string) {
|
|||||||
// walkPass enumerates every regular file under root. It never follows
|
// walkPass enumerates every regular file under root. It never follows
|
||||||
// symlinks, never descends into directories named .zfs, and warns and
|
// symlinks, never descends into directories named .zfs, and warns and
|
||||||
// continues on any per-path error.
|
// continues on any per-path error.
|
||||||
func walkPass(root string) (paths []string, errs int) {
|
func walkPass(root string) ([]string, int) {
|
||||||
|
var (
|
||||||
|
paths []string
|
||||||
|
errs int
|
||||||
|
)
|
||||||
|
|
||||||
prog := newProgress("walk", -1)
|
prog := newProgress("walk", -1)
|
||||||
walkErr := filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error {
|
walkErr := filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
errs++
|
errs++
|
||||||
|
|
||||||
prog.warnf("walk %s: %v", p, err)
|
prog.warnf("walk %s: %v", p, err)
|
||||||
|
|
||||||
if d != nil && d.IsDir() {
|
if d != nil && d.IsDir() {
|
||||||
return filepath.SkipDir
|
return filepath.SkipDir
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
if d.IsDir() {
|
if d.IsDir() {
|
||||||
// ZFS snapshot pseudo-dirs would list every file
|
// ZFS snapshot pseudo-dirs would list every file
|
||||||
// once per snapshot; never descend.
|
// once per snapshot; never descend.
|
||||||
if d.Name() == ".zfs" {
|
if d.Name() == ".zfs" {
|
||||||
return filepath.SkipDir
|
return filepath.SkipDir
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
// Regular files only: skip symlinks, sockets, FIFOs, and
|
// Regular files only: skip symlinks, sockets, FIFOs, and
|
||||||
@@ -83,27 +110,35 @@ func walkPass(root string) (paths []string, errs int) {
|
|||||||
if !d.Type().IsRegular() {
|
if !d.Type().IsRegular() {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
paths = append(paths, p)
|
paths = append(paths, p)
|
||||||
|
|
||||||
prog.increment()
|
prog.increment()
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
|
|
||||||
prog.finish()
|
prog.finish()
|
||||||
|
|
||||||
if walkErr != nil {
|
if walkErr != nil {
|
||||||
fatalf("walk %s: %v", root, walkErr)
|
fatalf("walk %s: %v", root, walkErr)
|
||||||
}
|
}
|
||||||
|
|
||||||
return paths, errs
|
return paths, errs
|
||||||
}
|
}
|
||||||
|
|
||||||
// statPass lstats every collected path in a worker pool, recording size
|
// statPass lstats every collected path in a worker pool, recording size
|
||||||
// and mtime. Paths that fail to stat (or are no longer regular files)
|
// and mtime. Paths that fail to stat (or are no longer regular files)
|
||||||
// are warned about and dropped.
|
// are warned about and dropped.
|
||||||
func statPass(paths []string, workers int) (recs []fileRec, errs int) {
|
func statPass(paths []string, workers int) ([]fileRec, int) {
|
||||||
type result struct {
|
type result struct {
|
||||||
rec fileRec
|
rec fileRec
|
||||||
err error
|
err error
|
||||||
}
|
}
|
||||||
jobs := make(chan string, 1024)
|
|
||||||
results := make(chan result, 1024)
|
jobs := make(chan string, workQueueDepth)
|
||||||
|
results := make(chan result, workQueueDepth)
|
||||||
|
|
||||||
for range workers {
|
for range workers {
|
||||||
go func() {
|
go func() {
|
||||||
for p := range jobs {
|
for p := range jobs {
|
||||||
@@ -114,7 +149,7 @@ func statPass(paths []string, workers int) (recs []fileRec, errs int) {
|
|||||||
case !fi.Mode().IsRegular():
|
case !fi.Mode().IsRegular():
|
||||||
results <- result{
|
results <- result{
|
||||||
rec: fileRec{path: p},
|
rec: fileRec{path: p},
|
||||||
err: fmt.Errorf("no longer a regular file"),
|
err: errNotRegular,
|
||||||
}
|
}
|
||||||
default:
|
default:
|
||||||
results <- result{rec: fileRec{
|
results <- result{rec: fileRec{
|
||||||
@@ -126,78 +161,114 @@ func statPass(paths []string, workers int) (recs []fileRec, errs int) {
|
|||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
}
|
}
|
||||||
|
|
||||||
go func() {
|
go func() {
|
||||||
for _, p := range paths {
|
for _, p := range paths {
|
||||||
jobs <- p
|
jobs <- p
|
||||||
}
|
}
|
||||||
|
|
||||||
close(jobs)
|
close(jobs)
|
||||||
}()
|
}()
|
||||||
|
|
||||||
prog := newProgress("stat", int64(len(paths)))
|
prog := newProgress("stat", int64(len(paths)))
|
||||||
recs = make([]fileRec, 0, len(paths))
|
|
||||||
|
var errs int
|
||||||
|
|
||||||
|
recs := make([]fileRec, 0, len(paths))
|
||||||
for range paths {
|
for range paths {
|
||||||
r := <-results
|
r := <-results
|
||||||
if r.err != nil {
|
if r.err != nil {
|
||||||
errs++
|
errs++
|
||||||
|
|
||||||
prog.warnf("stat %s: %v", r.rec.path, r.err)
|
prog.warnf("stat %s: %v", r.rec.path, r.err)
|
||||||
} else {
|
} else {
|
||||||
recs = append(recs, r.rec)
|
recs = append(recs, r.rec)
|
||||||
}
|
}
|
||||||
|
|
||||||
prog.increment()
|
prog.increment()
|
||||||
}
|
}
|
||||||
|
|
||||||
prog.finish()
|
prog.finish()
|
||||||
|
|
||||||
return recs, errs
|
return recs, errs
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// hashResult carries one file's head/tail hashes (or the error that
|
||||||
|
// prevented hashing it) from the hash workers to the main goroutine.
|
||||||
|
type hashResult struct {
|
||||||
|
rec fileRec
|
||||||
|
head string
|
||||||
|
tail string
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
// startHashWorkers starts the hash worker pool over recs and returns
|
||||||
|
// the channel its results arrive on (one per record, in completion
|
||||||
|
// order).
|
||||||
|
func startHashWorkers(recs []fileRec, workers int) <-chan hashResult {
|
||||||
|
jobs := make(chan fileRec, workQueueDepth)
|
||||||
|
results := make(chan hashResult, workQueueDepth)
|
||||||
|
|
||||||
|
for range workers {
|
||||||
|
go func() {
|
||||||
|
for rec := range jobs {
|
||||||
|
head, tail, err := hashHeadTail(rec.path, rec.size)
|
||||||
|
results <- hashResult{rec: rec, head: head, tail: tail, err: err}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
for _, rec := range recs {
|
||||||
|
jobs <- rec
|
||||||
|
}
|
||||||
|
|
||||||
|
close(jobs)
|
||||||
|
}()
|
||||||
|
|
||||||
|
return results
|
||||||
|
}
|
||||||
|
|
||||||
// hashPass hashes the first and last chunk bytes of every file in a
|
// hashPass hashes the first and last chunk bytes of every file in a
|
||||||
// worker pool and emits the output records on stdout from the main
|
// worker pool and emits the output records on stdout from the main
|
||||||
// goroutine. Files that fail to open or read are warned about and
|
// goroutine. Files that fail to open or read are warned about and
|
||||||
// dropped.
|
// dropped.
|
||||||
func hashPass(recs []fileRec, workers int) (emitted, errs int) {
|
func hashPass(recs []fileRec, workers int) (int, int) {
|
||||||
type result struct {
|
results := startHashWorkers(recs, workers)
|
||||||
rec fileRec
|
|
||||||
head string
|
|
||||||
tail string
|
|
||||||
err error
|
|
||||||
}
|
|
||||||
jobs := make(chan fileRec, 1024)
|
|
||||||
results := make(chan result, 1024)
|
|
||||||
for range workers {
|
|
||||||
go func() {
|
|
||||||
for rec := range jobs {
|
|
||||||
head, tail, err := hashHeadTail(rec.path, rec.size)
|
|
||||||
results <- result{rec: rec, head: head, tail: tail, err: err}
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
}
|
|
||||||
go func() {
|
|
||||||
for _, rec := range recs {
|
|
||||||
jobs <- rec
|
|
||||||
}
|
|
||||||
close(jobs)
|
|
||||||
}()
|
|
||||||
|
|
||||||
prog := newProgress("hash", int64(len(recs)))
|
prog := newProgress("hash", int64(len(recs)))
|
||||||
out := bufio.NewWriterSize(os.Stdout, 1<<20)
|
out := bufio.NewWriterSize(os.Stdout, ioBufSize)
|
||||||
|
|
||||||
|
var emitted, errs int
|
||||||
|
|
||||||
for range recs {
|
for range recs {
|
||||||
r := <-results
|
r := <-results
|
||||||
if r.err != nil {
|
if r.err != nil {
|
||||||
errs++
|
errs++
|
||||||
|
|
||||||
prog.warnf("hash %s: %v", r.rec.path, r.err)
|
prog.warnf("hash %s: %v", r.rec.path, r.err)
|
||||||
prog.increment()
|
prog.increment()
|
||||||
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if _, err := fmt.Fprintf(out, "%d\t%d\t%s\t%s\t%s\x00",
|
|
||||||
r.rec.size, r.rec.mtime, r.head, r.tail, r.rec.path); err != nil {
|
_, err := fmt.Fprintf(out, "%d\t%d\t%s\t%s\t%s\x00",
|
||||||
|
r.rec.size, r.rec.mtime, r.head, r.tail, r.rec.path)
|
||||||
|
if err != nil {
|
||||||
fatalf("write stdout: %v", err)
|
fatalf("write stdout: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
emitted++
|
emitted++
|
||||||
|
|
||||||
prog.increment()
|
prog.increment()
|
||||||
}
|
}
|
||||||
|
|
||||||
prog.finish()
|
prog.finish()
|
||||||
if err := out.Flush(); err != nil {
|
|
||||||
|
err := out.Flush()
|
||||||
|
if err != nil {
|
||||||
fatalf("write stdout: %v", err)
|
fatalf("write stdout: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return emitted, errs
|
return emitted, errs
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -206,26 +277,35 @@ func hashPass(recs []fileRec, workers int) (emitted, errs int) {
|
|||||||
// file at path. The two reads overlap when size < 2*chunk; for
|
// file at path. The two reads overlap when size < 2*chunk; for
|
||||||
// size == 0 both hashes are of the empty input. size is the value
|
// size == 0 both hashes are of the empty input. size is the value
|
||||||
// recorded by the stat pass.
|
// recorded by the stat pass.
|
||||||
func hashHeadTail(path string, size int64) (head, tail string, err error) {
|
func hashHeadTail(path string, size int64) (string, string, error) {
|
||||||
|
//nolint:gosec // hashing operator-supplied paths is the tool's purpose
|
||||||
f, err := os.Open(path)
|
f, err := os.Open(path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", "", err
|
return "", "", err
|
||||||
}
|
}
|
||||||
defer f.Close()
|
|
||||||
|
defer func() { _ = f.Close() }()
|
||||||
|
|
||||||
n := min(int64(chunk), size)
|
n := min(int64(chunk), size)
|
||||||
|
|
||||||
buf := make([]byte, n)
|
buf := make([]byte, n)
|
||||||
if n > 0 {
|
if n > 0 {
|
||||||
if _, err := f.ReadAt(buf, 0); err != nil {
|
_, err = f.ReadAt(buf, 0)
|
||||||
|
if err != nil {
|
||||||
return "", "", err
|
return "", "", err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
h := sha256.Sum256(buf)
|
h := sha256.Sum256(buf)
|
||||||
|
|
||||||
if n > 0 {
|
if n > 0 {
|
||||||
if _, err := f.ReadAt(buf, size-n); err != nil {
|
_, err = f.ReadAt(buf, size-n)
|
||||||
|
if err != nil {
|
||||||
return "", "", err
|
return "", "", err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
t := sha256.Sum256(buf)
|
t := sha256.Sum256(buf)
|
||||||
|
|
||||||
return hex.EncodeToString(h[:]), hex.EncodeToString(t[:]), nil
|
return hex.EncodeToString(h[:]), hex.EncodeToString(t[:]), nil
|
||||||
}
|
}
|
||||||
|
|||||||
113
trees.go
113
trees.go
@@ -38,16 +38,64 @@ type treeNode struct {
|
|||||||
func runTrees(args []string) {
|
func runTrees(args []string) {
|
||||||
in, name, closer := openScanInput(args)
|
in, name, closer := openScanInput(args)
|
||||||
defer closer()
|
defer closer()
|
||||||
|
|
||||||
recs, malformed := parseScanStream(in, name)
|
recs, malformed := parseScanStream(in, name)
|
||||||
records := len(recs) + malformed
|
records := len(recs) + malformed
|
||||||
|
|
||||||
// Build the hierarchy under a synthetic super-root. Paths are
|
super, allDirs := buildHierarchy(recs)
|
||||||
// split on "/"; for absolute paths the first component is empty,
|
super.compute()
|
||||||
// which simply becomes a top-level node representing "/".
|
|
||||||
|
dupes := collectTreeGroups(allDirs, super)
|
||||||
|
|
||||||
|
out := bufio.NewWriterSize(os.Stdout, ioBufSize)
|
||||||
|
|
||||||
|
_, err := fmt.Fprintln(out, "first\tdupe\tfiles\tsize")
|
||||||
|
if err != nil {
|
||||||
|
fatalf("write stdout: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
dupeTrees := 0
|
||||||
|
|
||||||
|
var reclaimable int64
|
||||||
|
|
||||||
|
for _, g := range dupes {
|
||||||
|
first := g[0]
|
||||||
|
for _, n := range g[1:] {
|
||||||
|
_, err = fmt.Fprintf(out, "%s\t%s\t%d\t%d\n",
|
||||||
|
first.path, n.path, first.fileCount, first.totalSize)
|
||||||
|
if err != nil {
|
||||||
|
fatalf("write stdout: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
dupeTrees++
|
||||||
|
reclaimable += first.totalSize
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
err = out.Flush()
|
||||||
|
if err != nil {
|
||||||
|
fatalf("write stdout: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Fprintf(os.Stderr,
|
||||||
|
"trees: %d records read%s, %d duplicate tree groups, %d dupe trees, %s reclaimable\n",
|
||||||
|
records, malformedNote(malformed), len(dupes), dupeTrees,
|
||||||
|
humanBytes(reclaimable))
|
||||||
|
}
|
||||||
|
|
||||||
|
// buildHierarchy reconstructs the directory hierarchy from the record
|
||||||
|
// paths under a synthetic super-root. Paths are split on "/"; for
|
||||||
|
// absolute paths the first component is empty, which simply becomes a
|
||||||
|
// top-level node representing "/". It returns the super-root and every
|
||||||
|
// directory node created.
|
||||||
|
func buildHierarchy(recs []scanRec) (*treeNode, []*treeNode) {
|
||||||
super := &treeNode{}
|
super := &treeNode{}
|
||||||
|
|
||||||
var allDirs []*treeNode
|
var allDirs []*treeNode
|
||||||
|
|
||||||
for _, r := range recs {
|
for _, r := range recs {
|
||||||
comps := strings.Split(r.path, "/")
|
comps := strings.Split(r.path, "/")
|
||||||
|
|
||||||
node := super
|
node := super
|
||||||
for _, c := range comps[:len(comps)-1] {
|
for _, c := range comps[:len(comps)-1] {
|
||||||
child := node.dirs[c]
|
child := node.dirs[c]
|
||||||
@@ -56,76 +104,68 @@ func runTrees(args []string) {
|
|||||||
if node != super {
|
if node != super {
|
||||||
childPath = node.path + "/" + c
|
childPath = node.path + "/" + c
|
||||||
}
|
}
|
||||||
|
|
||||||
child = &treeNode{path: childPath, parent: node}
|
child = &treeNode{path: childPath, parent: node}
|
||||||
if node.dirs == nil {
|
if node.dirs == nil {
|
||||||
node.dirs = make(map[string]*treeNode)
|
node.dirs = make(map[string]*treeNode)
|
||||||
}
|
}
|
||||||
|
|
||||||
node.dirs[c] = child
|
node.dirs[c] = child
|
||||||
allDirs = append(allDirs, child)
|
allDirs = append(allDirs, child)
|
||||||
}
|
}
|
||||||
|
|
||||||
node = child
|
node = child
|
||||||
}
|
}
|
||||||
|
|
||||||
if node.files == nil {
|
if node.files == nil {
|
||||||
node.files = make(map[string]fileSig)
|
node.files = make(map[string]fileSig)
|
||||||
}
|
}
|
||||||
|
|
||||||
node.files[comps[len(comps)-1]] = fileSig{
|
node.files[comps[len(comps)-1]] = fileSig{
|
||||||
size: r.size, head: r.head, tail: r.tail,
|
size: r.size, head: r.head, tail: r.tail,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
super.compute()
|
|
||||||
|
|
||||||
// Group directories by digest; two or more dirs sharing a digest
|
return super, allDirs
|
||||||
// are candidate duplicate trees.
|
}
|
||||||
|
|
||||||
|
// collectTreeGroups groups directories by digest and returns every
|
||||||
|
// maximal group with two or more members, each group's members sorted
|
||||||
|
// by path, groups ordered by tree size descending then by first path
|
||||||
|
// ascending.
|
||||||
|
func collectTreeGroups(allDirs []*treeNode, super *treeNode) [][]*treeNode {
|
||||||
groups := make(map[[sha256.Size]byte][]*treeNode)
|
groups := make(map[[sha256.Size]byte][]*treeNode)
|
||||||
for _, d := range allDirs {
|
for _, d := range allDirs {
|
||||||
groups[d.digest] = append(groups[d.digest], d)
|
groups[d.digest] = append(groups[d.digest], d)
|
||||||
}
|
}
|
||||||
|
|
||||||
var dupes [][]*treeNode
|
var dupes [][]*treeNode
|
||||||
|
|
||||||
for _, g := range groups {
|
for _, g := range groups {
|
||||||
if len(g) < 2 || suppressed(g, super) {
|
if len(g) < minGroupSize || suppressed(g, super) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
slices.SortFunc(g, func(a, b *treeNode) int {
|
slices.SortFunc(g, func(a, b *treeNode) int {
|
||||||
return strings.Compare(a.path, b.path)
|
return strings.Compare(a.path, b.path)
|
||||||
})
|
})
|
||||||
dupes = append(dupes, g)
|
dupes = append(dupes, g)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Biggest reclaimable space first; ties broken by first path.
|
// Biggest reclaimable space first; ties broken by first path.
|
||||||
slices.SortFunc(dupes, func(a, b []*treeNode) int {
|
slices.SortFunc(dupes, func(a, b []*treeNode) int {
|
||||||
if a[0].totalSize != b[0].totalSize {
|
if a[0].totalSize != b[0].totalSize {
|
||||||
if a[0].totalSize > b[0].totalSize {
|
if a[0].totalSize > b[0].totalSize {
|
||||||
return -1
|
return -1
|
||||||
}
|
}
|
||||||
|
|
||||||
return 1
|
return 1
|
||||||
}
|
}
|
||||||
|
|
||||||
return strings.Compare(a[0].path, b[0].path)
|
return strings.Compare(a[0].path, b[0].path)
|
||||||
})
|
})
|
||||||
|
|
||||||
out := bufio.NewWriterSize(os.Stdout, 1<<20)
|
return dupes
|
||||||
if _, err := fmt.Fprintln(out, "first\tdupe\tfiles\tsize"); err != nil {
|
|
||||||
fatalf("write stdout: %v", err)
|
|
||||||
}
|
|
||||||
dupeTrees := 0
|
|
||||||
var reclaimable int64
|
|
||||||
for _, g := range dupes {
|
|
||||||
first := g[0]
|
|
||||||
for _, n := range g[1:] {
|
|
||||||
if _, err := fmt.Fprintf(out, "%s\t%s\t%d\t%d\n",
|
|
||||||
first.path, n.path, first.fileCount, first.totalSize); err != nil {
|
|
||||||
fatalf("write stdout: %v", err)
|
|
||||||
}
|
|
||||||
dupeTrees++
|
|
||||||
reclaimable += first.totalSize
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err := out.Flush(); err != nil {
|
|
||||||
fatalf("write stdout: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
fmt.Fprintf(os.Stderr,
|
|
||||||
"trees: %d records read%s, %d duplicate tree groups, %d dupe trees, %s reclaimable\n",
|
|
||||||
records, malformedNote(malformed), len(dupes), dupeTrees,
|
|
||||||
humanBytes(reclaimable))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// compute fills in digest, fileCount, and totalSize for n and all of
|
// compute fills in digest, fileCount, and totalSize for n and all of
|
||||||
@@ -143,18 +183,22 @@ func (n *treeNode) compute() {
|
|||||||
n.fileCount++
|
n.fileCount++
|
||||||
n.totalSize += sig.size
|
n.totalSize += sig.size
|
||||||
}
|
}
|
||||||
|
|
||||||
for name, child := range n.dirs {
|
for name, child := range n.dirs {
|
||||||
child.compute()
|
child.compute()
|
||||||
entries = append(entries, "d\x00"+name+"\x00"+string(child.digest[:]))
|
entries = append(entries, "d\x00"+name+"\x00"+string(child.digest[:]))
|
||||||
n.fileCount += child.fileCount
|
n.fileCount += child.fileCount
|
||||||
n.totalSize += child.totalSize
|
n.totalSize += child.totalSize
|
||||||
}
|
}
|
||||||
|
|
||||||
slices.Sort(entries)
|
slices.Sort(entries)
|
||||||
|
|
||||||
h := sha256.New()
|
h := sha256.New()
|
||||||
for _, e := range entries {
|
for _, e := range entries {
|
||||||
h.Write([]byte(e))
|
h.Write([]byte(e))
|
||||||
h.Write([]byte{0})
|
h.Write([]byte{0})
|
||||||
}
|
}
|
||||||
|
|
||||||
copy(n.digest[:], h.Sum(nil))
|
copy(n.digest[:], h.Sum(nil))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -165,15 +209,19 @@ func (n *treeNode) compute() {
|
|||||||
// parent) or members whose parents differ are always reported.
|
// parent) or members whose parents differ are always reported.
|
||||||
func suppressed(g []*treeNode, super *treeNode) bool {
|
func suppressed(g []*treeNode, super *treeNode) bool {
|
||||||
seen := make(map[*treeNode]bool, len(g))
|
seen := make(map[*treeNode]bool, len(g))
|
||||||
|
|
||||||
var parentDigest [sha256.Size]byte
|
var parentDigest [sha256.Size]byte
|
||||||
|
|
||||||
for i, n := range g {
|
for i, n := range g {
|
||||||
p := n.parent
|
p := n.parent
|
||||||
if p == nil || p == super {
|
if p == nil || p == super {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
if seen[p] {
|
if seen[p] {
|
||||||
return false // siblings: not implied by any parent group
|
return false // siblings: not implied by any parent group
|
||||||
}
|
}
|
||||||
|
|
||||||
seen[p] = true
|
seen[p] = true
|
||||||
if i == 0 {
|
if i == 0 {
|
||||||
parentDigest = p.digest
|
parentDigest = p.digest
|
||||||
@@ -181,5 +229,6 @@ func suppressed(g []*treeNode, super *treeNode) bool {
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user