Stop counting directories and unchanged symlinks as files to back up (closes #305)

Every backup queued every directory and symlink for the second phase.
The line ending the walk counted directories, and their stat sizes, as
files to back up while its enumerated count left them out, and a
symlink was counted as backed up on every run, its target length as
data.

The walk now sets directories aside: they are recorded before the files
are backed up and never counted as files or data, and the walk's
progress estimate leaves out the directories the local index knows. A
symlink is checked against its record like a regular file, its target
included, so an unchanged one counts as unchanged.

Judgement call: every directory is still written to the local index on
every backup, as before; no report shows it.

Model: opus-5-5

internal/vaultik/snapshot_summary_test.go

@@ -13,6 +13,7 @@ import (
 	"github.com/spf13/afero"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"golang.org/x/sys/unix"
 	"sneak.berlin/go/vaultik/internal/config"
 	"sneak.berlin/go/vaultik/internal/database"
 	"sneak.berlin/go/vaultik/internal/log"
@@ -188,6 +189,42 @@ func (e *summaryEnv) dataLine(total, backedUp int64) string {
 		e.v.UI.Size(total), e.v.UI.Size(backedUp))
 }
 
+// enumeratedLine returns the line printed when the walk of a source path
+// ends.
+func (e *summaryEnv) enumeratedLine(
+	files int, total int64, toBackUp int, toBackUpSize int64,
+) string {
+	return fmt.Sprintf(
+		"Enumerated %d snapshot source files (%s total), %d to back up (%s).",
+		files, e.v.UI.Size(total), toBackUp, e.v.UI.Size(toBackUpSize))
+}
+
+// snapshotPaths returns the paths the snapshot lists, sorted.
+func (e *summaryEnv) snapshotPaths(t *testing.T, snapshotID string) []string {
+	t.Helper()
+
+	rows, err := e.db.Conn().QueryContext(context.Background(), `
+		SELECT f.path FROM snapshot_files sf JOIN files f ON f.id = sf.file_id
+		WHERE sf.snapshot_id = ? ORDER BY f.path`, snapshotID)
+	require.NoError(t, err)
+
+	defer func() { _ = rows.Close() }()
+
+	var paths []string
+
+	for rows.Next() {
+		var path string
+
+		require.NoError(t, rows.Scan(&path))
+
+		paths = append(paths, path)
+	}
+
+	require.NoError(t, rows.Err())
+
+	return paths
+}
+
 // A first backup stores copy.bin's chunks while backing up a.bin, so
 // copy.bin's chunks are deduplicated within the run. Each file and byte
 // is still counted once.
@@ -248,6 +285,85 @@ func TestSnapshotSummaryIncrementalRunWithDeduplicatedChunks(t *testing.T) {
 	assert.Equal(t, uncompressed, snap.BlobUncompressedSize)
 }
 
+// A backup of a tree with directories and a symlink, none of which
+// changed since the last backup, backs up nothing, and the snapshot
+// still lists every one of them. Both sides of the line that ends the
+// walk count the same entries, the files and the symlink, and no
+// directory's size. See https://git.eeqj.de/sneak/vaultik/issues/305.
+func TestSnapshotSummaryUnchangedRunWithDirectoriesAndSymlink(t *testing.T) {
+	log.Initialize(log.Config{})
+	t.Parallel()
+
+	env := newSummaryEnv(t)
+
+	const linkTarget = "a.bin"
+
+	require.NoError(t, os.Symlink(linkTarget,
+		filepath.Join(filepath.Dir(env.aPath), "link")))
+
+	// A symlink's size is the length of its target.
+	total := env.totalSize + int64(len(linkTarget))
+
+	first := env.backUp(t, "first", false)
+	assert.Contains(t, first, env.enumeratedLine(4, total, 4, total))
+	assert.Contains(t, first, filesLine(4, 4, 0))
+
+	second := env.backUp(t, "second", false)
+	assert.Contains(t, second, env.enumeratedLine(4, total, 0, 0))
+	assert.Contains(t, second, filesLine(4, 0, 4))
+	assert.Contains(t, second, env.dataLine(total, 0))
+
+	assert.Equal(t,
+		env.snapshotPaths(t, env.snapshot(t, "first").ID.String()),
+		env.snapshotPaths(t, env.snapshot(t, "second").ID.String()))
+}
+
+// A directory whose mtime changed and a symlink whose target changed are
+// recorded again by the next backup, which counts only the symlink as
+// backed up.
+func TestSnapshotSummaryChangedDirectoryAndSymlink(t *testing.T) {
+	log.Initialize(log.Config{})
+	t.Parallel()
+
+	env := newSummaryEnv(t)
+
+	// The local index keeps paths with symlinks resolved.
+	dir, err := filepath.EvalSymlinks(filepath.Dir(env.aPath))
+	require.NoError(t, err)
+
+	// The new link gets the old one's mtime, and a target, which need not
+	// exist, of the old target's length, a symlink's size: only the
+	// target differs.
+	linkTime := unix.NsecToTimeval(
+		time.Date(2029, time.March, 4, 5, 6, 7, 0, time.UTC).UnixNano())
+	link := filepath.Join(dir, "link")
+	require.NoError(t, os.Symlink("a.bin", link))
+	require.NoError(t, unix.Lutimes(link, []unix.Timeval{linkTime, linkTime}))
+
+	env.backUp(t, "first", false)
+
+	require.NoError(t, os.Remove(link))
+	require.NoError(t, os.Symlink("b.bin", link))
+	require.NoError(t, unix.Lutimes(link, []unix.Timeval{linkTime, linkTime}))
+
+	mtime := time.Date(2030, time.January, 2, 3, 4, 5, 0, time.UTC)
+	require.NoError(t, os.Chtimes(dir, mtime, mtime))
+
+	assert.Contains(t, env.backUp(t, "second", false), filesLine(4, 1, 3))
+
+	ctx := context.Background()
+
+	dirRow, err := env.repos.Files.GetByPath(ctx, dir)
+	require.NoError(t, err)
+	require.NotNil(t, dirRow)
+	assert.Equal(t, mtime.Unix(), dirRow.MTime.Unix())
+
+	linkRow, err := env.repos.Files.GetByPath(ctx, link)
+	require.NoError(t, err)
+	require.NotNil(t, linkRow)
+	assert.Equal(t, "b.bin", linkRow.LinkTarget.String())
+}
+
 // Under --cron the progress reporter is off; the upload figures must
 // still reach the summary and the snapshots row. The snapshot has two
 // paths, each backed up by its own scan.