Re-chunk a known file whose chunks no uploaded blob holds (closes #214)

File rows are shared by every snapshot and updated in place, while a
blob row is deleted once no snapshot references it. Removing the newest
snapshot, or the prune after an interrupted run, could drop the only
blob holding a changed file's current chunks while an older snapshot
kept the file row. The next backup compared metadata only, skipped the
file, and completed a snapshot that could not restore it.

The scanner now loads the IDs of known files that list a chunk no
uploaded blob holds and re-chunks them even when their metadata is
unchanged.

The tests append to a file, so the file keeps its first chunk in a blob
the first snapshot still references. Each backup run gets its own
snapshot name, so the second-precision snapshot IDs differ without
sleeping.

Model: opus-5-5

internal/vaultik/changed_file_restore_test.go

@@ -2,12 +2,15 @@ package vaultik_test
 
 import (
 	"context"
+	"crypto/rand"
 	"path/filepath"
+	"slices"
 	"strings"
 	"testing"
 
 	"github.com/spf13/afero"
 	"github.com/stretchr/testify/require"
+	"sneak.berlin/go/vaultik/internal/chunker"
 	"sneak.berlin/go/vaultik/internal/config"
 	"sneak.berlin/go/vaultik/internal/database"
 	"sneak.berlin/go/vaultik/internal/log"
@@ -51,14 +54,26 @@ func backUp(v *vaultik.Vaultik, name string) error {
 	})
 }
 
-// changeFile rewrites path with content of a different size, so the next
+// appendedFileSize is the size of the file before the tests append to it.
-// backup sees the file as changed, and records the new content in files.
+// It is twice the largest chunk the chunker cuts, so the file's first
-func changeFile(
+// chunk ends before the appended bytes and stays in a blob the first
-	t *testing.T, fs afero.Fs, files map[string][]byte, path string,
+// snapshot still references, while its new chunks are only in the blob
+// that gets dropped.
+const appendedFileSize = 2 * chunker.ChunkSizeSpread * faultChunkSize
+
+// appendRandomBytes appends n random bytes to the file at path, creating
+// the file if it does not exist, and records the new content in files.
+// Random content gives the chunker real cut points.
+func appendRandomBytes(
+	t *testing.T, fs afero.Fs, files map[string][]byte, path string, n int64,
 ) {
 	t.Helper()
 
-	content := bytesPattern("changed-", int(faultChunkSize*4))
+	added := make([]byte, n)
+	_, err := rand.Read(added)
+	require.NoError(t, err)
+
+	content := slices.Concat(files[path], added)
 	require.NoError(t, afero.WriteFile(fs, path, content, 0o644))
 
 	files[path] = content
@@ -111,10 +126,10 @@ func assertThirdSnapshotRestores(
 	assertRestoredTree(t, fs, restoreDir, files)
 }
 
-// Trigger 1: the file changes, a second snapshot backs it up, and that
+// Trigger 1: bytes are appended to the file, a second snapshot backs it
-// snapshot is removed. The first snapshot keeps the file row, which now
+// up, and that snapshot is removed. The first snapshot keeps the file row,
-// lists the changed content's chunks, while removal drops the blob that
+// which now lists the appended content's chunks, while removal drops the
-// held them.
+// blob that held them.
 //
 //nolint:paralleltest // installs the global logger via log.Initialize
 func TestBackupAfterRemovingNewestSnapshotRestoresChangedFile(t *testing.T) {
@@ -126,11 +141,14 @@ func TestBackupAfterRemovingNewestSnapshotRestoresChangedFile(t *testing.T) {
 	storeDir := filepath.Join(tempDir, "remote")
 	restoreDir := filepath.Join(tempDir, "restored")
 	dbPath := filepath.Join(tempDir, "index.sqlite")
+	changedPath := filepath.Join(dataDir, "appended.bin")
 
 	ctx := context.Background()
 	files := writeFaultSourceTree(t, fs, dataDir)
 	cfg := changedFileConfig(dataDir, dbPath)
 
+	appendRandomBytes(t, fs, files, changedPath, appendedFileSize)
+
 	store, err := storage.NewFileStorer(storeDir)
 	require.NoError(t, err)
 
@@ -142,7 +160,7 @@ func TestBackupAfterRemovingNewestSnapshotRestoresChangedFile(t *testing.T) {
 
 	require.NoError(t, backUp(v, "first"))
 
-	changeFile(t, fs, files, filepath.Join(dataDir, "a.bin"))
+	appendRandomBytes(t, fs, files, changedPath, faultChunkSize)
 	require.NoError(t, backUp(v, "second"))
 
 	_, err = v.RemoveSnapshot(localSnapshotID(ctx, t, repos, "second"),
@@ -155,11 +173,11 @@ func TestBackupAfterRemovingNewestSnapshotRestoresChangedFile(t *testing.T) {
 		ctx, t, cfg, store, repos, db, fs, restoreDir, files)
 }
 
-// Trigger 2: the file changes and the run that backs it up is interrupted
+// Trigger 2: bytes are appended to the file and the run that backs it up
-// at the manifest upload, after its blobs were uploaded. The next run's
+// is interrupted at the manifest upload, after its blobs were uploaded.
-// prune drops that incomplete snapshot and its blob, while the first
+// The next run's prune drops that incomplete snapshot and its blob, while
-// snapshot keeps the file row, which now lists the changed content's
+// the first snapshot keeps the file row, which now lists the appended
-// chunks.
+// content's chunks.
 //
 //nolint:paralleltest // installs the global logger via log.Initialize
 func TestBackupAfterInterruptedRunRestoresChangedFile(t *testing.T) {
@@ -171,11 +189,14 @@ func TestBackupAfterInterruptedRunRestoresChangedFile(t *testing.T) {
 	storeDir := filepath.Join(tempDir, "remote")
 	restoreDir := filepath.Join(tempDir, "restored")
 	dbPath := filepath.Join(tempDir, "index.sqlite")
+	changedPath := filepath.Join(dataDir, "appended.bin")
 
 	ctx := context.Background()
 	files := writeFaultSourceTree(t, fs, dataDir)
 	cfg := changedFileConfig(dataDir, dbPath)
 
+	appendRandomBytes(t, fs, files, changedPath, appendedFileSize)
+
 	inner, err := storage.NewFileStorer(storeDir)
 	require.NoError(t, err)
 
@@ -197,7 +218,7 @@ func TestBackupAfterInterruptedRunRestoresChangedFile(t *testing.T) {
 
 	require.NoError(t, backUp(v, "first"))
 
-	changeFile(t, fs, files, filepath.Join(dataDir, "a.bin"))
+	appendRandomBytes(t, fs, files, changedPath, faultChunkSize)
 
 	failManifest = true