Re-chunk a known file whose chunks no uploaded blob holds (closes #214)

File rows are shared by every snapshot and updated in place, while a
blob row is deleted once no snapshot references it. Removing the newest
snapshot, or the prune after an interrupted run, could drop the only
blob holding a changed file's current chunks while an older snapshot
kept the file row. The next backup compared metadata only, skipped the
file, and completed a snapshot that could not restore it.

The scanner now loads the IDs of known files that list a chunk no
uploaded blob holds and re-chunks them even when their metadata is
unchanged.

The tests give each backup run its own snapshot name, so the
second-precision snapshot IDs differ without sleeping.

Model: opus-5-5

internal/vaultik/changed_file_restore_test.go

@@ -2,15 +2,12 @@ package vaultik_test
 
 import (
 	"context"
-	"crypto/rand"
 	"path/filepath"
-	"slices"
 	"strings"
 	"testing"
 
 	"github.com/spf13/afero"
 	"github.com/stretchr/testify/require"
-	"sneak.berlin/go/vaultik/internal/chunker"
 	"sneak.berlin/go/vaultik/internal/config"
 	"sneak.berlin/go/vaultik/internal/database"
 	"sneak.berlin/go/vaultik/internal/log"
@@ -54,26 +51,14 @@ func backUp(v *vaultik.Vaultik, name string) error {
 	})
 }
 
-// appendedFileSize is the size of the file before the tests append to it.
+// changeFile rewrites path with content of a different size, so the next
-// It is twice the largest chunk the chunker cuts, so the file's first
+// backup sees the file as changed, and records the new content in files.
-// chunk ends before the appended bytes and stays in a blob the first
+func changeFile(
-// snapshot still references, while its new chunks are only in the blob
+	t *testing.T, fs afero.Fs, files map[string][]byte, path string,
-// that gets dropped.
-const appendedFileSize = 2 * chunker.ChunkSizeSpread * faultChunkSize
-
-// appendRandomBytes appends n random bytes to the file at path, creating
-// the file if it does not exist, and records the new content in files.
-// Random content gives the chunker real cut points.
-func appendRandomBytes(
-	t *testing.T, fs afero.Fs, files map[string][]byte, path string, n int64,
 ) {
 	t.Helper()
 
-	added := make([]byte, n)
+	content := bytesPattern("changed-", int(faultChunkSize*4))
-	_, err := rand.Read(added)
-	require.NoError(t, err)
-
-	content := slices.Concat(files[path], added)
 	require.NoError(t, afero.WriteFile(fs, path, content, 0o644))
 
 	files[path] = content
@@ -126,10 +111,10 @@ func assertThirdSnapshotRestores(
 	assertRestoredTree(t, fs, restoreDir, files)
 }
 
-// Trigger 1: bytes are appended to the file, a second snapshot backs it
+// Trigger 1: the file changes, a second snapshot backs it up, and that
-// up, and that snapshot is removed. The first snapshot keeps the file row,
+// snapshot is removed. The first snapshot keeps the file row, which now
-// which now lists the appended content's chunks, while removal drops the
+// lists the changed content's chunks, while removal drops the blob that
-// blob that held them.
+// held them.
 //
 //nolint:paralleltest // installs the global logger via log.Initialize
 func TestBackupAfterRemovingNewestSnapshotRestoresChangedFile(t *testing.T) {
@@ -141,14 +126,11 @@ func TestBackupAfterRemovingNewestSnapshotRestoresChangedFile(t *testing.T) {
 	storeDir := filepath.Join(tempDir, "remote")
 	restoreDir := filepath.Join(tempDir, "restored")
 	dbPath := filepath.Join(tempDir, "index.sqlite")
-	changedPath := filepath.Join(dataDir, "appended.bin")
 
 	ctx := context.Background()
 	files := writeFaultSourceTree(t, fs, dataDir)
 	cfg := changedFileConfig(dataDir, dbPath)
 
-	appendRandomBytes(t, fs, files, changedPath, appendedFileSize)
-
 	store, err := storage.NewFileStorer(storeDir)
 	require.NoError(t, err)
 
@@ -160,7 +142,7 @@ func TestBackupAfterRemovingNewestSnapshotRestoresChangedFile(t *testing.T) {
 
 	require.NoError(t, backUp(v, "first"))
 
-	appendRandomBytes(t, fs, files, changedPath, faultChunkSize)
+	changeFile(t, fs, files, filepath.Join(dataDir, "a.bin"))
 	require.NoError(t, backUp(v, "second"))
 
 	_, err = v.RemoveSnapshot(localSnapshotID(ctx, t, repos, "second"),
@@ -173,11 +155,11 @@ func TestBackupAfterRemovingNewestSnapshotRestoresChangedFile(t *testing.T) {
 		ctx, t, cfg, store, repos, db, fs, restoreDir, files)
 }
 
-// Trigger 2: bytes are appended to the file and the run that backs it up
+// Trigger 2: the file changes and the run that backs it up is interrupted
-// is interrupted at the manifest upload, after its blobs were uploaded.
+// at the manifest upload, after its blobs were uploaded. The next run's
-// The next run's prune drops that incomplete snapshot and its blob, while
+// prune drops that incomplete snapshot and its blob, while the first
-// the first snapshot keeps the file row, which now lists the appended
+// snapshot keeps the file row, which now lists the changed content's
-// content's chunks.
+// chunks.
 //
 //nolint:paralleltest // installs the global logger via log.Initialize
 func TestBackupAfterInterruptedRunRestoresChangedFile(t *testing.T) {
@@ -189,14 +171,11 @@ func TestBackupAfterInterruptedRunRestoresChangedFile(t *testing.T) {
 	storeDir := filepath.Join(tempDir, "remote")
 	restoreDir := filepath.Join(tempDir, "restored")
 	dbPath := filepath.Join(tempDir, "index.sqlite")
-	changedPath := filepath.Join(dataDir, "appended.bin")
 
 	ctx := context.Background()
 	files := writeFaultSourceTree(t, fs, dataDir)
 	cfg := changedFileConfig(dataDir, dbPath)
 
-	appendRandomBytes(t, fs, files, changedPath, appendedFileSize)
-
 	inner, err := storage.NewFileStorer(storeDir)
 	require.NoError(t, err)
 
@@ -218,7 +197,7 @@ func TestBackupAfterInterruptedRunRestoresChangedFile(t *testing.T) {
 
 	require.NoError(t, backUp(v, "first"))
 
-	appendRandomBytes(t, fs, files, changedPath, faultChunkSize)
+	changeFile(t, fs, files, filepath.Join(dataDir, "a.bin"))
 
 	failManifest = true