Remove the blob temp file when a backup stops part-way (closes #302)

When `snapshot create` stopped on a file error, a full temporary
directory or Ctrl-C, the blob being packed was never finalized, and its
`vaultik-blob-*.tmp` file (up to `blob_size_limit` of encrypted data)
stayed in `$TMPDIR`. Phase 2 now calls the new
`Packer.DiscardCurrentBlob` on the way out, which closes and removes
that file. After a successful flush no blob is in progress, so the call
then does nothing.

The blob's row in the local index is left, as before, for the next
run's repair of interrupted blobs to remove.

Model: opus-5-5

README.md

@@ -412,7 +412,8 @@ destination store.
 entirely. Remote storage is unaffected; the next backup will do a full
 scan and re-deduplicate against existing remote blobs, and the local
 index will re-bind to the currently configured storage destination.
-Use this after changing `storage_url` to a different destination.
+Use this after changing `storage_url`, or `s3.endpoint`, `s3.bucket` or
+`s3.prefix`, to a different destination.
 * `--force`: Skip confirmation prompt
 
 ---

TODO.md

@@ -22,15 +22,23 @@ the tag exists and is exercised; what is left is merging `next` to
 
 # Completed Steps
 
-- 2026-10-12: Made a `file://` destination list through a symlinked
-  directory ([issue #296](https://git.eeqj.de/sneak/vaultik/issues/296)).
-  A `metadata/` or `blobs/` directory that was a symlink to a directory
-  elsewhere was listed as one file, so the destination store listed as
-  empty while backups and restores still worked through the link.
-  `prune` then removed the local snapshot records and deleted every blob,
-  and `remote info` and `remote nuke` found no blobs. The listings now
-  read through the link as `Get` and `Stat` do; a link pointing back up
-  the tree makes them fail.
+- 2026-10-13: Made a failed or interrupted backup remove the temporary
+  file of the blob it was packing
+  ([issue #302](https://git.eeqj.de/sneak/vaultik/issues/302)). When
+  `snapshot create` stopped on a file error, a full temporary directory
+  or Ctrl-C, that blob was never finalized and its `vaultik-blob-*.tmp`
+  file, up to `blob_size_limit` of encrypted data, stayed in `$TMPDIR`;
+  every such run left another. Phase 2 now removes it whenever it stops
+  before the last blob is finalized.
+
+- 2026-10-12: Bound the local index to the destination an `s3.*` config
+  names ([issue #298](https://git.eeqj.de/sneak/vaultik/issues/298)).
+  Only `storage_url` was checked, so after `s3.bucket` or `s3.prefix`
+  changed, the next backup treated every chunk as already stored,
+  uploaded nothing, and completed a snapshot that could not be restored.
+  Without `storage_url`, the local index is now bound to `s3.endpoint`,
+  `s3.bucket` and `s3.prefix`, and a change is refused as a changed
+  `storage_url` is.
 
 - 2026-10-12: Rejected a retention duration longer than about 292 years
   ([issue #289](https://git.eeqj.de/sneak/vaultik/issues/289)). The

internal/blob/packer.go

@@ -245,6 +245,17 @@ func (p *Packer) Flush(ctx context.Context) error {
 	return nil
 }
 
+// DiscardCurrentBlob closes and removes the temp file of the blob being
+// packed, if any, and drops that blob. A caller that stops before Flush
+// must call it, or the temp file is left behind. Thread-safe.
+func (p *Packer) DiscardCurrentBlob() {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+
+	p.cleanupTempFile()
+	p.currentBlob = nil
+}
+
 // FinalizeBlob finalizes the current blob being assembled.
 // This compresses the accumulated chunks, encrypts the result, and computes
 // the content-addressed hash. The finalized blob is either passed to the

internal/snapshot/scanner.go

@@ -1295,6 +1295,11 @@ func (s *Scanner) batchAddFilesToSnapshot(
 func (s *Scanner) processPhase(
 	ctx context.Context, filesToProcess []*FileToProcess, result *ScanResult,
 ) error {
+	// On an error or cancellation the blob being packed is never
+	// finalized, so its temp file must be removed here. After a
+	// successful flush there is no such blob and this does nothing.
+	defer s.packer.DiscardCurrentBlob()
+
 	// Calculate total bytes to process
 	var totalBytes int64
 	for _, f := range filesToProcess {

internal/snapshot/skip_errors_test.go

@@ -381,3 +381,60 @@ func TestScannerCancelStopsSkipErrorsRun(t *testing.T) {
 		})
 	}
 }
+
+// TestScannerRemovesBlobTempFileWhenRunStops checks that a run stopped
+// part-way, by a file it cannot read or by a cancellation, removes the
+// temp file of the blob it was packing.
+func TestScannerRemovesBlobTempFileWhenRunStops(t *testing.T) {
+	t.Parallel()
+
+	// a.txt is backed up first and starts a blob; the run stops at b.txt.
+	const target = "/source/b.txt"
+
+	tests := []struct {
+		name  string
+		newFs func(cancel context.CancelFunc) afero.Fs
+	}{
+		{
+			name: "file cannot be read",
+			newFs: func(context.CancelFunc) afero.Fs {
+				return &readFailFs{Fs: afero.NewMemMapFs(), target: target}
+			},
+		},
+		{
+			name: "cancelled",
+			newFs: func(cancel context.CancelFunc) afero.Fs {
+				return &cancelOnOpenFs{
+					Fs: afero.NewMemMapFs(), target: target, cancel: cancel,
+				}
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			ctx, cancel := context.WithCancel(context.Background())
+			defer cancel()
+
+			fs := tt.newFs(cancel)
+			writeSkipErrorTestFile(t, fs, "/source/a.txt", "first file content")
+			writeSkipErrorTestFile(t, fs, target, "second file content")
+
+			_, err := runSkipErrorScan(ctx, t, fs, "/source", false, nil)
+			if err == nil {
+				t.Fatal("expected the scan to stop with an error, got nil")
+			}
+
+			left, err := afero.Glob(fs, filepath.Join(os.TempDir(), "vaultik-blob-*"))
+			if err != nil {
+				t.Fatalf("listing blob temp files: %v", err)
+			}
+
+			if len(left) != 0 {
+				t.Fatalf("expected no blob temp file left, got %v", left)
+			}
+		})
+	}
+}

internal/storage/file.go

@@ -120,9 +120,9 @@ func (f *FileStorer) Delete(_ context.Context, key string) error {
 	return nil
 }
 
-// List returns all keys with the given prefix, reading through symlinked
-// directories. It fails when the destination directory is missing; a
-// missing prefix under it is an empty listing.
+// List returns all keys with the given prefix. It fails when the
+// destination directory is missing; a missing prefix under it is an
+// empty listing.
 func (f *FileStorer) List(ctx context.Context, prefix string) ([]string, error) {
 	var keys []string
 
@@ -143,7 +143,7 @@ func (f *FileStorer) List(ctx context.Context, prefix string) ([]string, error)
 		return keys, nil // Empty list for non-existent prefix
 	}
 
-	err = f.walk(basePath, func(path string, info os.FileInfo, err error) error {
+	err = afero.Walk(f.fs, basePath, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
@@ -176,8 +176,7 @@ func (f *FileStorer) List(ctx context.Context, prefix string) ([]string, error)
 }
 
 // ListStream returns a channel of ObjectInfo for large result sets. Like
-// List, it reads through symlinked directories and sends an error when
-// the destination directory is missing.
+// List, it sends an error when the destination directory is missing.
 func (f *FileStorer) ListStream(ctx context.Context, prefix string) <-chan ObjectInfo {
 	ch := make(chan ObjectInfo)
 
@@ -205,7 +204,7 @@ func (f *FileStorer) ListStream(ctx context.Context, prefix string) <-chan Objec
 			return // Empty channel for non-existent prefix
 		}
 
-		_ = f.walk(basePath, func(path string, info os.FileInfo, err error) error {
+		_ = afero.Walk(f.fs, basePath, func(path string, info os.FileInfo, err error) error {
 			// Check context cancellation
 			select {
 			case <-ctx.Done():
@@ -353,17 +352,6 @@ func (f *FileStorer) writeAtomic(
 	return nil
 }
 
-// walk is afero.Walk, except that it reads through a symlinked directory
-// the way Get and Stat do, where afero.Walk reports the link as one file.
-// afero.Walk uses Lstat, which does not follow a link, whenever the
-// filesystem has one; the struct below carries only the afero.Fs methods,
-// so the walk uses Stat instead. A link back up the tree is followed until
-// the operating system refuses the path for too many levels of links, and
-// the walk then fails.
-func (f *FileStorer) walk(root string, fn filepath.WalkFunc) error {
-	return afero.Walk(struct{ afero.Fs }{f.fs}, root, fn)
-}
-
 // fullPath returns the full filesystem path for a key.
 func (f *FileStorer) fullPath(key string) string {
 	return filepath.Join(f.basePath, key)

internal/storage/file_backend_test.go

@@ -4,10 +4,7 @@ import (
 	"context"
 	"errors"
 	"io/fs"
-	"os"
 	"path/filepath"
-	"reflect"
-	"strings"
 	"testing"
 
 	"sneak.berlin/go/vaultik/internal/storage"
@@ -63,100 +60,3 @@ func TestFileStorerListMissingDestination(t *testing.T) {
 		t.Errorf("ListStream error = %v, want a not-exist error", streamErr)
 	}
 }
-
-// TestFileStorerListsThroughSymlinkedDirectory checks that List and
-// ListStream read through a symlinked metadata/ or blobs/ directory, as Get
-// and Stat do, instead of listing the link as one key. Listing the whole
-// destination meets the link part-way through the walk; listing its prefix
-// starts the walk at the link.
-func TestFileStorerListsThroughSymlinkedDirectory(t *testing.T) {
-	t.Parallel()
-
-	for _, dir := range []string{"metadata", "blobs"} {
-		t.Run(dir, func(t *testing.T) {
-			t.Parallel()
-
-			base := t.TempDir()
-			key := dir + "/aa/bb/object"
-
-			s, err := storage.NewFileStorer(base)
-			if err != nil {
-				t.Fatalf("NewFileStorer: %v", err)
-			}
-
-			err = s.Put(context.Background(), key, strings.NewReader("data"))
-			if err != nil {
-				t.Fatalf("Put: %v", err)
-			}
-
-			linkPath := filepath.Join(base, dir)
-			movedPath := filepath.Join(t.TempDir(), dir)
-
-			err = os.Rename(linkPath, movedPath)
-			if err != nil {
-				t.Fatalf("moving %s away: %v", dir, err)
-			}
-
-			err = os.Symlink(movedPath, linkPath)
-			if err != nil {
-				t.Fatalf("linking %s: %v", dir, err)
-			}
-
-			want := []string{key}
-
-			for _, prefix := range []string{"", dir + "/"} {
-				got := listSorted(t, s, prefix)
-				if !reflect.DeepEqual(got, want) {
-					t.Errorf("List(%q) = %v, want %v", prefix, got, want)
-				}
-
-				got = listStreamKeys(t, s, prefix)
-				if !reflect.DeepEqual(got, want) {
-					t.Errorf("ListStream(%q) = %v, want %v", prefix, got, want)
-				}
-			}
-		})
-	}
-}
-
-// TestFileStorerListFailsOnSymlinkLoop checks that a symlink pointing back
-// at the destination makes List and ListStream fail instead of walking it
-// forever.
-func TestFileStorerListFailsOnSymlinkLoop(t *testing.T) {
-	t.Parallel()
-
-	ctx := context.Background()
-	base := t.TempDir()
-
-	s, err := storage.NewFileStorer(base)
-	if err != nil {
-		t.Fatalf("NewFileStorer: %v", err)
-	}
-
-	err = s.Put(ctx, testBlobKey, strings.NewReader("data"))
-	if err != nil {
-		t.Fatalf("Put: %v", err)
-	}
-
-	err = os.Symlink(base, filepath.Join(base, "blobs", "loop"))
-	if err != nil {
-		t.Fatalf("Symlink: %v", err)
-	}
-
-	keys, err := s.List(ctx, "blobs/")
-	if err == nil {
-		t.Errorf("List = %d keys and no error, want an error", len(keys))
-	}
-
-	var streamErr error
-
-	for object := range s.ListStream(ctx, "blobs/") {
-		if object.Err != nil {
-			streamErr = object.Err
-		}
-	}
-
-	if streamErr == nil {
-		t.Error("ListStream sent no error, want one")
-	}
-}

internal/vaultik/storage_bind.go

@@ -9,7 +9,7 @@ import (
 )
 
 // errStorageBindingMismatch is returned when the local index database is
-// bound to a different destination than the configured storage_url.
+// bound to a different destination than the configured one.
 var errStorageBindingMismatch = errors.New(
 	"local index is bound to a different backup destination")
 
@@ -24,8 +24,9 @@ var errStorageBindingMismatch = errors.New(
 //
 // Behaviour:
 //   - On first use (empty stored value), record the configured
-//     storage_url and log the binding.
-//   - When the stored value matches the configured storage_url, do
+//     destination (storage_url, or the s3.* endpoint, bucket and
+//     prefix) and log the binding.
+//   - When the stored value matches the configured destination, do
 //     nothing and return nil.
 //   - When the two differ, refuse with an error that tells the user
 //     how to recover (revert the config, or run `vaultik database
@@ -46,12 +47,10 @@ func (v *Vaultik) EnsureStorageBinding() error {
 
 	configured := v.Config.StorageURL
 	if configured == "" {
-		// Some legacy configs still use the split s3.* keys instead of
-		// storage_url. Falling back to a synthetic URL for those would
-		// hide the fact that the binding is loose. Instead, treat
-		// unset as "nothing to bind against" — the check is
-		// necessarily best-effort for those configs.
-		return nil
+		// The s3.* form: bind on its endpoint, bucket and prefix,
+		// written the way storage_url would name the same destination.
+		s3 := v.Config.S3
+		configured = "s3://" + s3.Bucket + "/" + s3.Prefix + "?endpoint=" + s3.Endpoint
 	}
 
 	stored, err := v.Repositories.LocalMeta.Get(v.ctx, database.LocalMetaKeyStorageURL)
@@ -96,7 +95,8 @@ func buildBindingMismatchMessage(stored, configured string) string {
 		"future snapshots referencing blobs that don't exist at the new destination.\n" +
 		"\n" +
 		"To proceed, either:\n" +
-		"  - revert storage_url in your config to the bound destination, or\n" +
+		"  - revert storage_url (or the s3.* endpoint, bucket and prefix) in your\n" +
+		"    config to the bound destination, or\n" +
 		"  - run 'vaultik database delete' to discard the local index and rebuild it\n" +
 		"    from a fresh full backup against the new destination"
 }

internal/vaultik/storage_bind_test.go

@@ -76,3 +76,51 @@ func TestEnsureStorageBinding_MismatchRefuses(t *testing.T) {
 	assert.Contains(t, err.Error(), "file:///mnt/backups/new")
 	assert.Contains(t, err.Error(), "vaultik database delete")
 }
+
+// A config with no storage_url names its destination with s3.endpoint,
+// s3.bucket and s3.prefix; a change to any of them must be refused just
+// as a changed storage_url is.
+func TestEnsureStorageBinding_S3FieldsChangeRefuses(t *testing.T) {
+	log.Initialize(log.Config{})
+	t.Parallel()
+
+	bound := config.S3Config{
+		Endpoint: "https://s3.example.com",
+		Bucket:   "bucket-a",
+		Prefix:   "backups",
+	}
+
+	tests := []struct {
+		name    string
+		changed config.S3Config
+	}{
+		{"bucket", config.S3Config{
+			Endpoint: bound.Endpoint, Bucket: "bucket-b", Prefix: bound.Prefix,
+		}},
+		{"prefix", config.S3Config{
+			Endpoint: bound.Endpoint, Bucket: bound.Bucket, Prefix: "other",
+		}},
+		{"endpoint", config.S3Config{
+			Endpoint: "https://s3.other.com", Bucket: bound.Bucket, Prefix: bound.Prefix,
+		}},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			tv, _ := buildBindTestVaultik(t, "")
+
+			tv.Config.S3 = bound
+			require.NoError(t, tv.EnsureStorageBinding(), "first use binds")
+			require.NoError(t, tv.EnsureStorageBinding(), "same destination passes")
+
+			tv.Config.S3 = tt.changed
+			err := tv.EnsureStorageBinding()
+			require.Error(t, err)
+			assert.Contains(t, err.Error(),
+				"local index is bound to a different backup destination")
+			assert.Contains(t, err.Error(), "vaultik database delete")
+		})
+	}
+}

internal/vaultik/symlinked_destination_test.go

@@ -1,63 +0,0 @@
-package vaultik_test
-
-import (
-	"context"
-	"io/fs"
-	"os"
-	"path/filepath"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"sneak.berlin/go/vaultik/internal/log"
-	"sneak.berlin/go/vaultik/internal/vaultik"
-)
-
-// TestPruneDeletesNothingThroughSymlinkedDirectory checks that prune keeps
-// every blob and the local snapshot record when the metadata/ or blobs/
-// directory of a file:// destination is a symlink to a directory elsewhere
-// (https://git.eeqj.de/sneak/vaultik/issues/296). A backup writes through
-// such a link, so prune has to list through it too.
-func TestPruneDeletesNothingThroughSymlinkedDirectory(t *testing.T) {
-	log.Initialize(log.Config{})
-	t.Parallel()
-
-	for _, dir := range []string{"metadata", "blobs"} {
-		t.Run(dir, func(t *testing.T) {
-			t.Parallel()
-
-			ctx := context.Background()
-			storeDir := filepath.Join(t.TempDir(), "store")
-			v, repos, _ := backUpToFileDestination(ctx, t, storeDir)
-
-			var blobFiles []string
-
-			err := filepath.WalkDir(filepath.Join(storeDir, "blobs"),
-				func(path string, entry fs.DirEntry, err error) error {
-					if err == nil && !entry.IsDir() {
-						blobFiles = append(blobFiles, path)
-					}
-
-					return err
-				})
-			require.NoError(t, err)
-			require.NotEmpty(t, blobFiles)
-
-			linkPath := filepath.Join(storeDir, dir)
-			movedPath := filepath.Join(t.TempDir(), dir)
-			require.NoError(t, os.Rename(linkPath, movedPath))
-			require.NoError(t, os.Symlink(movedPath, linkPath))
-
-			require.NoError(t, v.Prune(&vaultik.PruneOptions{Force: true}))
-
-			// When blobs/ is the link, these paths run through it.
-			for _, blobFile := range blobFiles {
-				assert.FileExists(t, blobFile, "prune must delete no blob")
-			}
-
-			snapshots, err := repos.Snapshots.ListRecent(ctx, listRecentTestLimit)
-			require.NoError(t, err)
-			assert.Len(t, snapshots, 1, "prune must delete no local snapshot record")
-		})
-	}
-}