Count a file a backup could not store as failed (closes #280)

A file that phase 1 of a backup counted and phase 2 could not open,
because it was unreadable under --skip-errors or removed in between,
was added to the unchanged count while its size stayed in
BytesScanned. The summary showed it as unchanged with its bytes backed
up, and the snapshots row's file_count and total_size included it.

The scanner now counts such a file in FilesFailed and takes its size
out of BytesScanned. The summary's files line adds "N failed", and
file_count leaves the file out.

A directory phase 2 cannot record is not counted as failed, since
phase 1 counts no directories; that case has no test.

Model: opus-5-5

README.md

@@ -412,8 +412,7 @@ destination store.
 entirely. Remote storage is unaffected; the next backup will do a full
 scan and re-deduplicate against existing remote blobs, and the local
 index will re-bind to the currently configured storage destination.
-Use this after changing `storage_url`, or `s3.endpoint`, `s3.bucket` or
+Use this after changing `storage_url` to a different destination.
-`s3.prefix`, to a different destination.
 * `--force`: Skip confirmation prompt
 
 ---

TODO.md

@@ -22,58 +22,6 @@ the tag exists and is exercised; what is left is merging `next` to
 
 # Completed Steps
 
-- 2026-10-13: Made an exclude pattern starting with `**/` also match
-  directly under the snapshot path
-  ([issue #303](https://git.eeqj.de/sneak/vaultik/issues/303)).
-  `**/node_modules` and `**/*.pyc`, the patterns `config.example.yml`
-  uses, excluded `sub/node_modules` and `sub/y.pyc` but backed up
-  `node_modules/` and `x.pyc` at the top of the path, since the pattern
-  required a `/` before the name. As in gitignore, such a pattern now
-  matches at every depth, the top included.
-
-- 2026-10-12: Bound the local index to the destination an `s3.*` config
-  names ([issue #298](https://git.eeqj.de/sneak/vaultik/issues/298)).
-  Only `storage_url` was checked, so after `s3.bucket` or `s3.prefix`
-  changed, the next backup treated every chunk as already stored,
-  uploaded nothing, and completed a snapshot that could not be restored.
-  Without `storage_url`, the local index is now bound to `s3.endpoint`,
-  `s3.bucket` and `s3.prefix`, and a change is refused as a changed
-  `storage_url` is.
-
-- 2026-10-12: Rejected a retention duration longer than about 292 years
-  ([issue #289](https://git.eeqj.de/sneak/vaultik/issues/289)). The
-  number given to `--older-than` or `--keep-newer-than` was multiplied
-  by the length of its unit with no limit, so from `293y`, `15251w`,
-  `3559mo` or `106752d`, or a sum of parts such as `200y200y`, the
-  result wrapped to a negative duration. The cutoff then lay in the
-  future, every snapshot of the name was deleted, the one just made
-  included, and the command exited 0. Such a duration is now rejected
-  as invalid and nothing is deleted.
-
-- 2026-10-08: Made a backup cancelled under `--skip-errors` stop at once
-  ([issue #286](https://git.eeqj.de/sneak/vaultik/issues/286)). After
-  Ctrl-C or SIGTERM, phase 2 treated the cancellation error from each
-  remaining file like an unreadable file: it opened the file, printed an
-  error line for it, counted it as failed and went on to the next one.
-  The run now stops at the first file after the cancellation, and no
-  file is counted as failed because of it.
-
-- 2026-10-08: Made `remote nuke` delete the `.partial` files that
-  uploads cut off part-way leave on the destination store
-  ([issue #281](https://git.eeqj.de/sneak/vaultik/issues/281)). The
-  file and rclone listings skip such a file, so the command left it in
-  place and still reported the store empty. It now removes them under
-  `metadata/` and `blobs/` as its last step.
-
-- 2026-10-08: Made a local index error while recording a directory or
-  symlink stop a backup under `--skip-errors`
-  ([issue #284](https://git.eeqj.de/sneak/vaultik/issues/284)). Phase 2
-  only records such an entry in the local index, since it has no data
-  to open or read, but an error doing so was skipped like an unreadable
-  file. The snapshot then completed without the entry, and a restore
-  did not recreate it. The error now stops the backup, as it does
-  without the flag.
-
 - 2026-10-08: Counted a file that a backup could not store as failed
   ([issue #280](https://git.eeqj.de/sneak/vaultik/issues/280)). A file
   that phase 1 counted and phase 2 could not open, because it was

internal/snapshot/exclude_test.go

@@ -531,50 +531,3 @@ func TestExcludePatterns_UnanchoredPatternFile(t *testing.T) {
 	require.Equal(t, 1, result.FilesScanned,
 		"Unanchored pattern for file should match anywhere")
 }
-
-func TestExcludePatterns_DoubleStarDirectoryAtEveryDepth(t *testing.T) {
-	t.Parallel()
-
-	// As in gitignore, a leading **/ also matches directly under the source dir
-	fs := setupAnchoredTestFS(t)
-
-	scanner, repos, cleanup := createTestScanner(t, fs, []string{"**/projectname"})
-	defer cleanup()
-
-	ctx := context.Background()
-	createSnapshotRecord(ctx, t, repos, "test-snapshot")
-
-	result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
-	require.NoError(t, err)
-
-	// **/projectname should exclude BOTH:
-	// - /backup/projectname/file.txt
-	// - /backup/otherproject/projectname/file.txt
-	// Total files: 4, excluded: 2
-	require.Equal(t, 2, result.FilesScanned,
-		"**/ directory pattern should match at the top level and nested")
-}
-
-func TestExcludePatterns_DoubleStarFileAtEveryDepth(t *testing.T) {
-	t.Parallel()
-
-	// As in gitignore, a leading **/ also matches directly under the source dir
-	fs := setupAnchoredTestFS(t)
-
-	scanner, repos, cleanup := createTestScanner(t, fs, []string{"**/*.txt"})
-	defer cleanup()
-
-	ctx := context.Background()
-	createSnapshotRecord(ctx, t, repos, "test-snapshot")
-
-	result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
-	require.NoError(t, err)
-
-	// **/*.txt should exclude ALL .txt files:
-	// - /backup/file.txt
-	// - /backup/projectname/file.txt
-	// - /backup/otherproject/projectname/file.txt
-	// Total files: 4, excluded: 3
-	require.Equal(t, 1, result.FilesScanned,
-		"**/ file pattern should match at the top level and nested")
-}

internal/snapshot/scanner.go

@@ -1312,13 +1312,6 @@ func (s *Scanner) processPhase(
 
 	// Process each file
 	for _, fileToProcess := range filesToProcess {
-		// Check context cancellation
-		select {
-		case <-ctx.Done():
-			return ctx.Err()
-		default:
-		}
-
 		// Update progress
 		if s.progress != nil {
 			s.progress.GetStats().CurrentFile.Store(fileToProcess.Path)
@@ -1355,32 +1348,13 @@ func (s *Scanner) processPhase(
 	return s.finalizeProcessPhase(ctx, result)
 }
 
-// processFileWithErrorHandling records a directory or symlink, or wraps
+// processFileWithErrorHandling wraps processFileStreaming with error recovery for
-// processFileStreaming for a regular file with error recovery for deleted
+// deleted files and skip-errors mode. Returns (skipped, error).
-// files and skip-errors mode. Returns (skipped, error).
 func (s *Scanner) processFileWithErrorHandling(
 	ctx context.Context, fileToProcess *FileToProcess, result *ScanResult,
 ) (bool, error) {
-	// A directory or symlink has no data to open or read; it is only
-	// recorded in the local index. An error recording it stops the run
-	// even under --skip-errors, or the snapshot would complete without it.
-	mode := os.FileMode(fileToProcess.File.Mode)
-	if mode&os.ModeSymlink != 0 || mode.IsDir() {
-		err := s.recordNonRegularFile(ctx, fileToProcess)
-		if err != nil {
-			return false, fmt.Errorf("processing file %s: %w", fileToProcess.Path, err)
-		}
-
-		return false, nil
-	}
-
 	err := s.processFileStreaming(ctx, fileToProcess, result)
 	if err != nil {
-		// A cancelled run stops here even under --skip-errors, rather than
-		// counting this file as failed and going on to the next one.
-		if ctx.Err() != nil {
-			return false, fmt.Errorf("processing file %s: %w", fileToProcess.Path, err)
-		}
 		// A packer/database/encryption/upload failure means the chunk's data
 		// may not have been stored. Skipping the file would let the snapshot
 		// record a file whose chunk is in no blob and cannot be restored, so
@@ -1418,7 +1392,12 @@ func (s *Scanner) processFileWithErrorHandling(
 
 // countFailedFile counts a file that phase 2 could not store as failed
 // and takes its size back out of BytesScanned, where phase 1 put it.
+// Phase 1 counts no directories, so a directory is not counted here.
 func countFailedFile(fileToProcess *FileToProcess, result *ScanResult) {
+	if fileToProcess.FileInfo.IsDir() {
+		return
+	}
+
 	result.FilesFailed++
 	result.BytesScanned -= fileToProcess.FileInfo.Size()
 }
@@ -1791,11 +1770,16 @@ func (e *packerError) Error() string { return e.err.Error() }
 
 func (e *packerError) Unwrap() error { return e.err }
 
-// processFileStreaming processes a regular file by streaming chunks directly
+// processFileStreaming processes a file by streaming chunks directly to the packer
-// to the packer
 func (s *Scanner) processFileStreaming(
 	ctx context.Context, fileToProcess *FileToProcess, result *ScanResult,
 ) error {
+	// Symlinks and directories have no data to chunk — just record them in the DB.
+	mode := os.FileMode(fileToProcess.File.Mode)
+	if mode&os.ModeSymlink != 0 || mode.IsDir() {
+		return s.recordNonRegularFile(ctx, fileToProcess)
+	}
+
 	file, err := s.fs.Open(fileToProcess.Path)
 	if err != nil {
 		return fmt.Errorf("opening file: %w", wrapPermissionError(fileToProcess.Path, err))
@@ -2013,14 +1997,6 @@ func compileExcludePatterns(patterns []string) []compiledPattern {
 		// Remove trailing slash if present (directory indicator)
 		pattern = strings.TrimSuffix(pattern, "/")
 
-		// A leading "**/" demands a "/" before the name and so misses an
-		// entry directly under the source path. An unanchored pattern
-		// already matches at every depth, so drop the prefix: as in
-		// gitignore, "**/x" then also matches a top-level "x".
-		if !anchored {
-			pattern = strings.TrimPrefix(pattern, "**/")
-		}
-
 		// Compile the glob pattern
 		// For patterns without path separators, we need to match them as components
 		// e.g., ".git" should match ".git" anywhere in the path

internal/snapshot/skip_errors_test.go

@@ -82,32 +82,6 @@ func (f *readFailFs) Open(name string) (afero.File, error) {
 	return file, nil
 }
 
-// cancelOnOpenFs cancels the run when the scanner opens the target file to
-// back it up, as Ctrl-C partway through processing would, and records each
-// file opened after that.
-type cancelOnOpenFs struct {
-	afero.Fs
-
-	target            string
-	cancel            context.CancelFunc
-	cancelled         bool
-	openedAfterCancel []string
-}
-
-//nolint:ireturn // afero.Fs.Open is defined to return the interface.
-func (f *cancelOnOpenFs) Open(name string) (afero.File, error) {
-	if f.cancelled {
-		f.openedAfterCancel = append(f.openedAfterCancel, name)
-	}
-
-	if name == f.target {
-		f.cancel()
-		f.cancelled = true
-	}
-
-	return f.Fs.Open(name)
-}
-
 // linkRemovedAfterLstatFs is the real filesystem, except that the symlink at
 // target is removed right after the walk lstats it, as happens when a link is
 // deleted during a backup. The scanner's readlink of it then fails.
@@ -154,16 +128,15 @@ func writeSkipErrorTestFile(t *testing.T, fs afero.Fs, path, content string) {
 	}
 }
 
-// runSkipErrorScan scans source on fs under ctx with the given skip-errors
+// runSkipErrorScan scans source on fs with the given skip-errors setting,
-// setting, printing user-facing messages to uiw (nil discards them), and
+// printing user-facing messages to uiw (nil discards them), and returns the
-// returns the repositories (for inspection) and the scan error.
+// repositories (for inspection) and the scan error.
 func runSkipErrorScan(
-	ctx context.Context, t *testing.T, fs afero.Fs, source string,
+	t *testing.T, fs afero.Fs, source string, skipErrors bool, uiw *ui.Writer,
-	skipErrors bool, uiw *ui.Writer,
 ) (*database.Repositories, error) {
 	t.Helper()
 
-	db, err := database.New(ctx, ":memory:")
+	db, err := database.NewTestDB()
 	if err != nil {
 		t.Fatalf("create test db: %v", err)
 	}
@@ -188,6 +161,7 @@ func runSkipErrorScan(
 		SkipErrors:       skipErrors,
 	})
 
+	ctx := context.Background()
 	snapshotID := "test-snapshot-skip-errors"
 	createTestSnapshotRecord(ctx, t, repos, snapshotID)
 
@@ -211,7 +185,7 @@ func TestScannerPackingFailureAbortsUnderSkipErrors(t *testing.T) {
 	writeSkipErrorTestFile(t, fs, "/source/file1.txt", "first file content")
 	writeSkipErrorTestFile(t, fs, "/source/file2.txt", "second file content")
 
-	repos, err := runSkipErrorScan(context.Background(), t, fs, "/source", true, nil)
+	repos, err := runSkipErrorScan(t, fs, "/source", true, nil)
 	if err == nil {
 		t.Fatal("expected scan to abort on the packer error, got nil")
 	}
@@ -238,7 +212,7 @@ func TestScannerReadErrorAbortsWithoutSkipErrors(t *testing.T) {
 	fs := &readFailFs{Fs: afero.NewMemMapFs(), target: target}
 	writeSkipErrorTestFile(t, fs, target, "content that cannot be read")
 
-	_, err := runSkipErrorScan(context.Background(), t, fs, "/source", false, nil)
+	_, err := runSkipErrorScan(t, fs, "/source", false, nil)
 	if err == nil {
 		t.Fatal("expected scan to fail on the read error, got nil")
 	}
@@ -254,7 +228,7 @@ func TestScannerReadErrorSkippedWithSkipErrors(t *testing.T) {
 	fs := &readFailFs{Fs: afero.NewMemMapFs(), target: target}
 	writeSkipErrorTestFile(t, fs, target, "content that cannot be read")
 
-	repos, err := runSkipErrorScan(context.Background(), t, fs, "/source", true, nil)
+	repos, err := runSkipErrorScan(t, fs, "/source", true, nil)
 	if err != nil {
 		t.Fatalf("expected scan to complete with --skip-errors, got %v", err)
 	}
@@ -293,7 +267,7 @@ func TestScannerUnreadableSymlinkAbortsWithoutSkipErrors(t *testing.T) {
 	sourceDir, linkPath := writeSymlinkSource(t)
 	fs := &linkRemovedAfterLstatFs{t: t, target: linkPath}
 
-	_, err := runSkipErrorScan(context.Background(), t, fs, sourceDir, false, nil)
+	_, err := runSkipErrorScan(t, fs, sourceDir, false, nil)
 	if !errors.Is(err, os.ErrNotExist) {
 		t.Fatalf("expected scan to fail on the removed symlink, got %v", err)
 	}
@@ -309,7 +283,7 @@ func TestScannerUnreadableSymlinkSkippedWithSkipErrors(t *testing.T) {
 	fs := &linkRemovedAfterLstatFs{t: t, target: linkPath}
 	uiw := ui.NewWithColor(io.Discard, false)
 
-	repos, err := runSkipErrorScan(context.Background(), t, fs, sourceDir, true, uiw)
+	repos, err := runSkipErrorScan(t, fs, sourceDir, true, uiw)
 	if err != nil {
 		t.Fatalf("expected scan to complete with --skip-errors, got %v", err)
 	}
@@ -328,56 +302,3 @@ func TestScannerUnreadableSymlinkSkippedWithSkipErrors(t *testing.T) {
 		t.Fatalf("expected %s not to be recorded", linkPath)
 	}
 }
-
-// TestScannerCancelStopsSkipErrorsRun checks that a --skip-errors backup
-// cancelled partway through processing returns the cancellation error,
-// opens no further file, and reports no file as failed.
-func TestScannerCancelStopsSkipErrorsRun(t *testing.T) {
-	t.Parallel()
-
-	tests := []struct {
-		name          string
-		targetContent string
-	}{
-		// The cancellation lands while the target is being read.
-		{name: "while reading a file", targetContent: "first file content"},
-		// An empty target has no chunks, so the cancellation goes unnoticed
-		// until the run moves on to the next file.
-		{name: "between files", targetContent: ""},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			t.Parallel()
-
-			const target = "/source/a.txt"
-
-			ctx, cancel := context.WithCancel(context.Background())
-			defer cancel()
-
-			fs := &cancelOnOpenFs{
-				Fs: afero.NewMemMapFs(), target: target, cancel: cancel,
-			}
-			writeSkipErrorTestFile(t, fs, target, tt.targetContent)
-			writeSkipErrorTestFile(t, fs, "/source/b.txt", "second file content")
-			writeSkipErrorTestFile(t, fs, "/source/c.txt", "third file content")
-
-			uiw := ui.NewWithColor(io.Discard, false)
-
-			_, err := runSkipErrorScan(ctx, t, fs, "/source", true, uiw)
-			if !errors.Is(err, context.Canceled) {
-				t.Fatalf("expected the cancellation error, got %v", err)
-			}
-
-			if len(fs.openedAfterCancel) != 0 {
-				t.Fatalf("expected no file opened after the cancellation, got %v",
-					fs.openedAfterCancel)
-			}
-
-			if uiw.ErrorCount() != 0 {
-				t.Fatalf("expected no file reported as failed, got %d error lines",
-					uiw.ErrorCount())
-			}
-		})
-	}
-}

internal/storage/faultstore/faultstore.go

@@ -171,11 +171,6 @@ func (f *Storer) ListStream(
 	return f.inner.ListStream(ctx, prefix)
 }
 
-// DeletePartialUploads delegates unchanged.
-func (f *Storer) DeletePartialUploads(ctx context.Context, prefix string) error {
-	return f.inner.DeletePartialUploads(ctx, prefix)
-}
-
 // Info delegates unchanged.
 func (f *Storer) Info() storage.Info {
 	return f.inner.Info()

internal/storage/file.go

@@ -242,42 +242,6 @@ func (f *FileStorer) ListStream(ctx context.Context, prefix string) <-chan Objec
 	return ch
 }
 
-// DeletePartialUploads removes every file under prefix whose name ends in
-// tempSuffix. A missing prefix has none to remove.
-func (f *FileStorer) DeletePartialUploads(ctx context.Context, prefix string) error {
-	basePath := f.fullPath(prefix)
-
-	exists, err := afero.Exists(f.fs, basePath)
-	if err != nil {
-		return fmt.Errorf("checking path: %w", err)
-	}
-
-	if !exists {
-		return nil
-	}
-
-	err = afero.Walk(f.fs, basePath, func(path string, info os.FileInfo, err error) error {
-		if err != nil {
-			return err
-		}
-
-		if ctx.Err() != nil {
-			return ctx.Err()
-		}
-
-		if info.IsDir() || !strings.HasSuffix(info.Name(), tempSuffix) {
-			return nil
-		}
-
-		return f.fs.Remove(path)
-	})
-	if err != nil {
-		return fmt.Errorf("walking directory: %w", err)
-	}
-
-	return nil
-}
-
 // Info returns human-readable storage location information.
 func (f *FileStorer) Info() Info {
 	return Info{

internal/storage/file_atomic_test.go

@@ -120,45 +120,3 @@ func TestFileStorer_ListSkipsPartialFiles(t *testing.T) {
 		t.Fatalf("ListStream should return only the real key, got %v", streamed)
 	}
 }
-
-// TestFileStorer_DeletePartialUploads checks that a leftover temp file is
-// removed and the object at the real key is kept.
-func TestFileStorer_DeletePartialUploads(t *testing.T) {
-	t.Parallel()
-
-	base := t.TempDir()
-
-	f, err := storage.NewFileStorer(base)
-	if err != nil {
-		t.Fatalf("NewFileStorer: %v", err)
-	}
-
-	ctx := context.Background()
-
-	err = f.Put(ctx, testBlobKey, strings.NewReader("blob-bytes"))
-	if err != nil {
-		t.Fatalf("Put: %v", err)
-	}
-
-	leftover := filepath.Join(base, testBlobKey+"-123456.partial")
-
-	err = os.WriteFile(leftover, []byte("half"), 0o600)
-	if err != nil {
-		t.Fatalf("writing leftover temp file: %v", err)
-	}
-
-	err = f.DeletePartialUploads(ctx, "blobs/")
-	if err != nil {
-		t.Fatalf("DeletePartialUploads: %v", err)
-	}
-
-	_, err = os.Stat(leftover)
-	if !os.IsNotExist(err) {
-		t.Errorf("leftover temp file was not removed: %v", err)
-	}
-
-	_, err = f.Stat(ctx, testBlobKey)
-	if err != nil {
-		t.Errorf("Stat of the real key: %v", err)
-	}
-}

internal/storage/rclone.go

@@ -213,31 +213,6 @@ func (r *RcloneStorer) ListStream(
 	return ch
 }
 
-// DeletePartialUploads removes every object under prefix whose name ends
-// in tempSuffix.
-func (r *RcloneStorer) DeletePartialUploads(ctx context.Context, prefix string) error {
-	var partial []fs.Object
-
-	err := operations.ListFn(ctx, r.fsys, func(obj fs.Object) {
-		key := obj.Remote()
-		if strings.HasPrefix(key, prefix) && strings.HasSuffix(key, tempSuffix) {
-			partial = append(partial, obj)
-		}
-	})
-	if err != nil {
-		return fmt.Errorf("listing objects: %w", err)
-	}
-
-	for _, obj := range partial {
-		err = obj.Remove(ctx)
-		if err != nil {
-			return fmt.Errorf("removing object: %w", err)
-		}
-	}
-
-	return nil
-}
-
 // Info returns human-readable storage location information.
 func (r *RcloneStorer) Info() Info {
 	location := r.remote

internal/storage/rclone_test.go

@@ -198,47 +198,6 @@ func TestRcloneStorerListSkipsPartialFiles(t *testing.T) {
 	}
 }
 
-// TestRcloneStorerDeletePartialUploads checks that a temporary file left
-// by a killed upload is removed and the object at the real key is kept.
-//
-//nolint:paralleltest // NewRcloneStorer installs the process-global rclone config
-func TestRcloneStorerDeletePartialUploads(t *testing.T) {
-	dir := t.TempDir()
-	ctx := context.Background()
-
-	s, err := storage.NewRcloneStorer(ctx, ":local", dir)
-	if err != nil {
-		t.Fatalf("NewRcloneStorer: %v", err)
-	}
-
-	err = s.Put(ctx, testBlobKey, strings.NewReader("blob-bytes"))
-	if err != nil {
-		t.Fatalf("Put: %v", err)
-	}
-
-	leftover := filepath.Join(dir, testBlobKey+"-123456.partial")
-
-	err = os.WriteFile(leftover, []byte("half"), 0o600)
-	if err != nil {
-		t.Fatalf("writing leftover temp file: %v", err)
-	}
-
-	err = s.DeletePartialUploads(ctx, "blobs/")
-	if err != nil {
-		t.Fatalf("DeletePartialUploads: %v", err)
-	}
-
-	_, err = os.Stat(leftover)
-	if !os.IsNotExist(err) {
-		t.Errorf("leftover temp file was not removed: %v", err)
-	}
-
-	_, err = s.Stat(ctx, testBlobKey)
-	if err != nil {
-		t.Errorf("Stat of the real key: %v", err)
-	}
-}
-
 // newRcloneStorerOnWrappedLocal registers name as rclone's local backend
 // wrapped by wrap, and builds an rclone backend on it rooted at a fresh
 // temp directory. wrap changes the features the local backend reports, so

internal/storage/s3.go

@@ -99,12 +99,6 @@ func (s *S3Storer) ListStream(ctx context.Context, prefix string) <-chan ObjectI
 	return ch
 }
 
-// DeletePartialUploads has nothing to remove: S3 shows an object only once
-// its upload has completed, so an upload cut off part-way leaves none.
-func (s *S3Storer) DeletePartialUploads(_ context.Context, _ string) error {
-	return nil
-}
-
 // Info returns human-readable storage location information.
 func (s *S3Storer) Info() Info {
 	return Info{

internal/storage/storer.go

@@ -71,12 +71,6 @@ type Storer interface {
 	// If an error occurs during listing, the final item will have Err set.
 	ListStream(ctx context.Context, prefix string) <-chan ObjectInfo
 
-	// DeletePartialUploads removes every object under prefix that an
-	// upload cut off part-way left under a temporary name ending in
-	// `.partial`. The file and rclone backends' List and ListStream skip
-	// such an object.
-	DeletePartialUploads(ctx context.Context, prefix string) error
-
 	// Info returns human-readable storage location information.
 	Info() Info
 }

internal/vaultik/destination_validation_test.go

@@ -257,10 +257,6 @@ func (s *stubLister) List(_ context.Context, _ string) ([]string, error) {
 	return nil, errStubUnused
 }
 
-func (s *stubLister) DeletePartialUploads(_ context.Context, _ string) error {
-	return errStubUnused
-}
-
 func (s *stubLister) Info() storage.Info {
 	return storage.Info{}
 }

internal/vaultik/helpers.go

@@ -3,7 +3,6 @@ package vaultik
 import (
 	"errors"
 	"fmt"
-	"math"
 	"regexp"
 	"strconv"
 	"strings"
@@ -48,10 +47,6 @@ const (
 	year  = 365 * day
 )
 
-// maxDuration is the largest value a time.Duration can hold, about 292
-// years.
-const maxDuration = time.Duration(math.MaxInt64)
-
 // A snapshot ID split on "_" has at least a hostname and a trailing
 // timestamp.
 const minSnapshotIDParts = 2
@@ -150,10 +145,10 @@ func parseSnapshotName(snapshotID, hostname string) string {
 // parseDuration parses a duration string with support for human-friendly units:
 // d/day/days, w/week/weeks, mo/month/months, y/year/years, plus standard Go
 // duration units. Following Go, m is minutes and mo is months. A bare number,
-// an unknown unit, a negative value, and a total longer than maxDuration are
+// an unknown unit, and a negative value are all rejected. Outside the Go
-// all rejected. Outside the Go units the input must be whole numbers each
+// units the input must be whole numbers each followed directly by a unit
-// followed directly by a unit (2w3d), so 1.0y and 30 days are errors;
+// (2w3d), so 1.0y and 30 days are errors; decimals work only in Go units
-// decimals work only in Go units (1.5h).
+// (1.5h).
 func parseDuration(s string) (time.Duration, error) {
 	if strings.HasPrefix(strings.TrimSpace(s), "-") {
 		return 0, errNegativeDuration
@@ -186,30 +181,19 @@ func parseDuration(s string) (time.Duration, error) {
 			return 0, fmt.Errorf("invalid number %q: %w", match[1], err)
 		}
 
-		var unitLength time.Duration
-
 		unit := strings.ToLower(match[2])
 		switch unit {
 		case "d", "day", "days":
-			unitLength = day
+			total += time.Duration(n) * day
 		case "w", "week", "weeks":
-			unitLength = week
+			total += time.Duration(n) * week
 		case "mo", "month", "months":
-			unitLength = month
+			total += time.Duration(n) * month
 		case "y", "year", "years":
-			unitLength = year
+			total += time.Duration(n) * year
 		default:
 			return 0, fmt.Errorf("%w %q", errUnknownTimeUnit, unit)
 		}
-
-		// Past maxDuration the sum wraps to a negative value, which would
-		// put the purge cutoff in the future.
-		if time.Duration(n) > (maxDuration-total)/unitLength {
-			return 0, fmt.Errorf("%w: %q is longer than about 292 years",
-				errInvalidDuration, s)
-		}
-
-		total += time.Duration(n) * unitLength
 	}
 
 	return total, nil

internal/vaultik/helpers_test.go

@@ -111,17 +111,6 @@ func TestParseDuration(t *testing.T) {
 		{"30 days", 0, true},
 		{"30 days ago", 0, true},
 		{"x7d", 0, true},
-		// The largest time.Duration is about 292 years. A larger value
-		// must be rejected, not wrapped to a negative one, which would
-		// put the cutoff in the future and select every snapshot.
-		{"292y", 292 * 365 * 24 * time.Hour, false},
-		{"293y", 0, true},
-		{"3559mo", 0, true},
-		{"15251w", 0, true},
-		{"106752d", 0, true},
-		{"2562048h", 0, true},
-		{"200y200y", 0, true},
-		{"292y6mo", 0, true},
 	}
 
 	for _, tt := range tests {

internal/vaultik/integration_test.go

@@ -155,12 +155,6 @@ func (m *MockStorer) ListStream(
 	return ch
 }
 
-// DeletePartialUploads has nothing to remove: Put stores each object
-// under its key at once.
-func (m *MockStorer) DeletePartialUploads(_ context.Context, _ string) error {
-	return nil
-}
-
 func (m *MockStorer) Info() storage.Info {
 	return storage.Info{
 		Type:     "mock",

internal/vaultik/nuke_remote_test.go

@@ -1,64 +0,0 @@
-package vaultik_test
-
-import (
-	"context"
-	"io/fs"
-	"os"
-	"path/filepath"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"sneak.berlin/go/vaultik/internal/log"
-	"sneak.berlin/go/vaultik/internal/snapshot"
-)
-
-// TestNukeRemoteLeavesNoFiles checks that remote nuke leaves no file
-// under a file:// destination, including the `.partial` files uploads
-// killed part-way leave next to a snapshot's metadata and next to where a
-// blob would have been.
-func TestNukeRemoteLeavesNoFiles(t *testing.T) {
-	log.Initialize(log.Config{})
-	t.Parallel()
-
-	ctx := context.Background()
-	storeDir := filepath.Join(t.TempDir(), "store")
-	v, repos, _ := backUpToFileDestination(ctx, t, storeDir)
-
-	snapshots, err := repos.Snapshots.ListRecent(ctx, listRecentTestLimit)
-	require.NoError(t, err)
-	require.Len(t, snapshots, 1)
-
-	snapshotKey := snapshot.RemoteSnapshotKey(snapshots[0].ID.String())
-	hash := testBlobHashA
-	leftovers := []string{
-		filepath.Join(storeDir, "metadata", snapshotKey,
-			"db.zst.age-123456.partial"),
-		filepath.Join(storeDir, "blobs", hash[:2], hash[2:4],
-			hash+"-123456.partial"),
-	}
-
-	for _, leftover := range leftovers {
-		require.NoError(t, os.MkdirAll(filepath.Dir(leftover), 0o750))
-		require.NoError(t, os.WriteFile(leftover, []byte("half an upload"), 0o600))
-	}
-
-	require.NoError(t, v.NukeRemote(true))
-
-	var files []string
-
-	err = filepath.WalkDir(storeDir,
-		func(path string, entry fs.DirEntry, err error) error {
-			if err != nil {
-				return err
-			}
-
-			if !entry.IsDir() {
-				files = append(files, path)
-			}
-
-			return nil
-		})
-	require.NoError(t, err)
-	assert.Empty(t, files)
-}

internal/vaultik/prune.go

@@ -24,9 +24,8 @@ var errNukeRequiresForce = errors.New(
 const metadataDirName = "metadata"
 
 // NukeRemote deletes every snapshot's metadata and every blob from remote
-// storage, along with any object an upload cut off part-way left under a
+// storage. After this returns successfully the bucket prefix is empty and
-// temporary `.partial` name. After this returns successfully the bucket
+// the next backup starts from scratch.
-// prefix is empty and the next backup starts from scratch.
 //
 // Refuses to run unless force is true. The caller is responsible for
 // confirming with the user.
@@ -49,15 +48,6 @@ func (v *Vaultik) NukeRemote(force bool) error {
 		return fmt.Errorf("pruning blobs: %w", err)
 	}
 
-	// The file and rclone listings skip `.partial` objects, so the two
-	// steps above never delete them.
-	for _, prefix := range []string{"metadata/", "blobs/"} {
-		err = v.Storage.DeletePartialUploads(v.ctx, prefix)
-		if err != nil {
-			return fmt.Errorf("deleting partial uploads: %w", err)
-		}
-	}
-
 	v.UI.Completef("Backup destination store is now empty.")
 
 	return nil

internal/vaultik/purge_per_name_test.go

@@ -267,29 +267,6 @@ func TestPurgeOlderThan_WithNameFilter(t *testing.T) {
 	assert.Contains(t, remaining, snapHomeT0)
 }
 
-func TestPurgeOlderThan_TooLargeDeletesNothing(t *testing.T) {
-	log.Initialize(log.Config{})
-	t.Parallel()
-
-	snapshotIDs := []string{snapHomeT0, snapHomeT1}
-
-	v := setupPurgeTest(t, snapHostname, snapshotIDs)
-
-	err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{
-		OlderThan: "1000y",
-		Force:     true,
-	})
-	require.Error(t, err)
-
-	err = v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{
-		OlderThan: "292y",
-		Force:     true,
-	})
-	require.NoError(t, err)
-
-	assert.ElementsMatch(t, snapshotIDs, listRemainingSnapshots(t, v))
-}
-
 func TestPurgeKeepLatest_ThreeNames(t *testing.T) {
 	log.Initialize(log.Config{})
 	t.Parallel()

internal/vaultik/remove_snapshot_test.go

@@ -125,12 +125,6 @@ func (s *testStorer) ListStream(
 	return ch
 }
 
-// DeletePartialUploads has nothing to remove: Put stores each object
-// under its key at once.
-func (s *testStorer) DeletePartialUploads(_ context.Context, _ string) error {
-	return nil
-}
-
 func (s *testStorer) Info() storage.Info {
 	return storage.Info{
 		Type:     testLabel,

internal/vaultik/snapshot_skip_errors_test.go

@@ -1,75 +0,0 @@
-package vaultik_test
-
-import (
-	"context"
-	"fmt"
-	"path/filepath"
-	"testing"
-
-	"github.com/spf13/afero"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"sneak.berlin/go/vaultik/internal/config"
-	"sneak.berlin/go/vaultik/internal/database"
-	"sneak.berlin/go/vaultik/internal/log"
-	"sneak.berlin/go/vaultik/internal/storage"
-	"sneak.berlin/go/vaultik/internal/vaultik"
-)
-
-// --skip-errors skips only a file that cannot be opened or read. A local
-// index error while a directory is recorded stops the backup, and the
-// snapshot is not recorded as complete. See
-// https://git.eeqj.de/sneak/vaultik/issues/284.
-func TestSkipErrorsBackupStopsOnIndexErrorRecordingDirectory(t *testing.T) {
-	log.Initialize(log.Config{})
-	t.Parallel()
-
-	ctx := context.Background()
-
-	// The scan walks the source path with symlinks resolved, so dirPath
-	// must be spelled the same way to match the row the scan inserts.
-	tempDir, err := filepath.EvalSymlinks(t.TempDir())
-	require.NoError(t, err)
-
-	srcDir := filepath.Join(tempDir, "src")
-	dirPath := filepath.Join(srcDir, "dir")
-
-	fs := afero.NewOsFs()
-	require.NoError(t, fs.MkdirAll(dirPath, 0o755))
-	require.NoError(t, afero.WriteFile(fs,
-		filepath.Join(dirPath, "file.txt"), []byte("file content"), 0o644))
-
-	cfg := faultTestConfig()
-	cfg.IndexPath = filepath.Join(tempDir, "index.sqlite")
-	cfg.Snapshots = map[string]config.SnapshotConfig{
-		"tree": {Paths: []string{srcDir}},
-	}
-
-	store, err := storage.NewFileStorer(filepath.Join(tempDir, "remote"))
-	require.NoError(t, err)
-
-	db, err := database.New(ctx, cfg.IndexPath)
-	require.NoError(t, err)
-	t.Cleanup(func() { _ = db.Close() })
-
-	// The local index refuses the directory's files row.
-	_, err = db.Conn().ExecContext(ctx, fmt.Sprintf(`
-		CREATE TRIGGER refuse_directory BEFORE INSERT ON files
-		WHEN NEW.path = '%s'
-		BEGIN SELECT RAISE(ABORT, 'simulated index error'); END`, dirPath))
-	require.NoError(t, err)
-
-	repos := database.NewRepositories(db)
-	v := newBackupVaultik(ctx, cfg, store, repos, db, fs)
-
-	err = v.CreateSnapshot(&vaultik.SnapshotCreateOptions{
-		SkipErrors: true,
-		Snapshots:  []string{"tree"},
-	})
-	require.ErrorContains(t, err, "simulated index error")
-
-	snapshots, err := repos.Snapshots.ListRecent(ctx, listRecentTestLimit)
-	require.NoError(t, err)
-	require.Len(t, snapshots, 1)
-	assert.Nil(t, snapshots[0].CompletedAt)
-}

internal/vaultik/storage_bind.go

@@ -9,7 +9,7 @@ import (
 )
 
 // errStorageBindingMismatch is returned when the local index database is
-// bound to a different destination than the configured one.
+// bound to a different destination than the configured storage_url.
 var errStorageBindingMismatch = errors.New(
 	"local index is bound to a different backup destination")
 
@@ -24,9 +24,8 @@ var errStorageBindingMismatch = errors.New(
 //
 // Behaviour:
 //   - On first use (empty stored value), record the configured
-//     destination (storage_url, or the s3.* endpoint, bucket and
+//     storage_url and log the binding.
-//     prefix) and log the binding.
+//   - When the stored value matches the configured storage_url, do
-//   - When the stored value matches the configured destination, do
 //     nothing and return nil.
 //   - When the two differ, refuse with an error that tells the user
 //     how to recover (revert the config, or run `vaultik database
@@ -47,10 +46,12 @@ func (v *Vaultik) EnsureStorageBinding() error {
 
 	configured := v.Config.StorageURL
 	if configured == "" {
-		// The s3.* form: bind on its endpoint, bucket and prefix,
+		// Some legacy configs still use the split s3.* keys instead of
-		// written the way storage_url would name the same destination.
+		// storage_url. Falling back to a synthetic URL for those would
-		s3 := v.Config.S3
+		// hide the fact that the binding is loose. Instead, treat
-		configured = "s3://" + s3.Bucket + "/" + s3.Prefix + "?endpoint=" + s3.Endpoint
+		// unset as "nothing to bind against" — the check is
+		// necessarily best-effort for those configs.
+		return nil
 	}
 
 	stored, err := v.Repositories.LocalMeta.Get(v.ctx, database.LocalMetaKeyStorageURL)
@@ -95,8 +96,7 @@ func buildBindingMismatchMessage(stored, configured string) string {
 		"future snapshots referencing blobs that don't exist at the new destination.\n" +
 		"\n" +
 		"To proceed, either:\n" +
-		"  - revert storage_url (or the s3.* endpoint, bucket and prefix) in your\n" +
+		"  - revert storage_url in your config to the bound destination, or\n" +
-		"    config to the bound destination, or\n" +
 		"  - run 'vaultik database delete' to discard the local index and rebuild it\n" +
 		"    from a fresh full backup against the new destination"
 }

internal/vaultik/storage_bind_test.go

@@ -76,51 +76,3 @@ func TestEnsureStorageBinding_MismatchRefuses(t *testing.T) {
 	assert.Contains(t, err.Error(), "file:///mnt/backups/new")
 	assert.Contains(t, err.Error(), "vaultik database delete")
 }
-
-// A config with no storage_url names its destination with s3.endpoint,
-// s3.bucket and s3.prefix; a change to any of them must be refused just
-// as a changed storage_url is.
-func TestEnsureStorageBinding_S3FieldsChangeRefuses(t *testing.T) {
-	log.Initialize(log.Config{})
-	t.Parallel()
-
-	bound := config.S3Config{
-		Endpoint: "https://s3.example.com",
-		Bucket:   "bucket-a",
-		Prefix:   "backups",
-	}
-
-	tests := []struct {
-		name    string
-		changed config.S3Config
-	}{
-		{"bucket", config.S3Config{
-			Endpoint: bound.Endpoint, Bucket: "bucket-b", Prefix: bound.Prefix,
-		}},
-		{"prefix", config.S3Config{
-			Endpoint: bound.Endpoint, Bucket: bound.Bucket, Prefix: "other",
-		}},
-		{"endpoint", config.S3Config{
-			Endpoint: "https://s3.other.com", Bucket: bound.Bucket, Prefix: bound.Prefix,
-		}},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			t.Parallel()
-
-			tv, _ := buildBindTestVaultik(t, "")
-
-			tv.Config.S3 = bound
-			require.NoError(t, tv.EnsureStorageBinding(), "first use binds")
-			require.NoError(t, tv.EnsureStorageBinding(), "same destination passes")
-
-			tv.Config.S3 = tt.changed
-			err := tv.EnsureStorageBinding()
-			require.Error(t, err)
-			assert.Contains(t, err.Error(),
-				"local index is bound to a different backup destination")
-			assert.Contains(t, err.Error(), "vaultik database delete")
-		})
-	}
-}