Remove the blob temp file when a backup stops part-way (closes #302)

When `snapshot create` stopped on a file error, a full temporary
directory or Ctrl-C, the blob being packed was never finalized, and its
`vaultik-blob-*.tmp` file (up to `blob_size_limit` of encrypted data)
stayed in `$TMPDIR`. Phase 2 now calls the new
`Packer.DiscardCurrentBlob` on the way out, which closes and removes
that file. After a successful flush no blob is in progress, so the call
then does nothing.

The blob's row in the local index is left, as before, for the next
run's repair of interrupted blobs to remove.

Model: opus-5-5

README.md

@@ -401,7 +401,9 @@ orphaned blob figures are reported as unknown; `--json` gives them as
 `null`, lists the remote key of each unreadable manifest in
 `unreadable_manifests` and counts the manifests under skipped names in
 `skipped_manifest_count`. An unreadable manifest also leaves its
-snapshot's blob count and blob size unknown, `null` in `--json`.
+snapshot's blob count and blob size unknown, `null` in `--json`. A file
+under `blobs/` whose name is not a blob hash is not counted as a blob,
+since `prune` never deletes it.
 * `--json`: Output as JSON
 
 **`remote nuke`**: Delete every snapshot's metadata and every blob from the

TODO.md

@@ -22,14 +22,13 @@ the tag exists and is exercised; what is left is merging `next` to
 
 # Completed Steps
 
-- 2026-10-13: Made an S3 listing that the server cuts short an error
-  ([issue #297](https://git.eeqj.de/sneak/vaultik/issues/297)). A page
-  marked truncated with no continuation token, as a server answering V2
-  listings V1-style sends, ended the listing without an error, so
-  `prune` saw only the snapshots on the first page and could delete
-  blobs the rest still use. Such a page is now a listing error, and
-  `prune`, `snapshot list` and `remote nuke` handle it as a destination
-  that cannot be listed.
+- 2026-10-13: Stopped `remote info` counting a file under `blobs/` that
+  is not a blob as an orphaned blob
+  ([issue #295](https://git.eeqj.de/sneak/vaultik/issues/295)). A stray
+  file such as `.DS_Store` was reported as an orphaned blob and the
+  report advised running `prune`, which skips such a file and deleted
+  nothing. `remote info` now takes its blob list from the same listing
+  `prune` uses, in the table and in `--json`.
 
 - 2026-10-13: Made `snapshot remove` fail for a snapshot that does not
   exist ([issue #293](https://git.eeqj.de/sneak/vaultik/issues/293)).

internal/blob/packer.go

@@ -245,6 +245,17 @@ func (p *Packer) Flush(ctx context.Context) error {
 	return nil
 }
 
+// DiscardCurrentBlob closes and removes the temp file of the blob being
+// packed, if any, and drops that blob. A caller that stops before Flush
+// must call it, or the temp file is left behind. Thread-safe.
+func (p *Packer) DiscardCurrentBlob() {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+
+	p.cleanupTempFile()
+	p.currentBlob = nil
+}
+
 // FinalizeBlob finalizes the current blob being assembled.
 // This compresses the accumulated chunks, encrypts the result, and computes
 // the content-addressed hash. The finalized blob is either passed to the

internal/s3/client.go

@@ -18,13 +18,6 @@ import (
 	"github.com/aws/smithy-go/logging"
 )
 
-// ErrNoContinuationToken is returned by ListObjects and ListObjectsStream
-// when the server marks a page of a listing truncated but sends no
-// continuation token to fetch the rest with, as a server answering V2
-// listings V1-style does.
-var ErrNoContinuationToken = errors.New(
-	"server marked the listing truncated but sent no continuation token")
-
 // Client wraps the AWS S3 client for vaultik operations.
 // It provides a simplified interface for S3 operations with automatic
 // prefix handling and connection management. All operations are performed
@@ -221,7 +214,7 @@ func (c *Client) ListObjects(ctx context.Context, prefix string) ([]string, erro
 	})
 
 	for paginator.HasMorePages() {
-		page, err := nextListPage(ctx, paginator)
+		page, err := paginator.NextPage(ctx)
 		if err != nil {
 			return nil, err
 		}
@@ -242,26 +235,6 @@ func (c *Client) ListObjects(ctx context.Context, prefix string) ([]string, erro
 	return keys, nil
 }
 
-// nextListPage fetches the next page of a listing. The SDK paginator takes
-// a page marked truncated without a continuation token for the last page
-// and stops without an error, so such a page is ErrNoContinuationToken
-// here instead.
-func nextListPage(
-	ctx context.Context, paginator *s3.ListObjectsV2Paginator,
-) (*s3.ListObjectsV2Output, error) {
-	page, err := paginator.NextPage(ctx)
-	if err != nil {
-		return nil, err
-	}
-
-	truncated := aws.ToBool(page.IsTruncated)
-	if truncated && aws.ToString(page.NextContinuationToken) == "" {
-		return nil, ErrNoContinuationToken
-	}
-
-	return page, nil
-}
-
 // HeadObject checks if an object exists in S3.
 // Returns true if the object exists, false otherwise.
 // The key is automatically prefixed with the configured prefix.
@@ -327,7 +300,7 @@ func (c *Client) ListObjectsStream(
 		})
 
 		for paginator.HasMorePages() {
-			page, err := nextListPage(ctx, paginator)
+			page, err := paginator.NextPage(ctx)
 			if err != nil {
 				ch <- ObjectInfo{Err: err}
 

internal/snapshot/scanner.go

@@ -1295,6 +1295,11 @@ func (s *Scanner) batchAddFilesToSnapshot(
 func (s *Scanner) processPhase(
 	ctx context.Context, filesToProcess []*FileToProcess, result *ScanResult,
 ) error {
+	// On an error or cancellation the blob being packed is never
+	// finalized, so its temp file must be removed here. After a
+	// successful flush there is no such blob and this does nothing.
+	defer s.packer.DiscardCurrentBlob()
+
 	// Calculate total bytes to process
 	var totalBytes int64
 	for _, f := range filesToProcess {

internal/snapshot/skip_errors_test.go

@@ -381,3 +381,60 @@ func TestScannerCancelStopsSkipErrorsRun(t *testing.T) {
 		})
 	}
 }
+
+// TestScannerRemovesBlobTempFileWhenRunStops checks that a run stopped
+// part-way, by a file it cannot read or by a cancellation, removes the
+// temp file of the blob it was packing.
+func TestScannerRemovesBlobTempFileWhenRunStops(t *testing.T) {
+	t.Parallel()
+
+	// a.txt is backed up first and starts a blob; the run stops at b.txt.
+	const target = "/source/b.txt"
+
+	tests := []struct {
+		name  string
+		newFs func(cancel context.CancelFunc) afero.Fs
+	}{
+		{
+			name: "file cannot be read",
+			newFs: func(context.CancelFunc) afero.Fs {
+				return &readFailFs{Fs: afero.NewMemMapFs(), target: target}
+			},
+		},
+		{
+			name: "cancelled",
+			newFs: func(cancel context.CancelFunc) afero.Fs {
+				return &cancelOnOpenFs{
+					Fs: afero.NewMemMapFs(), target: target, cancel: cancel,
+				}
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			ctx, cancel := context.WithCancel(context.Background())
+			defer cancel()
+
+			fs := tt.newFs(cancel)
+			writeSkipErrorTestFile(t, fs, "/source/a.txt", "first file content")
+			writeSkipErrorTestFile(t, fs, target, "second file content")
+
+			_, err := runSkipErrorScan(ctx, t, fs, "/source", false, nil)
+			if err == nil {
+				t.Fatal("expected the scan to stop with an error, got nil")
+			}
+
+			left, err := afero.Glob(fs, filepath.Join(os.TempDir(), "vaultik-blob-*"))
+			if err != nil {
+				t.Fatalf("listing blob temp files: %v", err)
+			}
+
+			if len(left) != 0 {
+				t.Fatalf("expected no blob temp file left, got %v", left)
+			}
+		})
+	}
+}

internal/storage/s3_test.go

@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"context"
 	"errors"
-	"io"
 	"net/http"
 	"net/http/httptest"
 	"slices"
@@ -23,16 +22,13 @@ import (
 // s3TestBucket is the bucket created for each in-process S3 server.
 const s3TestBucket = "test-bucket"
 
-// Credentials for the tests' S3 clients. The test servers accept any.
+// Credentials for the tests that build a storer from a config.Config. The
+// in-process S3 server accepts any.
 const (
 	s3TestAccessKeyID     = "key"
 	s3TestSecretAccessKey = "secret"
 )
 
-// s3TestRegion is the region of the S3 clients built here with
-// s3.NewClient. The test servers ignore it.
-const s3TestRegion = "us-east-1"
-
 // newS3Storer builds an s3:// backend backed by a fresh in-process
 // S3 server. It reuses the same in-memory S3 harness (gofakes3 + s3mem
 // over httptest) that internal/s3 and the not-found regression test use,
@@ -59,7 +55,7 @@ func newS3Storer(t *testing.T) storage.Storer {
 		Bucket:          s3TestBucket,
 		AccessKeyID:     "test",
 		SecretAccessKey: "test",
-		Region:          s3TestRegion,
+		Region:          "us-east-1",
 	})
 	if err != nil {
 		t.Fatalf("new client: %v", err)
@@ -97,68 +93,6 @@ func TestS3StorerMissingKeyMapsToErrNotFound(t *testing.T) {
 	}
 }
 
-// TestS3ListingCutShortIsAnError checks that List and ListStream fail on a
-// listing page marked truncated that carries no continuation token, as a
-// server answering V2 listings V1-style sends it. Taken as the last page,
-// it would hide every later key, and prune would treat the blobs of the
-// snapshots listed there as unreferenced.
-func TestS3ListingCutShortIsAnError(t *testing.T) {
-	t.Parallel()
-
-	const (
-		prefix        = "metadata/"
-		truncatedPage = `<?xml version="1.0" encoding="UTF-8"?>
-<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
-  <Name>test-bucket</Name>
-  <Prefix>metadata/</Prefix>
-  <KeyCount>1</KeyCount>
-  <MaxKeys>1</MaxKeys>
-  <IsTruncated>true</IsTruncated>
-  <Contents>
-    <Key>metadata/snap/manifest.json.zst</Key>
-    <Size>8</Size>
-  </Contents>
-</ListBucketResult>`
-	)
-
-	srv := httptest.NewServer(http.HandlerFunc(
-		func(w http.ResponseWriter, _ *http.Request) {
-			_, _ = io.WriteString(w, truncatedPage)
-		}))
-	t.Cleanup(srv.Close)
-
-	client, err := s3.NewClient(context.Background(), s3.Config{
-		Endpoint:        srv.URL,
-		Bucket:          s3TestBucket,
-		AccessKeyID:     s3TestAccessKeyID,
-		SecretAccessKey: s3TestSecretAccessKey,
-		Region:          s3TestRegion,
-	})
-	if err != nil {
-		t.Fatalf("new client: %v", err)
-	}
-
-	storer := storage.NewS3Storer(client)
-	ctx := context.Background()
-
-	keys, err := storer.List(ctx, prefix)
-	if !errors.Is(err, s3.ErrNoContinuationToken) {
-		t.Errorf("List = %q, %v; want ErrNoContinuationToken", keys, err)
-	}
-
-	var streamErr error
-
-	for obj := range storer.ListStream(ctx, prefix) {
-		if obj.Err != nil {
-			streamErr = obj.Err
-		}
-	}
-
-	if !errors.Is(streamErr, s3.ErrNoContinuationToken) {
-		t.Errorf("ListStream error = %v, want ErrNoContinuationToken", streamErr)
-	}
-}
-
 // TestS3URLPrefixKeyLayout pins the bucket keys an s3:// URL reads and
 // writes: the README's remote storage layout, with the prefix joined to
 // each key by one "/". s3://b/p and s3://b/p/ must be the same

internal/vaultik/info.go

@@ -167,10 +167,6 @@ const (
 	// metadataKeyParts is the minimum "/"-separated segment count of a
 	// metadata object key (metadata/<snapshot-id>/<filename>).
 	metadataKeyParts = 3
-
-	// blobKeyParts is the minimum "/"-separated segment count of a blob
-	// object key (blobs/<aa>/<bb>/<hash>).
-	blobKeyParts = 4
 )
 
 // SnapshotMetadataInfo contains information about a single snapshot's metadata
@@ -454,23 +450,16 @@ func (v *Vaultik) scanRemoteBlobStorage(
 		v.stdoutf("Scanning blobs...\n")
 	}
 
-	blobCh := v.Storage.ListStream(v.ctx, "blobs/")
-	allBlobs := make(map[string]int64)
+	// Use prune's listing, so a file under blobs/ that prune would never
+	// delete is not counted as a blob, and so not as an orphaned one.
+	allBlobs, err := v.listAllRemoteBlobs()
+	if err != nil {
+		return err
+	}
 
-	for obj := range blobCh {
-		if obj.Err != nil {
-			return fmt.Errorf("listing blobs: %w", obj.Err)
-		}
-
-		parts := strings.Split(obj.Key, "/")
-		if len(parts) < blobKeyParts {
-			continue
-		}
-
-		hash := parts[3]
-		allBlobs[hash] = obj.Size
+	for _, size := range allBlobs {
 		result.TotalBlobCount++
-		result.TotalBlobSize += obj.Size
+		result.TotalBlobSize += size
 	}
 
 	// A blob named only by a manifest that could not be read, or by one

internal/vaultik/prune.go

@@ -23,6 +23,10 @@ var errNukeRequiresForce = errors.New(
 // metadata.
 const metadataDirName = "metadata"
 
+// blobKeyParts is the "/"-separated segment count of a blob object key
+// (blobs/<aa>/<bb>/<hash>).
+const blobKeyParts = 4
+
 // NukeRemote deletes every snapshot's metadata and every blob from remote
 // storage, along with any object an upload cut off part-way left under a
 // temporary `.partial` name. After this returns successfully the bucket

internal/vaultik/remote_info_test.go

@@ -185,6 +185,39 @@ func TestRemoteInfo_SkipsNonConformingMetadataName(t *testing.T) {
 	assert.NotContains(t, doc, "unreadable_manifests")
 }
 
+// TestRemoteInfo_DoesNotCountNonBlobFileUnderBlobs checks that a file
+// under blobs/ whose name is not a blob hash is not counted as a blob.
+// prune never deletes such a file, so counting it as an orphaned blob
+// would advise a prune that removes nothing.
+func TestRemoteInfo_DoesNotCountNonBlobFileUnderBlobs(t *testing.T) {
+	log.Initialize(log.Config{})
+	t.Parallel()
+
+	env := newListEnv(t)
+	env.addRemote(t, listRemoteID, time.Date(2026, 3, 2, 0, 0, 0, 0, time.UTC))
+	addBlob(t, env.store.testStorer, testBlobHashA)
+	require.NoError(t, env.store.Put(context.Background(),
+		"blobs/"+testBlobHashA[:2]+"/"+testBlobHashA[2:4]+"/.DS_Store",
+		bytes.NewReader([]byte("junk"))))
+
+	require.NoError(t, env.v.RemoteInfo(false))
+
+	text := env.stdout.String()
+	assert.Contains(t, text, "Total blobs on remote:      1 (")
+	assert.Contains(t, text, "Orphaned (unreferenced):    0 (")
+	assert.NotContains(t, text, "vaultik prune")
+
+	env.stdout.Reset()
+	require.NoError(t, env.v.RemoteInfo(true))
+
+	var doc map[string]any
+
+	require.NoError(t, json.Unmarshal(env.stdout.Bytes(), &doc))
+	assert.InDelta(t, 1, doc["total_blob_count"], 0)
+	assert.InDelta(t, 0, doc["orphaned_blob_count"], 0)
+	assert.InDelta(t, 0, doc["orphaned_blob_size"], 0)
+}
+
 // TestRemoteInfo_DirectoryWithoutManifestLeavesOrphansKnown checks that
 // a directory under metadata/ holding no manifest.json.zst, such as one
 // left by a backup interrupted before its manifest upload, leaves the