Report blob counts from snapshot verify --deep --json (closes #314)

Deep verify set verified only on success and never set missing, so a
failed --deep --json run reported zero for both. It now counts each blob
whose contents pass as it goes, and the existence check runs through
every blob, counting the missing and wrong-size ones as shallow verify
does, instead of stopping at the first; it fails if it found any.

Deep verify still stops at the first damaged blob and reads no blob
contents while any blob is missing or the wrong size, so verified is 0
there where shallow mode counts the present blobs.

Each missing or wrong-size blob is logged as a warning with its hash;
the error now gives counts instead of naming one blob.

Model: opus-5-5

Dockerfile

@@ -5,16 +5,7 @@
 FROM golangci/golangci-lint:v2.14.0@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a92406b2fd3898faf0bf98f AS lint
 WORKDIR /src
 COPY go.mod go.sum ./
-# Without the time limit a stalled connection to the module proxy holds
-# this step, and every build on the host sharing it, until the kernel
-# drops the connection, which can take an hour. The download normally
-# takes under half a minute.
-RUN for attempt in 1 2 3; do \
-        timeout 180 go mod download && exit 0; \
-        echo "go mod download attempt $attempt failed or ran over 180s" >&2; \
-    done; \
-    echo "module download failed: giving up after 3 attempts" >&2; \
-    exit 1
+RUN go mod download
 COPY . .
 # `golangci-lint run` silently ignores an unknown top-level key in
 # .golangci.yml, such as a misspelt `linters:`; `config verify` fails on it.
@@ -27,13 +18,7 @@ RUN golangci-lint run --config .golangci.yml ./...
 FROM golang:1.26.1@sha256:cd78d88e00afadbedd272f977d375a6247455f3a4b1178f8ae8bbcb201743a8a AS test
 WORKDIR /src
 COPY go.mod go.sum ./
-# Limited and retried as in the lint phase above.
-RUN for attempt in 1 2 3; do \
-        timeout 180 go mod download && exit 0; \
-        echo "go mod download attempt $attempt failed or ran over 180s" >&2; \
-    done; \
-    echo "module download failed: giving up after 3 attempts" >&2; \
-    exit 1
+RUN go mod download
 COPY . .
 RUN go test -timeout 90s -race -cover ./... || \
     { echo "--- Rerunning with -v for details ---"; \
@@ -51,13 +36,7 @@ RUN apk add --no-cache git
 RUN git config --system --add safe.directory /src
 WORKDIR /src
 COPY go.mod go.sum ./
-# Limited and retried as in the lint phase above.
-RUN for attempt in 1 2 3; do \
-        timeout 180 go mod download && exit 0; \
-        echo "go mod download attempt $attempt failed or ran over 180s" >&2; \
-    done; \
-    echo "module download failed: giving up after 3 attempts" >&2; \
-    exit 1
+RUN go mod download
 COPY . .
 
 # The VERSION build arg when one is given, otherwise

cmd/vaultik/dockermoddownload_test.go

@@ -1,42 +0,0 @@
-package main_test
-
-import (
-	"strings"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-// TestDockerfileBoundsEveryModuleDownload fails unless every
-// `go mod download` in the Dockerfile runs under a time limit, is tried
-// a fixed number of times, and then fails the build with a message that
-// names the module download (issue #351). Like the version tests, this
-// parses the Dockerfile; that a stalled download really is cut off and
-// retried was checked by hand.
-func TestDockerfileBoundsEveryModuleDownload(t *testing.T) {
-	t.Parallel()
-
-	var downloads []string
-
-	for _, instruction := range instructions(t, productDockerfile) {
-		if strings.Contains(instruction, "go mod download") {
-			downloads = append(downloads, instruction)
-		}
-	}
-
-	require.NotEmpty(t, downloads, "%s must download modules",
-		productDockerfile)
-
-	for _, download := range downloads {
-		assert.Contains(t, download, "timeout 180 go mod download",
-			"%s: a module download must have a time limit: %s",
-			productDockerfile, download)
-		assert.Contains(t, download, "for attempt in 1 2 3",
-			"%s: a module download must be retried: %s",
-			productDockerfile, download)
-		assert.Contains(t, download, "module download failed",
-			"%s: a module download that never succeeds must say so: %s",
-			productDockerfile, download)
-	}
-}

internal/cli/config.go

@@ -251,6 +251,12 @@ func NewConfigCommand() *cobra.Command {
 		Use:   "config",
 		Short: "Manage the configuration file",
 		Long:  "Commands for creating, editing, and querying the vaultik config file.",
+		// cobra checks Args only on a command that has a Run: without
+		// one, a mistyped subcommand printed help and exited 0.
+		Args: cobra.NoArgs,
+		Run: func(cmd *cobra.Command, _ []string) {
+			_ = cmd.Help()
+		},
 	}
 
 	cmd.AddCommand(newConfigInitCommand())

internal/cli/database.go

@@ -15,6 +15,12 @@ func NewDatabaseCommand() *cobra.Command {
 		Use:   "database",
 		Short: "Manage the local state database",
 		Long:  `Commands for managing the local SQLite state database.`,
+		// cobra checks Args only on a command that has a Run: without
+		// one, a mistyped subcommand printed help and exited 0.
+		Args: cobra.NoArgs,
+		Run: func(cmd *cobra.Command, _ []string) {
+			_ = cmd.Help()
+		},
 	}
 
 	cmd.AddCommand(

internal/cli/entry_unknown_subcommand_test.go

@@ -0,0 +1,43 @@
+package cli //nolint:testpackage // shares runEntry and the command-word constants
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+// TestEntryGroupRejectsUnknownSubcommand runs each group command with a
+// mistyped subcommand, as a crontab line might carry one. It must fail
+// with exit 1 and name the word, as `vaultik bogus` does at the top
+// level, instead of printing the group's help and exiting 0 having done
+// nothing. The group name alone still prints its help.
+//
+//nolint:paralleltest // replaces os.Args, os.Stdout and os.Stderr
+func TestEntryGroupRejectsUnknownSubcommand(t *testing.T) {
+	for _, testCase := range []struct {
+		group    string
+		mistyped string
+	}{
+		{group: cmdSnapshot, mistyped: "craete"},
+		{group: "config", mistyped: "inti"},
+		{group: cmdRemote, mistyped: "ifno"},
+		{group: "database", mistyped: "delte"},
+		{group: "completion", mistyped: "nosuchshell"},
+	} {
+		t.Run(testCase.group, func(t *testing.T) {
+			code, stdout, stderr := runEntry(t,
+				testCase.group, testCase.mistyped)
+
+			assert.Equal(t, 1, code)
+			assert.Empty(t, stdout)
+			assert.Contains(t, stderr,
+				`unknown command "`+testCase.mistyped+`"`)
+
+			code, stdout, _ = runEntry(t, testCase.group)
+
+			assert.Equal(t, 0, code)
+			assert.Contains(t, stdout,
+				programName+" "+testCase.group+" [command]")
+		})
+	}
+}

internal/cli/remote.go

@@ -18,6 +18,12 @@ func NewRemoteCommand() *cobra.Command {
 		Use:   "remote",
 		Short: "Remote storage management commands",
 		Long:  "Commands for inspecting and managing remote storage",
+		// cobra checks Args only on a command that has a Run: without
+		// one, a mistyped subcommand printed help and exited 0.
+		Args: cobra.NoArgs,
+		Run: func(cmd *cobra.Command, _ []string) {
+			_ = cmd.Help()
+		},
 	}
 
 	// Add subcommands

internal/cli/root.go

@@ -74,6 +74,21 @@ on the source system.`,
 		NewDatabaseCommand(),
 	)
 
+	// cobra adds the completion command only when Execute runs, and with
+	// no Run, so it skipped its own cobra.NoArgs and a mistyped shell name
+	// printed help and exited 0. Adding it here lets it have a Run. Its
+	// scripts go to os.Stdout as it is now: a later SetOut does not
+	// redirect them.
+	cmd.InitDefaultCompletionCmd()
+
+	for _, sub := range cmd.Commands() {
+		if sub.Name() == "completion" {
+			sub.Run = func(completion *cobra.Command, _ []string) {
+				_ = completion.Help()
+			}
+		}
+	}
+
 	return cmd
 }
 

internal/cli/snapshot.go

@@ -41,6 +41,12 @@ func NewSnapshotCommand() *cobra.Command {
 		Use:   "snapshot",
 		Short: "Snapshot management commands",
 		Long:  "Commands for creating, listing, and managing snapshots",
+		// cobra checks Args only on a command that has a Run: without
+		// one, a mistyped subcommand printed help and exited 0.
+		Args: cobra.NoArgs,
+		Run: func(cmd *cobra.Command, _ []string) {
+			_ = cmd.Help()
+		},
 	}
 
 	// Add subcommands

internal/snapshot/exclude_test.go

@@ -531,3 +531,50 @@ func TestExcludePatterns_UnanchoredPatternFile(t *testing.T) {
 	require.Equal(t, 1, result.FilesScanned,
 		"Unanchored pattern for file should match anywhere")
 }
+
+func TestExcludePatterns_DoubleStarDirectoryAtEveryDepth(t *testing.T) {
+	t.Parallel()
+
+	// As in gitignore, a leading **/ also matches directly under the source dir
+	fs := setupAnchoredTestFS(t)
+
+	scanner, repos, cleanup := createTestScanner(t, fs, []string{"**/projectname"})
+	defer cleanup()
+
+	ctx := context.Background()
+	createSnapshotRecord(ctx, t, repos, "test-snapshot")
+
+	result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
+	require.NoError(t, err)
+
+	// **/projectname should exclude BOTH:
+	// - /backup/projectname/file.txt
+	// - /backup/otherproject/projectname/file.txt
+	// Total files: 4, excluded: 2
+	require.Equal(t, 2, result.FilesScanned,
+		"**/ directory pattern should match at the top level and nested")
+}
+
+func TestExcludePatterns_DoubleStarFileAtEveryDepth(t *testing.T) {
+	t.Parallel()
+
+	// As in gitignore, a leading **/ also matches directly under the source dir
+	fs := setupAnchoredTestFS(t)
+
+	scanner, repos, cleanup := createTestScanner(t, fs, []string{"**/*.txt"})
+	defer cleanup()
+
+	ctx := context.Background()
+	createSnapshotRecord(ctx, t, repos, "test-snapshot")
+
+	result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
+	require.NoError(t, err)
+
+	// **/*.txt should exclude ALL .txt files:
+	// - /backup/file.txt
+	// - /backup/projectname/file.txt
+	// - /backup/otherproject/projectname/file.txt
+	// Total files: 4, excluded: 3
+	require.Equal(t, 1, result.FilesScanned,
+		"**/ file pattern should match at the top level and nested")
+}

internal/snapshot/scanner.go

@@ -2018,6 +2018,14 @@ func compileExcludePatterns(patterns []string) []compiledPattern {
 		// Remove trailing slash if present (directory indicator)
 		pattern = strings.TrimSuffix(pattern, "/")
 
+		// A leading "**/" demands a "/" before the name and so misses an
+		// entry directly under the source path. An unanchored pattern
+		// already matches at every depth, so drop the prefix: as in
+		// gitignore, "**/x" then also matches a top-level "x".
+		if !anchored {
+			pattern = strings.TrimPrefix(pattern, "**/")
+		}
+
 		// Compile the glob pattern
 		// For patterns without path separators, we need to match them as components
 		// e.g., ".git" should match ".git" anywhere in the path

internal/vaultik/deep_verify_test.go

@@ -1,15 +1,20 @@
 package vaultik_test
 
 import (
+	"bytes"
 	"context"
+	"crypto/rand"
 	"io"
 	"os"
 	"path/filepath"
+	"slices"
 	"testing"
 
 	"github.com/spf13/afero"
 	"github.com/stretchr/testify/require"
+	"sneak.berlin/go/vaultik/internal/config"
 	"sneak.berlin/go/vaultik/internal/log"
+	"sneak.berlin/go/vaultik/internal/storage"
 	"sneak.berlin/go/vaultik/internal/ui"
 	"sneak.berlin/go/vaultik/internal/vaultik"
 )
@@ -77,6 +82,116 @@ func TestDeepVerifyAcceptsHealthyAndRejectsCorruptBlob(t *testing.T) {
 		"deep verify should fail on a corrupted blob")
 }
 
+// TestDeepVerifyJSONCountsBlobs checks the verified and missing counts
+// that deep verify reports under --json: on a healthy snapshot, with a
+// damaged blob, and with missing blobs. Deep verify stops at the first
+// damaged blob, so the blobs read before it are the ones verified; when
+// blobs are missing it reads no blob contents, but counts every missing
+// blob.
+func TestDeepVerifyJSONCountsBlobs(t *testing.T) {
+	log.Initialize(log.Config{})
+	t.Parallel()
+
+	fs := afero.NewOsFs()
+	tempDir := t.TempDir()
+
+	dataDir := filepath.Join(tempDir, "source")
+	storeDir := filepath.Join(tempDir, "remote")
+	dbPath := filepath.Join(tempDir, "index.sqlite")
+
+	chunkSize := int64(32 * 1024)
+	maxBlobSize := int64(128 * 1024)
+
+	// Random data does not compress or deduplicate, so it fills several
+	// blobs.
+	data := make([]byte, maxBlobSize*4)
+	_, err := rand.Read(data)
+	require.NoError(t, err)
+	require.NoError(t, fs.MkdirAll(dataDir, 0o755))
+	require.NoError(t, afero.WriteFile(fs,
+		filepath.Join(dataDir, "data.bin"), data, 0o644))
+
+	ctx := context.Background()
+
+	cfg, storer, snapshotID := runFileStorageBackup(
+		ctx, t, fs, dataDir, storeDir, dbPath, chunkSize, maxBlobSize)
+
+	healthy, err := deepVerifyJSON(ctx, t, cfg, storer, snapshotID)
+	require.NoError(t, err)
+	require.Equal(t, "ok", healthy.Status)
+	require.GreaterOrEqual(t, healthy.BlobCount, 3)
+	require.Equal(t, healthy.BlobCount, healthy.Verified)
+	require.Zero(t, healthy.Missing)
+
+	// Deep verify reads blobs in hash order, which is the sorted order of
+	// their storage keys.
+	keys, err := storer.List(ctx, "blobs/")
+	require.NoError(t, err)
+	slices.Sort(keys)
+	require.Len(t, keys, healthy.BlobCount)
+
+	// Damage the last blob read, keeping its length, so every blob before
+	// it is read and passes first.
+	lastPath := filepath.Join(storeDir, keys[len(keys)-1])
+	blob, err := afero.ReadFile(fs, lastPath)
+	require.NoError(t, err)
+
+	blob[len(blob)/2] ^= 0xff
+	require.NoError(t, afero.WriteFile(fs, lastPath, blob, 0o600))
+
+	got, err := deepVerifyJSON(ctx, t, cfg, storer, snapshotID)
+	require.Error(t, err)
+	require.Equal(t, "failed", got.Status)
+	require.Equal(t, healthy.BlobCount-1, got.Verified)
+	require.Zero(t, got.Missing)
+
+	// Delete the first blob and the damaged last one: both must be
+	// counted as missing, not only the first one found.
+	var deletedSize int64
+
+	for _, key := range []string{keys[0], keys[len(keys)-1]} {
+		info, statErr := storer.Stat(ctx, key)
+		require.NoError(t, statErr)
+
+		deletedSize += info.Size
+
+		require.NoError(t, storer.Delete(ctx, key))
+	}
+
+	got, err = deepVerifyJSON(ctx, t, cfg, storer, snapshotID)
+	require.Error(t, err)
+	require.Equal(t, "failed", got.Status)
+	require.Equal(t, 2, got.Missing)
+	require.Equal(t, deletedSize, got.MissingSize)
+	require.Zero(t, got.Verified)
+}
+
+// deepVerifyJSON runs deep verify with --json on snapshotID and returns
+// the decoded result and the error RunDeepVerify returned.
+func deepVerifyJSON(
+	ctx context.Context, t *testing.T,
+	cfg *config.Config, storer storage.Storer, snapshotID string,
+) (vaultik.VerifyResult, error) {
+	t.Helper()
+
+	var out bytes.Buffer
+
+	v := &vaultik.Vaultik{
+		Config:  cfg,
+		Storage: storer,
+		Fs:      afero.NewOsFs(),
+		Stdout:  &out,
+		Stderr:  io.Discard,
+		UI:      ui.NewWithColor(io.Discard, false),
+	}
+	v.SetContext(ctx)
+
+	err := v.RunDeepVerify(snapshotID,
+		&vaultik.VerifyOptions{Deep: true, JSON: true})
+
+	return decodeVerifyResult(t, out.Bytes()), err
+}
+
 // corruptOneBlob flips a middle byte of the first blob file found under
 // blobsDir, leaving the file length unchanged.
 func corruptOneBlob(t *testing.T, fs afero.Fs, blobsDir string) {

internal/vaultik/verify.go

@@ -32,7 +32,9 @@ var (
 	errManifestExtraBlob   = errors.New("manifest contains blob not in database")
 	errManifestMissingBlob = errors.New(
 		"manifest omits blob present in database")
-	errBlobSizeMismatch = errors.New("blob size mismatch")
+	errBlobSizeMismatch  = errors.New("blob size mismatch")
+	errBlobsNotInStorage = errors.New(
+		"blobs missing from storage or stored with the wrong size")
 )
 
 // verifyStatusFailed is the JSON status value for a failed verification.
@@ -135,7 +137,6 @@ func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error {
 	}
 
 	result.Status = "ok"
-	result.Verified = len(dbBlobs)
 
 	if opts.JSON {
 		return v.outputVerifyJSON(result)
@@ -290,7 +291,7 @@ func (v *Vaultik) runVerificationSteps(
 		v.stdoutf("Checking blob existence in remote storage...\n")
 	}
 
-	err = v.verifyBlobExistenceFromDB(dbBlobs)
+	err = v.verifyBlobExistenceFromDB(dbBlobs, result)
 	if err != nil {
 		return v.deepVerifyFailure(result, opts, err.Error(), err)
 	}
@@ -301,7 +302,8 @@ func (v *Vaultik) runVerificationSteps(
 			len(dbBlobs), ubytes(totalSize))
 	}
 
-	err = v.performDeepVerificationFromDB(dbBlobs, tdb.db.Conn(), opts, identities)
+	err = v.performDeepVerificationFromDB(
+		dbBlobs, tdb.db.Conn(), opts, result, identities)
 	if err != nil {
 		return v.deepVerifyFailure(result, opts, err.Error(), err)
 	}
@@ -653,8 +655,13 @@ func (v *Vaultik) verifyManifestAgainstDatabase(
 	return nil
 }
 
-// verifyBlobExistenceFromDB checks that all blobs from database exist in S3
-func (v *Vaultik) verifyBlobExistenceFromDB(blobs []snapshot.BlobInfo) error {
+// verifyBlobExistenceFromDB checks that every blob from the database is
+// in storage with the size the database records. It checks them all,
+// counting the missing and wrong-size blobs in result, and fails if it
+// found any.
+func (v *Vaultik) verifyBlobExistenceFromDB(
+	blobs []snapshot.BlobInfo, result *VerifyResult,
+) error {
 	log.Info("Verifying blob existence in S3", "blob_count", len(blobs))
 
 	for i, blob := range blobs {
@@ -667,17 +674,21 @@ func (v *Vaultik) verifyBlobExistenceFromDB(blobs []snapshot.BlobInfo) error {
 		// Construct blob path
 		blobPath := fmt.Sprintf("blobs/%s/%s/%s", blob.Hash[:2], blob.Hash[2:4], blob.Hash)
 
-		// Check blob exists
 		stat, err := v.Storage.Stat(v.ctx, blobPath)
-		if err != nil {
-			return fmt.Errorf("blob %s missing from storage: %w", blob.Hash, err)
-		}
+		switch {
+		case err != nil:
+			log.Warn("Blob missing from storage",
+				"hash", blob.Hash, "error", err)
 
-		// Verify size matches
-		if stat.Size != blob.CompressedSize {
-			return fmt.Errorf(
-				"%w: blob %s: S3 has %d bytes, database has %d bytes",
-				errBlobSizeMismatch, blob.Hash, stat.Size, blob.CompressedSize)
+			result.Missing++
+			result.MissingSize += blob.CompressedSize
+		case stat.Size != blob.CompressedSize:
+			log.Warn("Blob has the wrong size in storage",
+				"hash", blob.Hash,
+				"storage_size", stat.Size,
+				"database_size", blob.CompressedSize)
+
+			result.Mismatched++
 		}
 
 		// Progress update every 100 blobs
@@ -691,16 +702,22 @@ func (v *Vaultik) verifyBlobExistenceFromDB(blobs []snapshot.BlobInfo) error {
 		}
 	}
 
+	if result.Missing > 0 || result.Mismatched > 0 {
+		return fmt.Errorf("%w: %d missing, %d with the wrong size",
+			errBlobsNotInStorage, result.Missing, result.Mismatched)
+	}
+
 	log.Info("✓ All blobs exist in storage")
 
 	return nil
 }
 
 // performDeepVerificationFromDB downloads and verifies the content of
-// each blob using the database as source.
+// each blob using the database as source, stopping at the first blob that
+// fails. Each blob that passes is counted in result.Verified.
 func (v *Vaultik) performDeepVerificationFromDB(
 	blobs []snapshot.BlobInfo, db *sql.DB, opts *VerifyOptions,
-	identities []age.Identity,
+	result *VerifyResult, identities []age.Identity,
 ) error {
 	// Calculate total bytes for ETA
 	var totalBytesExpected int64
@@ -723,6 +740,7 @@ func (v *Vaultik) performDeepVerificationFromDB(
 			return fmt.Errorf("blob %s verification failed: %w", blobInfo.Hash, err)
 		}
 
+		result.Verified++
 		bytesProcessed += blobInfo.CompressedSize
 		elapsed := time.Since(startTime)
 		remaining := len(blobs) - (i + 1)