Limit and retry the module download in the Docker build (closes #351)

Each `go mod download` in the Dockerfile now runs under a five-minute
time limit and is tried up to three times. Without a limit, a stalled
connection to the module proxy held the step, and every build on the
host sharing it, until the kernel dropped the connection. Five minutes
is well above the three minutes a working download can take on a busy
build host. When all three attempts fail, the step prints "module
download failed" and exits 1, so the build stops there instead of
failing as a test or lint failure. A test checks every download step
for the limit, the retries, the message and the final `exit 1`.

Model: opus-5-5

internal/vaultik/destination_validation_test.go

@@ -161,13 +161,14 @@ func TestVerifyManifestBlobsRejectsShortHash(t *testing.T) {
 		Blobs: []snapshot.BlobInfo{{Hash: "abc", CompressedSize: 1}},
 	}
 
-	verified, missing, mismatched, missingSize, err :=
+	verified, missing, mismatched, missingSize, unchecked, err :=
 		v.verifyManifestBlobs(manifest, &VerifyOptions{JSON: true})
 	require.ErrorIs(t, err, errInvalidBlobHash)
 	require.Zero(t, verified)
 	require.Zero(t, missing)
 	require.Zero(t, mismatched)
 	require.Zero(t, missingSize)
+	require.Empty(t, unchecked)
 }
 
 // TestVerifyBlobChunksRejectsNegativeLength proves a blob_chunks row with a

internal/vaultik/shallow_verify_test.go

@@ -4,15 +4,20 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"errors"
+	"fmt"
 	"io"
 	"os"
+	"path"
 	"path/filepath"
+	"slices"
 	"testing"
 
 	"github.com/spf13/afero"
 	"github.com/stretchr/testify/require"
 	"sneak.berlin/go/vaultik/internal/log"
 	"sneak.berlin/go/vaultik/internal/snapshot"
+	"sneak.berlin/go/vaultik/internal/storage"
 	"sneak.berlin/go/vaultik/internal/ui"
 	"sneak.berlin/go/vaultik/internal/vaultik"
 )
@@ -151,6 +156,224 @@ func TestShallowVerifyDetectsMissingDatabase(t *testing.T) {
 		"shallow verify must fail when db.zst.age is absent")
 }
 
+// errStatDenied stands in for any storage error other than not-found: a
+// permission error, an S3 403, a dropped connection.
+var errStatDenied = errors.New("injected stat failure")
+
+// statFailingStorer fails Stat with errStatDenied for the keys in
+// failKeys and passes every other call to the wrapped Storer.
+type statFailingStorer struct {
+	storage.Storer
+
+	failKeys []string
+}
+
+func (s statFailingStorer) Stat(
+	ctx context.Context, key string,
+) (*storage.ObjectInfo, error) {
+	if slices.Contains(s.failKeys, key) {
+		return nil, fmt.Errorf("stat %q: %w", key, errStatDenied)
+	}
+
+	return s.Storer.Stat(ctx, key)
+}
+
+// TestShallowVerifyReportsStatErrorsApartFromMissing backs up a real
+// snapshot, deletes one blob, and makes storage fail Stat with an error
+// other than not-found for a second blob and for the encrypted database.
+// Only the deleted blob may be counted as missing. The second blob and
+// the database must be reported as could not be checked, with the error
+// storage returned, and verify must fail, in the text output and under
+// --json.
+func TestShallowVerifyReportsStatErrorsApartFromMissing(t *testing.T) {
+	log.Initialize(log.Config{})
+	t.Parallel()
+
+	fs := afero.NewOsFs()
+	tempDir := t.TempDir()
+
+	dataDir := filepath.Join(tempDir, "source")
+	storeDir := filepath.Join(tempDir, "remote")
+	dbPath := filepath.Join(tempDir, "index.sqlite")
+
+	chunkSize := int64(32 * 1024)
+	maxBlobSize := int64(128 * 1024)
+
+	// Random content, so the snapshot spans several blobs.
+	require.NoError(t, fs.MkdirAll(dataDir, 0o755))
+	appendRandomBytes(t, fs, map[string][]byte{},
+		filepath.Join(dataDir, "data.bin"), maxBlobSize*4)
+
+	ctx := context.Background()
+
+	cfg, storer, snapshotID := runFileStorageBackup(
+		ctx, t, fs, dataDir, storeDir, dbPath, chunkSize, maxBlobSize)
+
+	blobKeys := storedBlobKeys(t, storeDir)
+	require.GreaterOrEqual(t, len(blobKeys), 2, "expected at least two blobs")
+
+	missingKey, uncheckedKey := blobKeys[0], blobKeys[1]
+	missingHash, uncheckedHash := path.Base(missingKey), path.Base(uncheckedKey)
+
+	require.NoError(t, os.Remove(filepath.Join(storeDir, missingKey)))
+
+	dbKey := "metadata/" + snapshot.RemoteSnapshotKey(snapshotID) + "/db.zst.age"
+	failing := statFailingStorer{
+		Storer:   storer,
+		failKeys: []string{uncheckedKey, dbKey},
+	}
+
+	newVerifier := func(out io.Writer) *vaultik.Vaultik {
+		v := &vaultik.Vaultik{
+			Config:  cfg,
+			Storage: failing,
+			Fs:      fs,
+			Stdout:  out,
+			Stderr:  io.Discard,
+			UI:      ui.NewWithColor(io.Discard, false),
+		}
+		v.SetContext(ctx)
+
+		return v
+	}
+
+	var out bytes.Buffer
+
+	err := newVerifier(&out).VerifySnapshotWithOptions(
+		snapshotID, &vaultik.VerifyOptions{JSON: true})
+	require.Error(t, err)
+
+	result := decodeVerifyResult(t, out.Bytes())
+	require.Equal(t, "failed", result.Status)
+	require.Equal(t, 1, result.Missing, "only the deleted blob is missing")
+	require.Equal(t, result.BlobCount-2, result.Verified)
+	require.Len(t, result.Unchecked, 1)
+	require.Equal(t, uncheckedHash, result.Unchecked[0].Hash)
+	require.Contains(t, result.Unchecked[0].Error, errStatDenied.Error())
+	require.False(t, result.DatabaseMissing,
+		"a database that could not be checked is not missing")
+	require.Contains(t, result.DatabaseCheckError, errStatDenied.Error())
+
+	out.Reset()
+	err = newVerifier(&out).VerifySnapshotWithOptions(
+		snapshotID, &vaultik.VerifyOptions{})
+	require.Error(t, err)
+
+	text := out.String()
+	require.Contains(t, text, "Missing: "+missingHash)
+	require.NotContains(t, text, "Missing: "+uncheckedHash)
+	require.Contains(t, text, "Could not check: "+uncheckedHash)
+	require.Contains(t, text, "Encrypted database:       could not be checked")
+	require.Contains(t, text, errStatDenied.Error())
+	require.Contains(t, text, "FAILED")
+}
+
+// TestShallowVerifyQuietFailsOnStatErrorAlone backs up a healthy snapshot,
+// then makes storage fail Stat, with an error other than not-found, for
+// one blob alone and then for the encrypted database alone. Each run must
+// fail verify, and under --quiet, which drops the report, the error
+// storage returned must still reach stderr.
+//
+//nolint:paralleltest // captureProcessStderr replaces os.Stderr
+func TestShallowVerifyQuietFailsOnStatErrorAlone(t *testing.T) {
+	log.Initialize(log.Config{})
+
+	fs := afero.NewOsFs()
+	tempDir := t.TempDir()
+
+	dataDir := filepath.Join(tempDir, "source")
+	storeDir := filepath.Join(tempDir, "remote")
+	dbPath := filepath.Join(tempDir, "index.sqlite")
+
+	chunkSize := int64(32 * 1024)
+	maxBlobSize := int64(128 * 1024)
+
+	require.NoError(t, fs.MkdirAll(dataDir, 0o755))
+	require.NoError(t, afero.WriteFile(fs,
+		filepath.Join(dataDir, "data.bin"),
+		bytesPattern("shallow-quiet-", int(maxBlobSize*2)), 0o644))
+
+	ctx := context.Background()
+
+	cfg, storer, snapshotID := runFileStorageBackup(
+		ctx, t, fs, dataDir, storeDir, dbPath, chunkSize, maxBlobSize)
+
+	newQuietVerifier := func(s storage.Storer) *vaultik.Vaultik {
+		v := &vaultik.Vaultik{
+			Config:  cfg,
+			Storage: s,
+			Fs:      fs,
+			Stdout:  io.Discard,
+			Stderr:  io.Discard,
+			UI:      ui.NewWithColor(io.Discard, false),
+		}
+		v.SetContext(ctx)
+		v.UI.SetQuiet(true)
+
+		return v
+	}
+
+	require.NoError(t,
+		newQuietVerifier(storer).VerifySnapshotWithOptions(
+			snapshotID, &vaultik.VerifyOptions{}),
+		"shallow verify should pass on a healthy snapshot")
+
+	blobKeys := storedBlobKeys(t, storeDir)
+	require.NotEmpty(t, blobKeys)
+
+	dbKey := "metadata/" + snapshot.RemoteSnapshotKey(snapshotID) + "/db.zst.age"
+
+	for _, failKey := range []string{blobKeys[0], dbKey} {
+		v := newQuietVerifier(statFailingStorer{
+			Storer:   storer,
+			failKeys: []string{failKey},
+		})
+
+		var err error
+
+		stderr := captureProcessStderr(t, func() {
+			err = v.VerifySnapshotWithOptions(
+				snapshotID, &vaultik.VerifyOptions{})
+		})
+
+		require.Error(t, err,
+			"verify must fail when %s could not be checked", failKey)
+		require.Contains(t, stderr, errStatDenied.Error(),
+			"under --quiet the error for %s must still reach stderr", failKey)
+	}
+}
+
+// storedBlobKeys returns the storage key of every blob file under
+// storeDir, sorted.
+func storedBlobKeys(t *testing.T, storeDir string) []string {
+	t.Helper()
+
+	var keys []string
+
+	err := filepath.Walk(filepath.Join(storeDir, "blobs"),
+		func(p string, info os.FileInfo, err error) error {
+			if err != nil {
+				return err
+			}
+
+			if info.IsDir() {
+				return nil
+			}
+
+			rel, err := filepath.Rel(storeDir, p)
+			if err != nil {
+				return err
+			}
+
+			keys = append(keys, filepath.ToSlash(rel))
+
+			return nil
+		})
+	require.NoError(t, err)
+
+	return keys
+}
+
 // growOneBlob appends bytes to the first blob file found under blobsDir,
 // changing its on-disk size so it no longer matches the manifest.
 func growOneBlob(t *testing.T, fs afero.Fs, blobsDir string) {

internal/vaultik/snapshot.go

@@ -14,6 +14,7 @@ import (
 
 	"sneak.berlin/go/vaultik/internal/log"
 	"sneak.berlin/go/vaultik/internal/snapshot"
+	"sneak.berlin/go/vaultik/internal/storage"
 	"sneak.berlin/go/vaultik/internal/types"
 )
 
@@ -777,18 +778,10 @@ func (v *Vaultik) VerifySnapshotWithOptions(
 		v.stdoutf("Checking blob presence and sizes...\n")
 	}
 
-	// A snapshot is only restorable if its encrypted database is present
-	// alongside the blobs. Shallow verify checks that the object exists; it
-	// does not decrypt it (that is deep verify's job).
-	dbPath := fmt.Sprintf("metadata/%s/db.zst.age", remoteKey)
+	v.checkSnapshotDatabase(remoteKey, result)
 
-	_, dbErr := v.Storage.Stat(v.ctx, dbPath)
-	if dbErr != nil {
-		result.DatabaseMissing = true
-	}
-
-	result.Verified, result.Missing, result.Mismatched, result.MissingSize, err =
-		v.verifyManifestBlobs(manifest, opts)
+	result.Verified, result.Missing, result.Mismatched, result.MissingSize,
+		result.Unchecked, err = v.verifyManifestBlobs(manifest, opts)
 	if err != nil {
 		if opts.JSON {
 			result.Status = verifyStatusFailed
@@ -825,25 +818,46 @@ func (v *Vaultik) printVerifyHeader(snapshotID string, opts *VerifyOptions) {
 	}
 }
 
+// checkSnapshotDatabase records in result whether the snapshot's
+// encrypted database is missing or could not be checked. A snapshot is
+// only restorable if its database is present alongside the blobs.
+// Shallow verify checks that the object exists; it does not decrypt it
+// (that is deep verify's job).
+func (v *Vaultik) checkSnapshotDatabase(remoteKey string, result *VerifyResult) {
+	dbPath := fmt.Sprintf("metadata/%s/db.zst.age", remoteKey)
+
+	_, err := v.Storage.Stat(v.ctx, dbPath)
+
+	switch {
+	case errors.Is(err, storage.ErrNotFound):
+		result.DatabaseMissing = true
+	case err != nil:
+		result.DatabaseCheckError = err.Error()
+	}
+}
+
 // verifyManifestBlobs checks that each blob in the manifest is present in
 // storage with the size the manifest records, returning the counts of
 // blobs that were present with the right size, absent, and present but the
-// wrong size, plus the total bytes of the absent blobs. It does not read
-// blob contents; deep verification (RunDeepVerify) does that. The size
-// comparison matches the deep path (see verifyBlobExistenceFromDB).
+// wrong size, the total bytes of the absent blobs, and the blobs it could
+// not check because storage returned an error other than not-found. It
+// does not read blob contents; deep verification (RunDeepVerify) does
+// that. The size comparison matches the deep path (see
+// verifyBlobExistenceFromDB).
 func (v *Vaultik) verifyManifestBlobs(
 	manifest *snapshot.Manifest, opts *VerifyOptions,
-) (int, int, int, int64, error) {
+) (int, int, int, int64, []UncheckedBlob, error) {
 	var (
 		verified, missing, mismatched int
 		missingSize                   int64
+		unchecked                     []UncheckedBlob
 	)
 
 	for _, blob := range manifest.Blobs {
 		// The manifest is unauthenticated, so its blob hashes are checked
 		// before being spliced into a storage path.
 		if !isBlobHash(blob.Hash) {
-			return 0, 0, 0, 0, fmt.Errorf("%w: %s",
+			return 0, 0, 0, 0, nil, fmt.Errorf("%w: %s",
 				errInvalidBlobHash, shortHash(blob.Hash))
 		}
 
@@ -852,7 +866,7 @@ func (v *Vaultik) verifyManifestBlobs(
 
 		stat, err := v.Storage.Stat(v.ctx, blobPath)
 		switch {
-		case err != nil:
+		case errors.Is(err, storage.ErrNotFound):
 			if !opts.JSON && !v.UI.Quiet() {
 				v.stdoutf("  Missing: %s (%s)\n",
 					blob.Hash, ubytes(blob.CompressedSize))
@@ -860,6 +874,13 @@ func (v *Vaultik) verifyManifestBlobs(
 
 			missing++
 			missingSize += blob.CompressedSize
+		case err != nil:
+			if !opts.JSON && !v.UI.Quiet() {
+				v.stdoutf("  Could not check: %s (%v)\n", blob.Hash, err)
+			}
+
+			unchecked = append(unchecked,
+				UncheckedBlob{Hash: blob.Hash, Error: err.Error()})
 		case stat.Size != blob.CompressedSize:
 			if !opts.JSON && !v.UI.Quiet() {
 				v.stdoutf("  Wrong size: %s (store has %s, manifest lists %s)\n",
@@ -872,7 +893,7 @@ func (v *Vaultik) verifyManifestBlobs(
 		}
 	}
 
-	return verified, missing, mismatched, missingSize, nil
+	return verified, missing, mismatched, missingSize, unchecked, nil
 }
 
 // formatVerifyResult outputs the final verification results as JSON or
@@ -893,9 +914,19 @@ func (v *Vaultik) formatVerifyResult(
 		return v.outputVerifyJSON(result)
 	}
 
-	// Under --quiet a failure is still returned, and the cli layer
-	// prints it on stderr.
-	if !v.UI.Quiet() {
+	// --quiet drops the report but not the errors storage returned, which
+	// are logged on stderr. A failure is still returned either way, and
+	// the cli layer prints it on stderr.
+	if v.UI.Quiet() {
+		for _, blob := range result.Unchecked {
+			log.Warn("Could not check blob", "blob", blob.Hash, "error", blob.Error)
+		}
+
+		if result.DatabaseCheckError != "" {
+			log.Warn("Could not check the encrypted database",
+				"snapshot", result.SnapshotID, "error", result.DatabaseCheckError)
+		}
+	} else {
 		v.printVerifySummary(result, failure)
 	}
 
@@ -921,10 +952,19 @@ func (v *Vaultik) printVerifySummary(result *VerifyResult, failure string) {
 		v.stdoutf("  Wrong size:               %d blobs\n", result.Mismatched)
 	}
 
+	if len(result.Unchecked) > 0 {
+		v.stdoutf("  Could not check:          %d blobs\n", len(result.Unchecked))
+	}
+
 	if result.DatabaseMissing {
 		v.stdoutf("  Encrypted database:       missing\n")
 	}
 
+	if result.DatabaseCheckError != "" {
+		v.stdoutf("  Encrypted database:       could not be checked (%s)\n",
+			result.DatabaseCheckError)
+	}
+
 	v.stdoutf("  Status:       ")
 
 	if failure != "" {
@@ -952,10 +992,19 @@ func shallowVerifyFailure(result *VerifyResult) string {
 			fmt.Sprintf("%d blobs have the wrong size", result.Mismatched))
 	}
 
+	if len(result.Unchecked) > 0 {
+		parts = append(parts, fmt.Sprintf("%d blobs could not be checked",
+			len(result.Unchecked)))
+	}
+
 	if result.DatabaseMissing {
 		parts = append(parts, "the encrypted database is missing")
 	}
 
+	if result.DatabaseCheckError != "" {
+		parts = append(parts, "the encrypted database could not be checked")
+	}
+
 	return strings.Join(parts, "; ")
 }
 

internal/vaultik/verify.go

@@ -57,11 +57,25 @@ type VerifyResult struct {
 	Missing     int    `json:"missing"`
 	MissingSize int64  `json:"missing_size,omitempty"`
 	Mismatched  int    `json:"mismatched,omitempty"`
+	// Unchecked lists the blobs shallow verify could not check, because
+	// storage answered with an error other than not-found, so whether
+	// they are present is unknown.
+	Unchecked []UncheckedBlob `json:"unchecked,omitempty"`
 	// DatabaseMissing is set by shallow verify when the snapshot's
 	// encrypted database (metadata/<key>/db.zst.age) is absent, which
 	// makes the snapshot unrestorable regardless of the blobs.
-	DatabaseMissing bool   `json:"database_missing,omitempty"`
-	ErrorMessage    string `json:"error,omitempty"`
+	DatabaseMissing bool `json:"database_missing,omitempty"`
+	// DatabaseCheckError is set by shallow verify to the error storage
+	// returned when the encrypted database could not be checked.
+	DatabaseCheckError string `json:"database_check_error,omitempty"`
+	ErrorMessage       string `json:"error,omitempty"`
+}
+
+// UncheckedBlob is a blob shallow verify could not check, with the error
+// storage returned for it.
+type UncheckedBlob struct {
+	Hash  string `json:"hash"`
+	Error string `json:"error"`
 }
 
 // deepVerifyFailure records a failure in the result and returns it appropriately