Compare commits

..
Author SHA1 Message Date
sneak a10d031669 Limit and retry the module download in the Docker build (closes #351)
check / check (push) Waiting to run
Each `go mod download` in the Dockerfile now runs under a 180-second
time limit and is tried up to three times. Without the limit, a stalled
connection to the module proxy held the step, and every build on the
host sharing it, until the kernel dropped the connection. When all
three attempts fail, the step prints "module download failed" and
exits 1, so the build stops there and does not look like a test or
lint failure. The loop is repeated in each of the three stages because
a Dockerfile cannot share shell code between stages. A test checks
every download step for the limit, the retries, the message and the
final `exit 1`.

Judgement call: 180 seconds and three attempts, against a normal
download of 13 to 24 seconds.
Deviation: REPO_POLICIES.md's model Go Dockerfile still shows a bare
`RUN go mod download`.

Model: opus-5-5
2026-10-13 14:31:42 +00:00
clawbot f4ab1ba5ae Exit as interrupted when Ctrl-C lands during a local index write (closes #301)
check / check (push) Waiting to run
Once the run's context is cancelled, database/sql rolls an open
transaction back itself, so the rollback in WithTx returned
sql.ErrTxDone. WithTx treated any rollback error as fatal and called
os.Exit(1): an interrupted backup printed "FATAL: failed to rollback
transaction", exited 1 instead of 130 and skipped its deferred cleanup.

WithTx now ignores sql.ErrTxDone from the rollback and returns the
body's error alone. A rollback that really fails is returned together
with the body's error instead of ending the process; on a panic it is
dropped and the panic carries on. The cancel test checks that the
returned error carries no sql.ErrTxDone; the other tests make the
rollback really fail by running ROLLBACK inside the body.

Model: opus-5-5
2026-10-13 16:28:59 +02:00
clawbot 005927a7fc Print snapshot list and purge sizes like every other command (closes #312)
check / check (push) Waiting to run
`snapshot list`, the `snapshot purge` listing and the `bytes` value in
terminal log lines divided by 1024 while labelling the result KB or MB,
so 6,329,198 bytes read 6.0 MB there and 6.3 MB in every other command.
All three now use humanize.Bytes, as the rest of the program does, and
the 1024-based helper in internal/vaultik is removed.

The size shared by the list tests is now 5,000,000 bytes, which the two
ways of counting print differently.

Judgement call: the log handler's copy of the formatter is changed too,
since the issue asks that every command print a byte count the same way.

Model: opus-5-5
2026-10-13 16:22:03 +02:00
clawbot 09a0c01c43 Stop a stalled blob read when restore or deep verify is interrupted (closes #325)
check / check (push) Waiting to run
Restore and deep verify looked for a cancel only between blobs, and the
file backend's reader ignores the context. A blob read stalled on a
slow USB stick or network share outlasted the 30-second shutdown
timeout, so the process exited before removing the decrypted snapshot
database and blob cache from the temp directory.

FetchBlob, through which both read every blob, now closes the storage
reader as soon as the context is cancelled, ending a blocked read on
any backend; Read then returns the context's error, not the closed
reader's.

Limitation: a read stuck in the kernel that closing cannot interrupt,
such as one on an unresponsive hard NFS mount, still cannot be stopped.

Model: opus-5-5
2026-10-13 15:48:00 +02:00
15 changed files with 448 additions and 362 deletions
+24 -3
View File
@@ -5,7 +5,16 @@
FROM golangci/golangci-lint:v2.14.0@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a92406b2fd3898faf0bf98f AS lint
WORKDIR /src
COPY go.mod go.sum ./
RUN go mod download
# Without the time limit a stalled connection to the module proxy holds
# this step, and every build on the host sharing it, until the kernel
# drops the connection, which can take an hour. The download normally
# takes under half a minute.
RUN for attempt in 1 2 3; do \
timeout 180 go mod download && exit 0; \
echo "go mod download attempt $attempt failed or ran over 180s" >&2; \
done; \
echo "module download failed: giving up after 3 attempts" >&2; \
exit 1
COPY . .
# `golangci-lint run` silently ignores an unknown top-level key in
# .golangci.yml, such as a misspelt `linters:`; `config verify` fails on it.
@@ -18,7 +27,13 @@ RUN golangci-lint run --config .golangci.yml ./...
FROM golang:1.26.1@sha256:cd78d88e00afadbedd272f977d375a6247455f3a4b1178f8ae8bbcb201743a8a AS test
WORKDIR /src
COPY go.mod go.sum ./
RUN go mod download
# Limited and retried as in the lint phase above.
RUN for attempt in 1 2 3; do \
timeout 180 go mod download && exit 0; \
echo "go mod download attempt $attempt failed or ran over 180s" >&2; \
done; \
echo "module download failed: giving up after 3 attempts" >&2; \
exit 1
COPY . .
RUN go test -timeout 90s -race -cover ./... || \
{ echo "--- Rerunning with -v for details ---"; \
@@ -36,7 +51,13 @@ RUN apk add --no-cache git
RUN git config --system --add safe.directory /src
WORKDIR /src
COPY go.mod go.sum ./
RUN go mod download
# Limited and retried as in the lint phase above.
RUN for attempt in 1 2 3; do \
timeout 180 go mod download && exit 0; \
echo "go mod download attempt $attempt failed or ran over 180s" >&2; \
done; \
echo "module download failed: giving up after 3 attempts" >&2; \
exit 1
COPY . .
# The VERSION build arg when one is given, otherwise
+45
View File
@@ -0,0 +1,45 @@
package main_test
import (
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// TestDockerfileBoundsEveryModuleDownload fails unless every
// `go mod download` in the Dockerfile runs under a time limit, is tried
// a fixed number of times, and then fails the build with a message that
// names the module download (issue #351). Like the version tests, this
// parses the Dockerfile; that a stalled download really is cut off and
// retried was checked by hand.
func TestDockerfileBoundsEveryModuleDownload(t *testing.T) {
t.Parallel()
var downloads []string
for _, instruction := range instructions(t, productDockerfile) {
if strings.Contains(instruction, "go mod download") {
downloads = append(downloads, instruction)
}
}
require.NotEmpty(t, downloads, "%s must download modules",
productDockerfile)
for _, download := range downloads {
assert.Contains(t, download, "timeout 180 go mod download",
"%s: a module download must have a time limit: %s",
productDockerfile, download)
assert.Contains(t, download, "for attempt in 1 2 3",
"%s: a module download must be retried: %s",
productDockerfile, download)
assert.Contains(t, download, "module download failed",
"%s: a module download that never succeeds must say so: %s",
productDockerfile, download)
assert.True(t, strings.HasSuffix(download, "exit 1"),
"%s: a module download that never succeeds must fail the"+
" build: %s", productDockerfile, download)
}
}
+12 -9
View File
@@ -3,6 +3,7 @@ package database
import (
"context"
"database/sql"
"errors"
"fmt"
)
@@ -60,22 +61,24 @@ func (r *Repositories) WithTx(ctx context.Context, fn TxFunc) error {
defer func() {
if p := recover(); p != nil {
rollbackErr := tx.Rollback()
if rollbackErr != nil {
Fatalf("failed to rollback transaction: %v", rollbackErr)
}
// The panic is the failure to report, so a rollback error
// is dropped.
_ = tx.Rollback()
panic(p)
} else if err != nil {
rollbackErr := tx.Rollback()
if rollbackErr != nil {
Fatalf("failed to rollback transaction: %v", rollbackErr)
}
}
}()
err = fn(ctx, tx)
if err != nil {
// Once ctx is cancelled, database/sql rolls the transaction
// back itself and Rollback returns sql.ErrTxDone.
rollbackErr := tx.Rollback()
if rollbackErr != nil && !errors.Is(rollbackErr, sql.ErrTxDone) {
return fmt.Errorf("%w; rolling back transaction: %w",
err, rollbackErr)
}
return err
}
+120
View File
@@ -4,6 +4,7 @@ import (
"context"
"database/sql"
"errors"
"strings"
"testing"
"time"
@@ -235,6 +236,125 @@ func TestRepositoriesTransactionRollback(t *testing.T) {
}
}
// When the context is cancelled during the body, database/sql rolls the
// transaction back itself, so WithTx's own rollback finds it already
// done. WithTx must return the body's error alone, without that rollback.
func TestRepositoriesTransactionContextCancelled(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
repos := database.NewRepositories(db)
err := repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
file := &database.File{
Path: "/test/cancelled_file.txt",
MTime: time.Now().Truncate(time.Second),
Size: 1024,
Mode: 0644,
UID: 1000,
GID: 1000,
}
err := repos.Files.Create(ctx, tx, file)
if err != nil {
return err
}
cancel()
// database/sql rolls back from a goroutine of its own; wait
// until it has, so the test does not depend on that race.
for {
_, err = tx.ExecContext(context.WithoutCancel(ctx), "SELECT 1")
if errors.Is(err, sql.ErrTxDone) {
return ctx.Err()
}
time.Sleep(time.Millisecond)
}
})
if !errors.Is(err, context.Canceled) {
t.Fatalf("expected context.Canceled, got: %v", err)
}
if errors.Is(err, sql.ErrTxDone) {
t.Fatalf("the already-done rollback was reported: %v", err)
}
file, err := repos.Files.GetByPath(
context.Background(), "/test/cancelled_file.txt",
)
if err != nil {
t.Fatalf("error checking for file: %v", err)
}
if file != nil {
t.Error("file should not exist after the cancelled transaction")
}
}
// A body that ends the transaction itself with ROLLBACK makes WithTx's
// own rollback fail. WithTx must return that failure together with the
// body's error.
func TestRepositoriesTransactionRollbackFails(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repos := database.NewRepositories(db)
err := repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
_, err := tx.ExecContext(ctx, "ROLLBACK")
if err != nil {
return err
}
return errIntentionalRollback
})
if !errors.Is(err, errIntentionalRollback) {
t.Fatalf("expected the body's error, got: %v", err)
}
if !strings.Contains(err.Error(), "rolling back transaction") {
t.Errorf("expected the failed rollback in the error, got: %v", err)
}
}
// When the body ends the transaction itself with ROLLBACK and then
// panics, WithTx's own rollback fails; the panic must still reach the
// caller.
func TestRepositoriesTransactionPanicRollbackFails(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repos := database.NewRepositories(db)
defer func() {
if recover() == nil {
t.Error("expected the body's panic to reach the caller")
}
}()
_ = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
_, err := tx.ExecContext(ctx, "ROLLBACK")
if err != nil {
return err
}
panic(errIntentionalRollback)
})
}
func TestRepositoriesReadTransaction(t *testing.T) {
t.Parallel()
+7 -10
View File
@@ -10,6 +10,8 @@ import (
"sync"
"time"
"unicode"
"github.com/dustin/go-humanize"
)
// groupSeparator joins an open group path to an attribute key. This
@@ -304,18 +306,13 @@ func formatDuration(d time.Duration) string {
}
}
// formatBytes formats bytes in a human-readable way
// formatBytes formats a byte count with humanize.Bytes, as every command
// prints one. humanize.Bytes takes no negative count, so one is printed
// as it is.
func formatBytes(b int64) string {
const unit = 1024
if b < unit {
if b < 0 {
return fmt.Sprintf("%d B", b)
}
div, exp := int64(unit), 0
for n := b / unit; n >= unit; n /= unit {
div *= unit
exp++
}
return fmt.Sprintf("%.1f %cB", float64(b)/float64(div), "KMGTPE"[exp])
return humanize.Bytes(uint64(b))
}
+5 -4
View File
@@ -200,7 +200,8 @@ func TestTTYHandlerWithGroupQualifiesKeys(t *testing.T) {
func TestTTYHandlerByteFormattingSurvivesGrouping(t *testing.T) {
t.Parallel()
const oneAndAHalfKiB = 1536
// Printed as "1.0 MB", in powers of 1000 like every other byte count.
const oneMegabyte = 1_000_000
for name, testCase := range map[string]struct {
derive func(*slog.Logger) *slog.Logger
@@ -223,13 +224,13 @@ func TestTTYHandlerByteFormattingSurvivesGrouping(t *testing.T) {
var buf bytes.Buffer
logger := slog.New(log.NewTTYHandler(&buf, debugHandlerOptions()))
testCase.derive(logger).Info("uploaded", "bytes", oneAndAHalfKiB)
testCase.derive(logger).Info("uploaded", "bytes", oneMegabyte)
line := ansiEscape.ReplaceAllString(buf.String(), "")
assert.Contains(t, line, testCase.key+"=1.5 KB",
assert.Contains(t, line, testCase.key+"=1.0 MB",
"a byte count must be human-readable however it is qualified")
assert.NotContains(t, line, strconv.Itoa(oneAndAHalfKiB),
assert.NotContains(t, line, strconv.Itoa(oneMegabyte),
"the raw number must not survive the formatting")
})
}
+45 -1
View File
@@ -103,6 +103,8 @@ func (v *Vaultik) FetchAndDecryptBlob(
}
// FetchBlob downloads a blob and returns a reader for the encrypted data.
// The reader is closed as soon as ctx is cancelled, which ends a read
// blocked inside it.
func (v *Vaultik) FetchBlob(
ctx context.Context, blobHash string,
) (io.ReadCloser, error) {
@@ -120,5 +122,47 @@ func (v *Vaultik) FetchBlob(
return nil, fmt.Errorf("downloading blob %s: %w", shortHash(blobHash), err)
}
return rc, nil
return newCloseOnCancelReader(ctx, rc), nil
}
// closeOnCancelReader closes the reader it wraps when ctx is cancelled.
// Not every backend's reader stops on cancel by itself (the file
// backend's does not), and a read stalled on a slow or hung source would
// otherwise outlast the shutdown timeout, so the process would exit
// before restore or deep verify removed its decrypted temporary files.
type closeOnCancelReader struct {
io.ReadCloser
ctx context.Context //nolint:containedctx // Read reports its error
stop func() bool
}
func newCloseOnCancelReader(
ctx context.Context, rc io.ReadCloser,
) *closeOnCancelReader {
return &closeOnCancelReader{
ReadCloser: rc,
ctx: ctx,
stop: context.AfterFunc(ctx, func() { _ = rc.Close() }),
}
}
// Read returns the context's error, not the closed reader's, once ctx
// has been cancelled.
func (r *closeOnCancelReader) Read(p []byte) (int, error) {
n, err := r.ReadCloser.Read(p)
if err != nil && r.ctx.Err() != nil {
return n, r.ctx.Err()
}
return n, err
}
// Close closes the wrapped reader, unless the cancel already closed it.
func (r *closeOnCancelReader) Close() error {
if !r.stop() {
return nil
}
return r.ReadCloser.Close()
}
@@ -161,14 +161,13 @@ func TestVerifyManifestBlobsRejectsShortHash(t *testing.T) {
Blobs: []snapshot.BlobInfo{{Hash: "abc", CompressedSize: 1}},
}
verified, missing, mismatched, missingSize, unchecked, err :=
verified, missing, mismatched, missingSize, err :=
v.verifyManifestBlobs(manifest, &VerifyOptions{JSON: true})
require.ErrorIs(t, err, errInvalidBlobHash)
require.Zero(t, verified)
require.Zero(t, missing)
require.Zero(t, mismatched)
require.Zero(t, missingSize)
require.Empty(t, unchecked)
}
// TestVerifyBlobChunksRejectsNegativeLength proves a blob_chunks row with a
-16
View File
@@ -89,22 +89,6 @@ type SnapshotInfo struct {
RemotePresent *bool `json:"remote_present"`
}
// formatBytes formats bytes in a human-readable format
func formatBytes(bytes int64) string {
const unit = 1024
if bytes < unit {
return fmt.Sprintf("%d B", bytes)
}
div, exp := int64(unit), 0
for n := bytes / unit; n >= unit; n /= unit {
div *= unit
exp++
}
return fmt.Sprintf("%.1f %cB", float64(bytes)/float64(div), "KMGTPE"[exp])
}
// parseSnapshotTimestamp extracts the timestamp from a snapshot ID
// Format: hostname_snapshotname_2026-01-12T14:41:15Z
func parseSnapshotTimestamp(snapshotID string) (time.Time, error) {
+118 -2
View File
@@ -142,14 +142,14 @@ func TestRestoreCleansTempDirOnInterrupt(t *testing.T) {
}
// scratchEntries returns the vaultik blob-cache and snapshot-database
// scratch entries currently present in dir.
// scratch entries (restore's and deep verify's) currently present in dir.
func scratchEntries(t *testing.T, dir string) []string {
t.Helper()
var matches []string
for _, pattern := range []string{
"vaultik-blobcache-*", "vaultik-restore-*",
"vaultik-blobcache-*", "vaultik-restore-*", "vaultik-verify-*",
} {
found, err := filepath.Glob(filepath.Join(dir, pattern))
require.NoError(t, err)
@@ -206,3 +206,119 @@ func TestRestoreSkipErrorsCancelDuringBlobDownload(t *testing.T) {
assert.NotContains(t, out.String(), srcPath,
"the cancel was reported as a failed file")
}
// stallingBlobStorer wraps a Storer so that every blob download hands
// over the first half of the blob and then stalls, ignoring the context,
// as a file:// blob on a hung USB stick or network share does. Only
// closing the returned reader ends the stall.
type stallingBlobStorer struct {
storage.Storer
once sync.Once
stalled chan struct{}
}
func newStallingBlobStorer(inner storage.Storer) *stallingBlobStorer {
return &stallingBlobStorer{Storer: inner, stalled: make(chan struct{})}
}
func (s *stallingBlobStorer) Get(
ctx context.Context, key string,
) (io.ReadCloser, error) {
rc, err := s.Storer.Get(ctx, key)
if err != nil || !strings.HasPrefix(key, "blobs/") {
return rc, err
}
blob, err := io.ReadAll(rc)
_ = rc.Close()
if err != nil {
return nil, err
}
// The writer is never closed, so once the first half has been read
// the next Read blocks until the reader is closed.
pr, pw := io.Pipe()
go func() {
_, _ = pw.Write(blob[:len(blob)/2])
s.once.Do(func() { close(s.stalled) })
}()
return pr, nil
}
// TestInterruptDuringStalledBlobReadCleansTempDir stops a restore and a
// deep verify while a blob read is stalled on a source that ignores the
// context. Stopping must end that read, so the operation unwinds and
// removes its decrypted scratch files well within the shutdown timeout
// (issue #325).
//
// Not parallel: it points TMPDIR at a private directory (via t.Setenv)
// so it can assert on exactly the scratch files the operation created.
func TestInterruptDuringStalledBlobReadCleansTempDir(t *testing.T) {
log.Initialize(log.Config{})
operations := map[string]func(v *Vaultik, snapshotID, targetDir string) error{
"restore": func(v *Vaultik, snapshotID, targetDir string) error {
return v.Restore(&RestoreOptions{
SnapshotID: snapshotID,
TargetDir: targetDir,
})
},
"deep verify": func(v *Vaultik, snapshotID, _ string) error {
return v.RunDeepVerify(snapshotID, &VerifyOptions{Deep: true})
},
}
for name, op := range operations {
t.Run(name, func(t *testing.T) {
fs := afero.NewOsFs()
root := t.TempDir()
// Random content, so the blob stays large enough to stall
// part-way through its compressed, encrypted body.
cfg, storer, snapshotID, _ := backupOneFile(context.Background(),
t, fs, root, "a.bin", randomBytes(t, 256*1024), 0o644)
scratch := filepath.Join(root, "scratch")
require.NoError(t, fs.MkdirAll(scratch, 0o755))
t.Setenv("TMPDIR", scratch)
stalling := newStallingBlobStorer(storer)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
v := newRestoreVaultik(ctx, cfg, stalling, fs)
v.cancel = cancel
var opErr error
stop := v.StartOperation(func() {
opErr = op(v, snapshotID, filepath.Join(root, "restored"))
})
select {
case <-stalling.stalled:
case <-time.After(30 * time.Second):
t.Fatal("the operation never reached a blob read")
}
require.NotEmpty(t, scratchEntries(t, scratch),
"expected decrypted scratch files to exist mid-read")
stopCtx, stopCancel := context.WithTimeout(
context.Background(), 10*time.Second)
defer stopCancel()
require.True(t, stop(stopCtx),
"stop timed out; the stalled blob read was not interrupted")
require.ErrorIs(t, opErr, context.Canceled)
require.Empty(t, scratchEntries(t, scratch),
"decrypted scratch files remained after the interrupt")
})
}
}
-223
View File
@@ -4,20 +4,15 @@ import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path"
"path/filepath"
"slices"
"testing"
"github.com/spf13/afero"
"github.com/stretchr/testify/require"
"sneak.berlin/go/vaultik/internal/log"
"sneak.berlin/go/vaultik/internal/snapshot"
"sneak.berlin/go/vaultik/internal/storage"
"sneak.berlin/go/vaultik/internal/ui"
"sneak.berlin/go/vaultik/internal/vaultik"
)
@@ -156,224 +151,6 @@ func TestShallowVerifyDetectsMissingDatabase(t *testing.T) {
"shallow verify must fail when db.zst.age is absent")
}
// errStatDenied stands in for any storage error other than not-found: a
// permission error, an S3 403, a dropped connection.
var errStatDenied = errors.New("injected stat failure")
// statFailingStorer fails Stat with errStatDenied for the keys in
// failKeys and passes every other call to the wrapped Storer.
type statFailingStorer struct {
storage.Storer
failKeys []string
}
func (s statFailingStorer) Stat(
ctx context.Context, key string,
) (*storage.ObjectInfo, error) {
if slices.Contains(s.failKeys, key) {
return nil, fmt.Errorf("stat %q: %w", key, errStatDenied)
}
return s.Storer.Stat(ctx, key)
}
// TestShallowVerifyReportsStatErrorsApartFromMissing backs up a real
// snapshot, deletes one blob, and makes storage fail Stat with an error
// other than not-found for a second blob and for the encrypted database.
// Only the deleted blob may be counted as missing. The second blob and
// the database must be reported as could not be checked, with the error
// storage returned, and verify must fail, in the text output and under
// --json.
func TestShallowVerifyReportsStatErrorsApartFromMissing(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
fs := afero.NewOsFs()
tempDir := t.TempDir()
dataDir := filepath.Join(tempDir, "source")
storeDir := filepath.Join(tempDir, "remote")
dbPath := filepath.Join(tempDir, "index.sqlite")
chunkSize := int64(32 * 1024)
maxBlobSize := int64(128 * 1024)
// Random content, so the snapshot spans several blobs.
require.NoError(t, fs.MkdirAll(dataDir, 0o755))
appendRandomBytes(t, fs, map[string][]byte{},
filepath.Join(dataDir, "data.bin"), maxBlobSize*4)
ctx := context.Background()
cfg, storer, snapshotID := runFileStorageBackup(
ctx, t, fs, dataDir, storeDir, dbPath, chunkSize, maxBlobSize)
blobKeys := storedBlobKeys(t, storeDir)
require.GreaterOrEqual(t, len(blobKeys), 2, "expected at least two blobs")
missingKey, uncheckedKey := blobKeys[0], blobKeys[1]
missingHash, uncheckedHash := path.Base(missingKey), path.Base(uncheckedKey)
require.NoError(t, os.Remove(filepath.Join(storeDir, missingKey)))
dbKey := "metadata/" + snapshot.RemoteSnapshotKey(snapshotID) + "/db.zst.age"
failing := statFailingStorer{
Storer: storer,
failKeys: []string{uncheckedKey, dbKey},
}
newVerifier := func(out io.Writer) *vaultik.Vaultik {
v := &vaultik.Vaultik{
Config: cfg,
Storage: failing,
Fs: fs,
Stdout: out,
Stderr: io.Discard,
UI: ui.NewWithColor(io.Discard, false),
}
v.SetContext(ctx)
return v
}
var out bytes.Buffer
err := newVerifier(&out).VerifySnapshotWithOptions(
snapshotID, &vaultik.VerifyOptions{JSON: true})
require.Error(t, err)
result := decodeVerifyResult(t, out.Bytes())
require.Equal(t, "failed", result.Status)
require.Equal(t, 1, result.Missing, "only the deleted blob is missing")
require.Equal(t, result.BlobCount-2, result.Verified)
require.Len(t, result.Unchecked, 1)
require.Equal(t, uncheckedHash, result.Unchecked[0].Hash)
require.Contains(t, result.Unchecked[0].Error, errStatDenied.Error())
require.False(t, result.DatabaseMissing,
"a database that could not be checked is not missing")
require.Contains(t, result.DatabaseCheckError, errStatDenied.Error())
out.Reset()
err = newVerifier(&out).VerifySnapshotWithOptions(
snapshotID, &vaultik.VerifyOptions{})
require.Error(t, err)
text := out.String()
require.Contains(t, text, "Missing: "+missingHash)
require.NotContains(t, text, "Missing: "+uncheckedHash)
require.Contains(t, text, "Could not check: "+uncheckedHash)
require.Contains(t, text, "Encrypted database: could not be checked")
require.Contains(t, text, errStatDenied.Error())
require.Contains(t, text, "FAILED")
}
// TestShallowVerifyQuietFailsOnStatErrorAlone backs up a healthy snapshot,
// then makes storage fail Stat, with an error other than not-found, for
// one blob alone and then for the encrypted database alone. Each run must
// fail verify, and under --quiet, which drops the report, the error
// storage returned must still reach stderr.
//
//nolint:paralleltest // captureProcessStderr replaces os.Stderr
func TestShallowVerifyQuietFailsOnStatErrorAlone(t *testing.T) {
log.Initialize(log.Config{})
fs := afero.NewOsFs()
tempDir := t.TempDir()
dataDir := filepath.Join(tempDir, "source")
storeDir := filepath.Join(tempDir, "remote")
dbPath := filepath.Join(tempDir, "index.sqlite")
chunkSize := int64(32 * 1024)
maxBlobSize := int64(128 * 1024)
require.NoError(t, fs.MkdirAll(dataDir, 0o755))
require.NoError(t, afero.WriteFile(fs,
filepath.Join(dataDir, "data.bin"),
bytesPattern("shallow-quiet-", int(maxBlobSize*2)), 0o644))
ctx := context.Background()
cfg, storer, snapshotID := runFileStorageBackup(
ctx, t, fs, dataDir, storeDir, dbPath, chunkSize, maxBlobSize)
newQuietVerifier := func(s storage.Storer) *vaultik.Vaultik {
v := &vaultik.Vaultik{
Config: cfg,
Storage: s,
Fs: fs,
Stdout: io.Discard,
Stderr: io.Discard,
UI: ui.NewWithColor(io.Discard, false),
}
v.SetContext(ctx)
v.UI.SetQuiet(true)
return v
}
require.NoError(t,
newQuietVerifier(storer).VerifySnapshotWithOptions(
snapshotID, &vaultik.VerifyOptions{}),
"shallow verify should pass on a healthy snapshot")
blobKeys := storedBlobKeys(t, storeDir)
require.NotEmpty(t, blobKeys)
dbKey := "metadata/" + snapshot.RemoteSnapshotKey(snapshotID) + "/db.zst.age"
for _, failKey := range []string{blobKeys[0], dbKey} {
v := newQuietVerifier(statFailingStorer{
Storer: storer,
failKeys: []string{failKey},
})
var err error
stderr := captureProcessStderr(t, func() {
err = v.VerifySnapshotWithOptions(
snapshotID, &vaultik.VerifyOptions{})
})
require.Error(t, err,
"verify must fail when %s could not be checked", failKey)
require.Contains(t, stderr, errStatDenied.Error(),
"under --quiet the error for %s must still reach stderr", failKey)
}
}
// storedBlobKeys returns the storage key of every blob file under
// storeDir, sorted.
func storedBlobKeys(t *testing.T, storeDir string) []string {
t.Helper()
var keys []string
err := filepath.Walk(filepath.Join(storeDir, "blobs"),
func(p string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if info.IsDir() {
return nil
}
rel, err := filepath.Rel(storeDir, p)
if err != nil {
return err
}
keys = append(keys, filepath.ToSlash(rel))
return nil
})
require.NoError(t, err)
return keys
}
// growOneBlob appends bytes to the first blob file found under blobsDir,
// changing its on-disk size so it no longer matches the manifest.
func growOneBlob(t *testing.T, fs afero.Fs, blobsDir string) {
+22 -71
View File
@@ -14,7 +14,6 @@ import (
"sneak.berlin/go/vaultik/internal/log"
"sneak.berlin/go/vaultik/internal/snapshot"
"sneak.berlin/go/vaultik/internal/storage"
"sneak.berlin/go/vaultik/internal/types"
)
@@ -634,7 +633,7 @@ func (v *Vaultik) confirmAndExecutePurge(
v.stdoutf(" %s (%s, %s)\n",
snap.ID,
snap.Timestamp.Format("2006-01-02 15:04:05"),
formatBytes(snap.CompressedSize))
ubytes(snap.CompressedSize))
}
}
@@ -775,10 +774,18 @@ func (v *Vaultik) VerifySnapshotWithOptions(
v.stdoutf("Checking blob presence and sizes...\n")
}
v.checkSnapshotDatabase(remoteKey, result)
// A snapshot is only restorable if its encrypted database is present
// alongside the blobs. Shallow verify checks that the object exists; it
// does not decrypt it (that is deep verify's job).
dbPath := fmt.Sprintf("metadata/%s/db.zst.age", remoteKey)
result.Verified, result.Missing, result.Mismatched, result.MissingSize,
result.Unchecked, err = v.verifyManifestBlobs(manifest, opts)
_, dbErr := v.Storage.Stat(v.ctx, dbPath)
if dbErr != nil {
result.DatabaseMissing = true
}
result.Verified, result.Missing, result.Mismatched, result.MissingSize, err =
v.verifyManifestBlobs(manifest, opts)
if err != nil {
if opts.JSON {
result.Status = verifyStatusFailed
@@ -815,46 +822,25 @@ func (v *Vaultik) printVerifyHeader(snapshotID string, opts *VerifyOptions) {
}
}
// checkSnapshotDatabase records in result whether the snapshot's
// encrypted database is missing or could not be checked. A snapshot is
// only restorable if its database is present alongside the blobs.
// Shallow verify checks that the object exists; it does not decrypt it
// (that is deep verify's job).
func (v *Vaultik) checkSnapshotDatabase(remoteKey string, result *VerifyResult) {
dbPath := fmt.Sprintf("metadata/%s/db.zst.age", remoteKey)
_, err := v.Storage.Stat(v.ctx, dbPath)
switch {
case errors.Is(err, storage.ErrNotFound):
result.DatabaseMissing = true
case err != nil:
result.DatabaseCheckError = err.Error()
}
}
// verifyManifestBlobs checks that each blob in the manifest is present in
// storage with the size the manifest records, returning the counts of
// blobs that were present with the right size, absent, and present but the
// wrong size, the total bytes of the absent blobs, and the blobs it could
// not check because storage returned an error other than not-found. It
// does not read blob contents; deep verification (RunDeepVerify) does
// that. The size comparison matches the deep path (see
// verifyBlobExistenceFromDB).
// wrong size, plus the total bytes of the absent blobs. It does not read
// blob contents; deep verification (RunDeepVerify) does that. The size
// comparison matches the deep path (see verifyBlobExistenceFromDB).
func (v *Vaultik) verifyManifestBlobs(
manifest *snapshot.Manifest, opts *VerifyOptions,
) (int, int, int, int64, []UncheckedBlob, error) {
) (int, int, int, int64, error) {
var (
verified, missing, mismatched int
missingSize int64
unchecked []UncheckedBlob
)
for _, blob := range manifest.Blobs {
// The manifest is unauthenticated, so its blob hashes are checked
// before being spliced into a storage path.
if !isBlobHash(blob.Hash) {
return 0, 0, 0, 0, nil, fmt.Errorf("%w: %s",
return 0, 0, 0, 0, fmt.Errorf("%w: %s",
errInvalidBlobHash, shortHash(blob.Hash))
}
@@ -863,7 +849,7 @@ func (v *Vaultik) verifyManifestBlobs(
stat, err := v.Storage.Stat(v.ctx, blobPath)
switch {
case errors.Is(err, storage.ErrNotFound):
case err != nil:
if !opts.JSON && !v.UI.Quiet() {
v.stdoutf(" Missing: %s (%s)\n",
blob.Hash, ubytes(blob.CompressedSize))
@@ -871,13 +857,6 @@ func (v *Vaultik) verifyManifestBlobs(
missing++
missingSize += blob.CompressedSize
case err != nil:
if !opts.JSON && !v.UI.Quiet() {
v.stdoutf(" Could not check: %s (%v)\n", blob.Hash, err)
}
unchecked = append(unchecked,
UncheckedBlob{Hash: blob.Hash, Error: err.Error()})
case stat.Size != blob.CompressedSize:
if !opts.JSON && !v.UI.Quiet() {
v.stdoutf(" Wrong size: %s (store has %s, manifest lists %s)\n",
@@ -890,7 +869,7 @@ func (v *Vaultik) verifyManifestBlobs(
}
}
return verified, missing, mismatched, missingSize, unchecked, nil
return verified, missing, mismatched, missingSize, nil
}
// formatVerifyResult outputs the final verification results as JSON or
@@ -911,19 +890,9 @@ func (v *Vaultik) formatVerifyResult(
return v.outputVerifyJSON(result)
}
// --quiet drops the report but not the errors storage returned, which
// are logged on stderr. A failure is still returned either way, and
// the cli layer prints it on stderr.
if v.UI.Quiet() {
for _, blob := range result.Unchecked {
log.Warn("Could not check blob", "blob", blob.Hash, "error", blob.Error)
}
if result.DatabaseCheckError != "" {
log.Warn("Could not check the encrypted database",
"snapshot", result.SnapshotID, "error", result.DatabaseCheckError)
}
} else {
// Under --quiet a failure is still returned, and the cli layer
// prints it on stderr.
if !v.UI.Quiet() {
v.printVerifySummary(result, failure)
}
@@ -949,19 +918,10 @@ func (v *Vaultik) printVerifySummary(result *VerifyResult, failure string) {
v.stdoutf(" Wrong size: %d blobs\n", result.Mismatched)
}
if len(result.Unchecked) > 0 {
v.stdoutf(" Could not check: %d blobs\n", len(result.Unchecked))
}
if result.DatabaseMissing {
v.stdoutf(" Encrypted database: missing\n")
}
if result.DatabaseCheckError != "" {
v.stdoutf(" Encrypted database: could not be checked (%s)\n",
result.DatabaseCheckError)
}
v.stdoutf(" Status: ")
if failure != "" {
@@ -989,19 +949,10 @@ func shallowVerifyFailure(result *VerifyResult) string {
fmt.Sprintf("%d blobs have the wrong size", result.Mismatched))
}
if len(result.Unchecked) > 0 {
parts = append(parts, fmt.Sprintf("%d blobs could not be checked",
len(result.Unchecked)))
}
if result.DatabaseMissing {
parts = append(parts, "the encrypted database is missing")
}
if result.DatabaseCheckError != "" {
parts = append(parts, "the encrypted database could not be checked")
}
return strings.Join(parts, "; ")
}
+3 -3
View File
@@ -553,8 +553,8 @@ func (v *Vaultik) printSnapshotTable(snapshots []SnapshotInfo) error {
if snap.LocallyTracked {
id = snap.ID.String()
uncompressed = formatBytes(snap.UncompressedSize)
newChunks = formatBytes(snap.NewChunkSize)
uncompressed = ubytes(snap.UncompressedSize)
newChunks = ubytes(snap.NewChunkSize)
} else {
id = formatRemoteOnlyID(snap.RemoteKey)
uncompressed = remoteOnlyCell
@@ -564,7 +564,7 @@ func (v *Vaultik) printSnapshotTable(snapshots []SnapshotInfo) error {
_, err = fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n",
id,
snap.Timestamp.Format("2006-01-02 15:04:05"),
formatBytes(snap.CompressedSize),
ubytes(snap.CompressedSize),
uncompressed,
newChunks)
if err != nil {
+44 -2
View File
@@ -212,8 +212,8 @@ const (
listConfiguredName = "home"
listLocalID = "testhost_home_2026-03-01T10:00:00Z"
listRemoteID = "otherhost_media_2026-03-02T11:22:33Z"
// fiveMegabytes formats as "5.0 MB" through formatBytes.
fiveMegabytes = 5 * 1024 * 1024
// fiveMegabytes is printed as "5.0 MB" by every command.
fiveMegabytes = 5_000_000
)
// TestListSnapshots_RemoteWithoutSecretKey is the regression guard for
@@ -306,6 +306,48 @@ func TestListSnapshots_RemoteOnlyRowRendering(t *testing.T) {
"are not in the local index")
}
// TestSnapshotListAndPurgePrintSizesLikeRemoteInfo checks that the
// `snapshot list` table and the purge listing print a byte count the way
// `remote info` does: in powers of 1000, as the MB label says.
func TestSnapshotListAndPurgePrintSizesLikeRemoteInfo(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
const wantSize = "5.0 MB" // fiveMegabytes
env := newListEnv(t)
// A remote-only snapshot: `snapshot list` and `remote info` both take
// its size from the manifest.
remoteKey := env.addRemote(t, listRemoteID,
time.Date(2026, 3, 2, 11, 22, 33, 0, time.UTC))
// A locally tracked snapshot of the same size for the purge to list.
// The purge prints the snapshot's blob size, the last argument here.
start := time.Date(2026, 3, 1, 10, 0, 0, 0, time.UTC)
env.addLocal(t, listLocalID, start)
env.addRemote(t, listLocalID, start)
require.NoError(t, env.v.Repositories.Snapshots.UpdateCounts(
context.Background(), nil, listLocalID, 0, 0, 0, 0, fiveMegabytes))
require.NoError(t, env.v.RemoteInfo(false))
assert.Contains(t, findTableRow(t, env.stdout.String(), remoteKey[:16]),
wantSize, "remote info")
env.stdout.Reset()
require.NoError(t, env.v.ListSnapshots(false))
assert.Contains(t, findTableRow(t, env.stdout.String(),
"<remote only:"+remoteKey[:12]+">"), wantSize, "snapshot list")
// Without --force and with nothing on stdin the purge lists what it
// would delete, reads no confirmation, and deletes nothing.
env.stdout.Reset()
require.NoError(t, env.v.PurgeSnapshotsWithOptions(
&vaultik.SnapshotPurgeOptions{OlderThan: "1d"}))
assert.Contains(t, findTableRow(t, env.stdout.String(), listLocalID),
wantSize, "snapshot purge")
}
// findTableRow returns the single output line containing needle.
func findTableRow(t *testing.T, out, needle string) string {
t.Helper()
+2 -16
View File
@@ -57,25 +57,11 @@ type VerifyResult struct {
Missing int `json:"missing"`
MissingSize int64 `json:"missing_size,omitempty"`
Mismatched int `json:"mismatched,omitempty"`
// Unchecked lists the blobs shallow verify could not check, because
// storage answered with an error other than not-found, so whether
// they are present is unknown.
Unchecked []UncheckedBlob `json:"unchecked,omitempty"`
// DatabaseMissing is set by shallow verify when the snapshot's
// encrypted database (metadata/<key>/db.zst.age) is absent, which
// makes the snapshot unrestorable regardless of the blobs.
DatabaseMissing bool `json:"database_missing,omitempty"`
// DatabaseCheckError is set by shallow verify to the error storage
// returned when the encrypted database could not be checked.
DatabaseCheckError string `json:"database_check_error,omitempty"`
ErrorMessage string `json:"error,omitempty"`
}
// UncheckedBlob is a blob shallow verify could not check, with the error
// storage returned for it.
type UncheckedBlob struct {
Hash string `json:"hash"`
Error string `json:"error"`
DatabaseMissing bool `json:"database_missing,omitempty"`
ErrorMessage string `json:"error,omitempty"`
}
// deepVerifyFailure records a failure in the result and returns it appropriately