Author SHA1 Message Date
sneak 8dee0179d8 Honor NO_COLOR in the terminal log handler (closes #402)
check / check (push) Successful in 10m43s
The TTY log handler always wrote ANSI color, unlike internal/ui, which
disables color when NO_COLOR is set. Add a color flag to TTYHandler,
set from NO_COLOR with the same test internal/ui uses, and route every
color code through a paint helper that returns plain text when color is
off. The README's color sentence now covers log lines too.

Model: deepseek-v4-pro (implementation); opus-5-5 (rebase)
2026-11-10 22:25:49 +00:00
20 changed files with 1360 additions and 456 deletions

No files matched your search

+2 -1
View File
@@ -831,7 +831,8 @@ Work planned after 1.0. Loosely ordered by priority.
Every command's user-facing output is governed by `internal/ui`, in one of two Every command's user-facing output is governed by `internal/ui`, in one of two
ways. Color is enabled when the stream written to is a TTY and the `NO_COLOR` ways. Color is enabled when the stream written to is a TTY and the `NO_COLOR`
environment variable is unset (https://no-color.org/). environment variable is unset (https://no-color.org/). The same rule governs the
colorized log lines `internal/log` writes to a terminal.
- **Status, progress, warnings, and errors** go through the `internal/ui` - **Status, progress, warnings, and errors** go through the `internal/ui`
message methods below: marker-prefixed, colored on a TTY, and — except message methods below: marker-prefixed, colored on a TTY, and — except
+1205 -3
View File
File diff suppressed because it is too large. Load diff
-34
View File
@@ -2,10 +2,8 @@ package blobgen_test
import ( import (
"bytes" "bytes"
"errors"
"fmt" "fmt"
"testing" "testing"
"testing/iotest"
"filippo.io/age" "filippo.io/age"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
@@ -122,38 +120,6 @@ func TestNewReaderTrailingAndGarbage(t *testing.T) {
}) })
} }
// TestNewReaderGarbageErrorQuotesNoBytes checks that lines of text in
// place of a blob fail with ErrNotAgeStream, and that the error carries
// none of their bytes. The lines end in a newline because age's own error
// quotes the data's first line only when it reads a whole line.
func TestNewReaderGarbageErrorQuotesNoBytes(t *testing.T) {
t.Parallel()
id, _ := makeIdentity(t)
garbage := bytes.Repeat([]byte("garbage-bytes-\n"), 100)
_, err := blobgen.NewReader(bytes.NewReader(garbage), id)
require.ErrorIs(t, err, blobgen.ErrNotAgeStream)
assert.NotContains(t, err.Error(), "garbage-bytes-")
}
// errConnectionReset stands for a storage read that fails.
var errConnectionReset = errors.New("connection reset")
// TestNewReaderReadFailure checks that a failed read of the age header is
// returned as that read error, not as ErrNotAgeStream.
func TestNewReaderReadFailure(t *testing.T) {
t.Parallel()
id, _ := makeIdentity(t)
_, err := blobgen.NewReader(iotest.ErrReader(errConnectionReset), id)
require.ErrorIs(t, err, errConnectionReset)
assert.NotErrorIs(t, err, blobgen.ErrNotAgeStream)
}
// TestNewWriterInvalidLevel covers the rejected end of issue case 9: an // TestNewWriterInvalidLevel covers the rejected end of issue case 9: an
// out-of-range compression level errors and writes nothing to the destination. // out-of-range compression level errors and writes nothing to the destination.
func TestNewWriterInvalidLevel(t *testing.T) { func TestNewWriterInvalidLevel(t *testing.T) {
+3 -39
View File
@@ -11,11 +11,6 @@ import (
"github.com/klauspost/compress/zstd" "github.com/klauspost/compress/zstd"
) )
// ErrNotAgeStream is returned by NewReader when the data cannot be opened
// as an age stream, for example because its header is malformed. It
// replaces age's own error, which can quote the data's first bytes.
var ErrNotAgeStream = errors.New("not a valid age stream")
// Reader wraps decompression and decryption with SHA256 verification // Reader wraps decompression and decryption with SHA256 verification
type Reader struct { type Reader struct {
reader io.Reader reader io.Reader
@@ -29,25 +24,11 @@ type Reader struct {
// NewReader creates a new Reader that decrypts, decompresses, and verifies // NewReader creates a new Reader that decrypts, decompresses, and verifies
// data. Every supplied identity is offered to age.Decrypt, so a blob // data. Every supplied identity is offered to age.Decrypt, so a blob
// encrypted to any one of them can be read. // encrypted to any one of them can be read.
//
// A failure to read r is returned as that error, and a blob encrypted to
// none of the identities as age.NoIdentityMatchError. Any other failure
// to open the age stream is ErrNotAgeStream.
func NewReader(r io.Reader, identities ...age.Identity) (*Reader, error) { func NewReader(r io.Reader, identities ...age.Identity) (*Reader, error) {
src := &readErrorRecorder{reader: r} // Create decryption reader
decReader, err := age.Decrypt(r, identities...)
decReader, err := age.Decrypt(src, identities...)
if err != nil { if err != nil {
var noMatch *age.NoIdentityMatchError return nil, fmt.Errorf("creating decryption reader: %w", err)
switch {
case src.err != nil:
return nil, fmt.Errorf("reading age header: %w", src.err)
case errors.As(err, &noMatch):
return nil, fmt.Errorf("creating decryption reader: %w", err)
default:
return nil, ErrNotAgeStream
}
} }
// Create decompression reader // Create decompression reader
@@ -71,23 +52,6 @@ func NewReader(r io.Reader, identities ...age.Identity) (*Reader, error) {
}, nil }, nil
} }
// readErrorRecorder passes reads through to reader and keeps the first
// error other than io.EOF, so NewReader can tell a failed read from bad
// data.
type readErrorRecorder struct {
reader io.Reader
err error
}
func (r *readErrorRecorder) Read(p []byte) (int, error) {
n, err := r.reader.Read(p)
if err != nil && !errors.Is(err, io.EOF) && r.err == nil {
r.err = err
}
return n, err
}
// Read implements io.Reader // Read implements io.Reader
func (r *Reader) Read(p []byte) (int, error) { func (r *Reader) Read(p []byte) (int, error) {
n, err := r.teeReader.Read(p) n, err := r.teeReader.Read(p)
+3 -4
View File
@@ -7,7 +7,6 @@ import (
"github.com/spf13/cobra" "github.com/spf13/cobra"
"sneak.berlin/go/vaultik/internal/config" "sneak.berlin/go/vaultik/internal/config"
"sneak.berlin/go/vaultik/internal/log" "sneak.berlin/go/vaultik/internal/log"
"sneak.berlin/go/vaultik/internal/vaultik"
) )
// NewDatabaseCommand creates the database command group // NewDatabaseCommand creates the database command group
@@ -91,12 +90,12 @@ Use --force to skip the confirmation prompt.`,
var confirm string var confirm string
// A failure to read the answer means no confirmation. _, err = fmt.Scanln(&confirm)
_, err = fmt.Fscanln(cmd.InOrStdin(), &confirm)
if err != nil || confirm != "yes" { if err != nil || confirm != "yes" {
_, _ = fmt.Fprintln(w, "Aborted.") _, _ = fmt.Fprintln(w, "Aborted.")
return vaultik.ErrNotConfirmed //nolint:nilerr // a failed/aborted confirmation is a clean abort
return nil
} }
} }
-48
View File
@@ -1,48 +0,0 @@
package cli //nolint:testpackage // sets the unexported rootFlags directly
import (
"bytes"
"errors"
"io"
"os"
"strings"
"testing"
"sneak.berlin/go/vaultik/internal/vaultik"
)
// TestDatabaseDeleteDeclinedIsError checks that answering anything but
// "yes" keeps the database and returns ErrNotConfirmed, so the command
// exits non-zero.
//
//nolint:paralleltest // mutates the process-global rootFlags
func TestDatabaseDeleteDeclinedIsError(t *testing.T) {
dir := t.TempDir()
dbPath, cfgPath := seedDatabaseDeleteConfig(t, dir)
setRootFlags(t, RootFlags{ConfigPath: cfgPath})
var out bytes.Buffer
cmd := newDatabaseDeleteCommand()
cmd.SetOut(&out)
cmd.SetErr(io.Discard)
cmd.SetIn(strings.NewReader("no\n"))
cmd.SetArgs(nil)
err := cmd.Execute()
if !errors.Is(err, vaultik.ErrNotConfirmed) {
t.Errorf("declined database delete returned %v, want %v",
err, vaultik.ErrNotConfirmed)
}
_, err = os.Stat(dbPath)
if err != nil {
t.Errorf("declined database delete removed the file: %v", err)
}
if !strings.Contains(out.String(), "Aborted.") {
t.Errorf("declined database delete printed %q, want Aborted.",
out.String())
}
}
+32 -11
View File
@@ -5,6 +5,7 @@ import (
"fmt" "fmt"
"io" "io"
"log/slog" "log/slog"
"os"
"strconv" "strconv"
"strings" "strings"
"sync" "sync"
@@ -60,6 +61,12 @@ type TTYHandler struct {
mu *sync.Mutex mu *sync.Mutex
out io.Writer out io.Writer
// color reports whether records are written with ANSI color. It is
// false when NO_COLOR is set, matching internal/ui's shouldColor
// test; the caller already chose this handler for a terminal, so
// TTY-ness is not re-checked here.
color bool
// attrs are the attributes accumulated through WithAttrs, emitted // attrs are the attributes accumulated through WithAttrs, emitted
// ahead of each record's own attributes. Their keys already carry // ahead of each record's own attributes. Their keys already carry
// the group path that was open when they were added, so no // the group path that was open when they were added, so no
@@ -72,16 +79,19 @@ type TTYHandler struct {
groups []string groups []string
} }
// NewTTYHandler creates a new TTY handler with colored output. // NewTTYHandler creates a new TTY handler. Color is enabled unless the
// NO_COLOR environment variable is set to a non-empty value, the same
// test internal/ui applies to status lines.
func NewTTYHandler(out io.Writer, opts *slog.HandlerOptions) *TTYHandler { func NewTTYHandler(out io.Writer, opts *slog.HandlerOptions) *TTYHandler {
if opts == nil { if opts == nil {
opts = &slog.HandlerOptions{} opts = &slog.HandlerOptions{}
} }
return &TTYHandler{ return &TTYHandler{
out: out, out: out,
opts: *opts, opts: *opts,
mu: &sync.Mutex{}, mu: &sync.Mutex{},
color: os.Getenv("NO_COLOR") == "",
} }
} }
@@ -124,10 +134,10 @@ func (h *TTYHandler) Handle(_ context.Context, r slog.Record) error {
// are written around it: it can carry text from an untrusted source // are written around it: it can carry text from an untrusted source
// (a storage error, for one), and a raw control character would // (a storage error, for one), and a raw control character would
// otherwise reach the terminal. // otherwise reach the terminal.
_, _ = fmt.Fprintf(h.out, "%s%s%s %s%s%s %s%s%s", _, _ = fmt.Fprintf(h.out, "%s %s %s",
colorGray, timestamp, colorReset, h.paint(colorGray, timestamp),
levelColor, level, colorReset, h.paint(levelColor, level),
colorBold, sanitize(r.Message), colorReset) h.paint(colorBold, sanitize(r.Message)))
// Attributes carried by the handler come first, then the record's // Attributes carried by the handler come first, then the record's
// own. Handler attributes were qualified when they were added; the // own. Handler attributes were qualified when they were added; the
@@ -236,6 +246,7 @@ func (h *TTYHandler) clone() *TTYHandler {
opts: h.opts, opts: h.opts,
mu: h.mu, mu: h.mu,
out: h.out, out: h.out,
color: h.color,
attrs: make([]slog.Attr, len(h.attrs), len(h.attrs)+1), attrs: make([]slog.Attr, len(h.attrs), len(h.attrs)+1),
groups: make([]string, len(h.groups), len(h.groups)+1), groups: make([]string, len(h.groups), len(h.groups)+1),
} }
@@ -273,9 +284,19 @@ func (h *TTYHandler) writeAttr(a slog.Attr) {
// the snapshot database — so a control character in one of them must // the snapshot database — so a control character in one of them must
// be rendered as an escape sequence rather than reaching the terminal, // be rendered as an escape sequence rather than reaching the terminal,
// where it could move the cursor or inject its own colours. // where it could move the cursor or inject its own colours.
_, _ = fmt.Fprintf(h.out, " %s%s%s=%s%s%s", _, _ = fmt.Fprintf(h.out, " %s=%s",
colorCyan, sanitize(a.Key), colorReset, h.paint(colorCyan, sanitize(a.Key)),
colorBlue, sanitize(value), colorReset) h.paint(colorBlue, sanitize(value)))
}
// paint wraps s in the given ANSI color when color is enabled, and
// returns s unchanged otherwise.
func (h *TTYHandler) paint(color, s string) string {
if !h.color {
return s
}
return color + s + colorReset
} }
// sanitize returns s unchanged when every rune in it is printable, and a // sanitize returns s unchanged when every rune in it is printable, and a
+41
View File
@@ -421,3 +421,44 @@ func TestTTYHandlerEmptyGroupAndAttrsAreNoOps(t *testing.T) {
assert.Contains(t, plain, "kept=yes") assert.Contains(t, plain, "kept=yes")
assert.NotContains(t, plain, " =") assert.NotContains(t, plain, " =")
} }
// TestTTYHandlerRespectsNoColor checks that only a non-empty NO_COLOR
// turns color off, matching internal/ui's shouldColor test. The handler
// is only ever chosen for a terminal, so this is the one remaining
// condition that can still leave color in log output.
func TestTTYHandlerRespectsNoColor(t *testing.T) {
cases := []struct {
name string
noColor string
colored bool
}{
{
name: "NO_COLOR turns color off",
noColor: "1",
},
{
name: "empty NO_COLOR keeps color on",
noColor: "",
colored: true,
},
}
for _, testCase := range cases {
t.Run(testCase.name, func(t *testing.T) {
t.Setenv("NO_COLOR", testCase.noColor)
var buf bytes.Buffer
logger := slog.New(log.NewTTYHandler(&buf, debugHandlerOptions()))
logger.Info("hello", "key", "value")
if testCase.colored {
assert.Contains(t, buf.String(), "\x1b[",
"an empty NO_COLOR must leave ANSI color in log output")
} else {
assert.NotContains(t, buf.String(), "\x1b[",
"NO_COLOR must suppress every ANSI escape in log output")
}
})
}
}
+11 -83
View File
@@ -7,7 +7,6 @@ import (
"os" "os"
"path/filepath" "path/filepath"
"strings" "strings"
"sync"
"testing" "testing"
"github.com/spf13/afero" "github.com/spf13/afero"
@@ -25,28 +24,19 @@ import (
// that is not plugged in, cannot be listed. It is not an empty store, so // that is not plugged in, cannot be listed. It is not an empty store, so
// no command may conclude from it that the local snapshots are gone. // no command may conclude from it that the local snapshots are gone.
// newFileDestinationVaultik writes the fault source tree and returns a // newFileDestinationVaultik returns a Vaultik that backs up the
// Vaultik that backs it up to a file:// destination at storeDir. // snapshot named "first" to a file:// destination at storeDir.
func newFileDestinationVaultik( func newFileDestinationVaultik(
ctx context.Context, t *testing.T, storeDir string, ctx context.Context, t *testing.T, storeDir string,
) (*vaultik.Vaultik, *database.Repositories) { ) (*vaultik.Vaultik, *database.Repositories) {
t.Helper() t.Helper()
osFs := afero.NewOsFs()
tempDir := t.TempDir() tempDir := t.TempDir()
dataDir := filepath.Join(tempDir, "src") dataDir := filepath.Join(tempDir, "src")
writeFaultSourceTree(t, afero.NewOsFs(), dataDir) dbPath := filepath.Join(tempDir, "index.sqlite")
return openFileDestinationVaultik(ctx, t, storeDir, dataDir, writeFaultSourceTree(t, osFs, dataDir)
filepath.Join(tempDir, "index.sqlite"))
}
// openFileDestinationVaultik returns a Vaultik that backs up dataDir to a
// file:// destination at storeDir, with its local index at dbPath. The
// index is closed when the test ends.
func openFileDestinationVaultik(
ctx context.Context, t *testing.T, storeDir, dataDir, dbPath string,
) (*vaultik.Vaultik, *database.Repositories) {
t.Helper()
store, err := storage.NewFileStorer(storeDir) store, err := storage.NewFileStorer(storeDir)
require.NoError(t, err) require.NoError(t, err)
@@ -58,84 +48,22 @@ func openFileDestinationVaultik(
repos := database.NewRepositories(db) repos := database.NewRepositories(db)
cfg := changedFileConfig(dataDir, dbPath) cfg := changedFileConfig(dataDir, dbPath)
return newBackupVaultik(ctx, cfg, store, repos, db, afero.NewOsFs()), repos return newBackupVaultik(ctx, cfg, store, repos, db, osFs), repos
}
// firstBackup is one backup of the snapshot named "first" to a file://
// destination. dir holds its source tree in src, its store in store and
// its closed local index in index/index.sqlite. The first test that
// needs it makes it, and TestMain removes it. Every test that copies it
// backs up the same source tree, so no test may change that tree.
//
//nolint:gochecknoglobals // made once and shared by the tests that copy it
var firstBackup struct {
once sync.Once
dir string
ready bool
}
// TestMain removes firstBackup once every test has run.
func TestMain(m *testing.M) {
code := m.Run()
_ = os.RemoveAll(firstBackup.dir)
os.Exit(code)
}
// makeFirstBackup writes the fault source tree and backs it up as the
// snapshot named "first", all in a new directory that it records in
// firstBackup.
func makeFirstBackup(ctx context.Context, t *testing.T) {
t.Helper()
dir, err := os.MkdirTemp("", "vaultik-first-backup-")
require.NoError(t, err)
firstBackup.dir = dir
storeDir := filepath.Join(dir, "store")
dataDir := filepath.Join(dir, "src")
indexDir := filepath.Join(dir, "index")
require.NoError(t, os.Mkdir(storeDir, 0o750))
require.NoError(t, os.Mkdir(indexDir, 0o750))
writeFaultSourceTree(t, afero.NewOsFs(), dataDir)
v, _ := openFileDestinationVaultik(ctx, t, storeDir, dataDir,
filepath.Join(indexDir, "index.sqlite"))
require.NoError(t, backUp(v, "first"))
require.NoError(t, v.DB.Close())
firstBackup.ready = true
} }
// backUpToFileDestination creates the directory storeDir, as a user must // backUpToFileDestination creates the directory storeDir, as a user must
// before the first backup, and returns a Vaultik whose file:// // before the first backup, and backs up the snapshot named "first" to a
// destination there holds a backup of the snapshot named "first". The // file:// destination there. Everything the returned Vaultik prints
// store and the local index are copies of firstBackup's, which is faster // after the backup goes to the returned buffer.
// than backing up in every test. The returned Vaultik backs up a source
// tree shared with every other caller, so no test may change that tree.
// Everything the returned Vaultik prints after the backup goes to the
// returned buffer.
func backUpToFileDestination( func backUpToFileDestination(
ctx context.Context, t *testing.T, storeDir string, ctx context.Context, t *testing.T, storeDir string,
) (*vaultik.Vaultik, *database.Repositories, *bytes.Buffer) { ) (*vaultik.Vaultik, *database.Repositories, *bytes.Buffer) {
t.Helper() t.Helper()
firstBackup.once.Do(func() { makeFirstBackup(ctx, t) })
require.True(t, firstBackup.ready, "another test failed to make firstBackup")
require.NoError(t, os.MkdirAll(storeDir, 0o750)) require.NoError(t, os.MkdirAll(storeDir, 0o750))
require.NoError(t, os.CopyFS(storeDir,
os.DirFS(filepath.Join(firstBackup.dir, "store"))))
indexDir := t.TempDir() v, repos := newFileDestinationVaultik(ctx, t, storeDir)
require.NoError(t, os.CopyFS(indexDir, require.NoError(t, backUp(v, "first"))
os.DirFS(filepath.Join(firstBackup.dir, "index"))))
v, repos := openFileDestinationVaultik(ctx, t, storeDir,
filepath.Join(firstBackup.dir, "src"),
filepath.Join(indexDir, "index.sqlite"))
out := &bytes.Buffer{} out := &bytes.Buffer{}
v.Stdout = out v.Stdout = out
+1 -1
View File
@@ -267,7 +267,7 @@ func (v *Vaultik) PruneBlobs(opts *PruneOptions) error {
if !v.confirmPrune(opts, len(unreferencedBlobs), totalSize, if !v.confirmPrune(opts, len(unreferencedBlobs), totalSize,
dirsWithoutManifest) { dirsWithoutManifest) {
return ErrNotConfirmed return nil
} }
v.deleteUnreferencedBlobs(unreferencedBlobs, allBlobs, result) v.deleteUnreferencedBlobs(unreferencedBlobs, allBlobs, result)
+1 -2
View File
@@ -176,8 +176,7 @@ func TestPruneDeclinedKeepsMetadataDirWithoutManifest(t *testing.T) {
env.v.Stdin = strings.NewReader("n\n") env.v.Stdin = strings.NewReader("n\n")
require.ErrorIs(t, env.v.PruneBlobs(&vaultik.PruneOptions{}), require.NoError(t, env.v.PruneBlobs(&vaultik.PruneOptions{}))
vaultik.ErrNotConfirmed)
assert.True(t, env.store.hasKey(leftDB), assert.True(t, env.store.hasKey(leftDB),
"a declined prune must keep the directory") "a declined prune must keep the directory")
+3 -5
View File
@@ -54,10 +54,9 @@ func TestPurgeSnapshots_QuietPromptListsSnapshots(t *testing.T) {
v.UI.SetQuiet(true) v.UI.SetQuiet(true)
// Stdin is empty, so the prompt reads no answer and cancels. // Stdin is empty, so the prompt reads no answer and cancels.
err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{ require.NoError(t, v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{
KeepLatest: true, KeepLatest: true,
}) }))
require.ErrorIs(t, err, vaultik.ErrNotConfirmed)
assert.Contains(t, out.String(), snapHomeT0) assert.Contains(t, out.String(), snapHomeT0)
assert.Contains(t, out.String(), "Delete 1 snapshot(s)? [y/N]") assert.Contains(t, out.String(), "Delete 1 snapshot(s)? [y/N]")
@@ -78,8 +77,7 @@ func TestPruneBlobs_QuietPromptShowsBlobTotal(t *testing.T) {
addBlob(t, env.store.testStorer, testBlobHashA) addBlob(t, env.store.testStorer, testBlobHashA)
// Stdin is empty, so the prompt reads no answer and cancels. // Stdin is empty, so the prompt reads no answer and cancels.
require.ErrorIs(t, env.v.PruneBlobs(&vaultik.PruneOptions{}), require.NoError(t, env.v.PruneBlobs(&vaultik.PruneOptions{}))
vaultik.ErrNotConfirmed)
assert.Contains(t, env.stdout.String(), assert.Contains(t, env.stdout.String(),
"Found 1 unreferenced blob(s) totaling") "Found 1 unreferenced blob(s) totaling")
-21
View File
@@ -337,27 +337,6 @@ func TestRemoveSnapshot_DefaultRemovesMetadataNotBlobs(t *testing.T) {
assert.Contains(t, out, "vaultik prune") assert.Contains(t, out, "vaultik prune")
} }
// TestRemoveSnapshot_DeclinedIsError checks that answering no to the
// prompt keeps the snapshot and returns ErrNotConfirmed, so the command
// exits non-zero.
func TestRemoveSnapshot_DeclinedIsError(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
store := newTestStorer()
addManifest(t, store, "snapshot-001", nil)
tv := vaultik.NewForTesting(store)
tv.Repositories = newLocalIndex(t, "snapshot-001")
tv.Stdin.WriteString("n\n")
_, err := tv.RemoveSnapshot("snapshot-001", &vaultik.RemoveOptions{})
require.ErrorIs(t, err, vaultik.ErrNotConfirmed)
assert.True(t, store.hasKey(remoteKeyPath("snapshot-001")))
assert.Contains(t, tv.Stdout.String(), "Cancelled")
}
func TestRemoveSnapshot_DryRun(t *testing.T) { func TestRemoveSnapshot_DryRun(t *testing.T) {
log.Initialize(log.Config{}) log.Initialize(log.Config{})
t.Parallel() t.Parallel()
+11 -55
View File
@@ -106,12 +106,7 @@ type RestoreResult struct {
BytesDownloaded int64 BytesDownloaded int64
Duration time.Duration Duration time.Duration
FilesFailed int FilesFailed int
FailedFiles []string // Paths of files that failed to restore for their own reason FailedFiles []string // Paths of files that failed to restore
// Blobs that could not be downloaded, and the paths of the files
// skipped because they need one of them. Those files are counted in
// FilesFailed but not listed in the summary.
BlobsFailed int
FilesNeedingFailedBlob []string
// Verification results (only populated if Verify option is set) // Verification results (only populated if Verify option is set)
FilesVerified int FilesVerified int
BytesVerified int64 BytesVerified int64
@@ -231,11 +226,6 @@ func (v *Vaultik) finishRestore(
if result.FilesFailed > 0 { if result.FilesFailed > 0 {
v.UI.Warningf("%d file(s) failed to restore:", result.FilesFailed) v.UI.Warningf("%d file(s) failed to restore:", result.FilesFailed)
if result.BlobsFailed > 0 {
v.UI.Detailf("%d of them need a blob that could not be read (%d blob(s) in all).",
len(result.FilesNeedingFailedBlob), result.BlobsFailed)
}
for _, path := range result.FailedFiles { for _, path := range result.FailedFiles {
v.UI.Detailf("%s", v.UI.Path(path)) v.UI.Detailf("%s", v.UI.Path(path))
} }
@@ -504,8 +494,8 @@ func (v *Vaultik) runRestoreLoop(
// Returns false when nothing is pending download (the caller stops). // Returns false when nothing is pending download (the caller stops).
// //
// A blob that cannot be downloaded is reported through // A blob that cannot be downloaded is reported through
// handleRestoreBlobError, so it aborts the restore unless --skip-errors // handleRestoreFileError for every pending file that references it, so
// is set. // it aborts the restore unless --skip-errors is set.
func (s *restoreSession) downloadNextBlobSet( func (s *restoreSession) downloadNextBlobSet(
plan *restorePlan, filesByID map[types.FileID]*database.File, plan *restorePlan, filesByID map[types.FileID]*database.File,
) (bool, error) { ) (bool, error) {
@@ -538,11 +528,12 @@ func (s *restoreSession) downloadNextBlobSet(
return false, err return false, err
} }
blobErr := s.v.handleRestoreBlobError( for _, fileID := range plan.filesReferencingBlob(hash) {
plan, s.opts, s.result, filesByID, fileErr := s.v.handleRestoreFileError(
plan.filesReferencingBlob(hash), err) plan, s.opts, s.result, filesByID[fileID], fileID, err)
if blobErr != nil { if fileErr != nil {
return false, blobErr return false, fileErr
}
} }
// next is among the failed files, so the rest of its blob set // next is among the failed files, so the rest of its blob set
@@ -657,41 +648,6 @@ func (v *Vaultik) handleRestoreFileError(
return nil return nil
} }
// handleRestoreBlobError records a blob that could not be downloaded.
// fileIDs are the pending files that need it, never empty. Without
// --skip-errors the first of them fails the restore. With it, the blob is
// reported once, and its files are counted as failed and dropped from
// the plan without being reported one by one.
func (v *Vaultik) handleRestoreBlobError(
plan *restorePlan, opts *RestoreOptions, result *RestoreResult,
filesByID map[types.FileID]*database.File, fileIDs []types.FileID,
err error,
) error {
if !opts.SkipErrors {
return v.handleRestoreFileError(
plan, opts, result, filesByID[fileIDs[0]], fileIDs[0], err)
}
log.Error("Failed to restore files", "files", len(fileIDs), "error", err)
v.UI.Errorf("Failed to restore %d file(s): %v. Skipping them (--skip-errors).",
len(fileIDs), err)
result.BlobsFailed++
for _, fileID := range fileIDs {
path := filesByID[fileID].Path.String()
log.Debug("Skipped file that needs a failed blob", "path", path)
result.FilesFailed++
result.FilesNeedingFailedBlob = append(result.FilesNeedingFailedBlob, path)
plan.finishFile(fileID)
}
return nil
}
// printRestoreProgress emits a periodic restore-phase status line via // printRestoreProgress emits a periodic restore-phase status line via
// the UI writer, mirroring scanner.printProcessingProgress so the two // the UI writer, mirroring scanner.printProcessingProgress so the two
// long-running commands have the same on-screen rhythm. // long-running commands have the same on-screen rhythm.
@@ -763,8 +719,8 @@ func (v *Vaultik) handleRestoreVerification(
) error { ) error {
// A file that failed to restore was not written and is already // A file that failed to restore was not written and is already
// reported as such, so it is not verified. // reported as such, so it is not verified.
failedToRestore := make(map[string]bool, result.FilesFailed) failedToRestore := make(map[string]bool, len(result.FailedFiles))
for _, path := range slices.Concat(result.FailedFiles, result.FilesNeedingFailedBlob) { for _, path := range result.FailedFiles {
failedToRestore[path] = true failedToRestore[path] = true
} }
+7 -7
View File
@@ -46,11 +46,11 @@ import (
// slice out a few KB — currently the dominant cost in restore. // slice out a few KB — currently the dominant cost in restore.
// //
// The test deliberately constructs an adversarial scenario: three // The test deliberately constructs an adversarial scenario: three
// blobs A/B/C of five 64 KiB chunks each, nine files distributed across // blobs A/B/C of ~6 MB each, nine files distributed across them, and
// them, and path-ordered names that interleave the blobs (a1, b1, c1, // path-ordered names that interleave the blobs (a1, b1, c1, a2, b2,
// a2, b2, c2, …) so naive path-order processing would touch every blob // c2, …) so naive path-order processing would touch every blob before
// before finishing any of them. // finishing any of them.
// localitySource is one 64 KiB fixture file used by the locality test. // localitySource is one 1 MiB fixture file used by the locality test.
type localitySource struct { type localitySource struct {
path string path string
data []byte data []byte
@@ -65,10 +65,10 @@ type localityCopy struct {
// localitySourceBytes is the size of each source file of the locality // localitySourceBytes is the size of each source file of the locality
// fixture. // fixture.
const localitySourceBytes = 64 * 1024 const localitySourceBytes = 1024 * 1024
// buildLocalityFixture writes the adversarial source layout described // buildLocalityFixture writes the adversarial source layout described
// in TestRestoreLocalityAndReadAt: 15 sources of 64 KiB each (which the // in TestRestoreLocalityAndReadAt: 15 sources of 1 MiB each (which the
// backup packs into 3 blobs of 5 chunks) plus 9 interleaved-name copies // backup packs into 3 blobs of 5 chunks) plus 9 interleaved-name copies
// (3 per blob group). // (3 per blob group).
func buildLocalityFixture( func buildLocalityFixture(
+15 -89
View File
@@ -51,72 +51,27 @@ func TestRestoreSkipErrorsSkipsFilesOfMissingBlob(t *testing.T) {
t.Parallel() t.Parallel()
ctx := context.Background() ctx := context.Background()
backup := backupThenDeleteOneBlob(ctx, t) assertRestoreSkipsLostFiles(ctx, t, backupThenDeleteOneBlob(ctx, t))
out := assertRestoreSkipsLostFiles(ctx, t, backup)
assertLostFilesCountedNotListed(t, backup, out)
}
// TestRestoreSkipErrorsReportsUnreadableBlobOnce restores with SkipErrors
// after one blob of a two-blob snapshot was overwritten with other bytes
// of the same size. The blob must be reported once, by its hash, as not a
// valid age stream, without quoting its bytes, and its files must be
// counted, not listed.
func TestRestoreSkipErrorsReportsUnreadableBlobOnce(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
ctx := context.Background()
backup, lostBlob := backupThenOverwriteOneBlob(ctx, t)
out := assertRestoreSkipsLostFiles(ctx, t, backup)
assert.Equal(t, 1, strings.Count(out, "not a valid age stream"), out)
assert.Contains(t, out, lostBlob[:16])
assert.NotContains(t, out, overwriteBlobPattern)
assertLostFilesCountedNotListed(t, backup, out)
} }
// TestRestoreSkipErrorsSkipsFilesWithChunkInNoBlob restores with // TestRestoreSkipErrorsSkipsFilesWithChunkInNoBlob restores with
// SkipErrors a snapshot whose metadata lists chunks that no blob holds, as // SkipErrors a snapshot whose metadata lists chunks that no blob holds, as
// earlier backup defects left behind. Each such file fails for its own // earlier backup defects left behind.
// reason, so each is listed.
func TestRestoreSkipErrorsSkipsFilesWithChunkInNoBlob(t *testing.T) { func TestRestoreSkipErrorsSkipsFilesWithChunkInNoBlob(t *testing.T) {
log.Initialize(log.Config{}) log.Initialize(log.Config{})
t.Parallel() t.Parallel()
ctx := context.Background() ctx := context.Background()
backup := backupThenDropOneBlobFromMetadata(ctx, t) assertRestoreSkipsLostFiles(ctx, t, backupThenDropOneBlobFromMetadata(ctx, t))
out := assertRestoreSkipsLostFiles(ctx, t, backup)
for path := range backup.lost {
assert.Containsf(t, out, path, "%s must be reported", path)
}
} }
// assertLostFilesCountedNotListed checks that the restore output out // assertRestoreSkipsLostFiles restores backup with SkipErrors. Every file
// counts the files of the lost blob and names none of them. // stored in the lost blob must be reported as failed and left absent,
func assertLostFilesCountedNotListed( // every other file must be restored intact, and Restore must still return
t *testing.T, backup *missingBlobBackup, out string, // an error.
) {
t.Helper()
assert.Contains(t, out, fmt.Sprintf(
"%d of them need a blob that could not be read (1 blob(s) in all)",
len(backup.lost)))
for path := range backup.lost {
assert.NotContainsf(t, out, path,
"%s needs the lost blob and must be counted, not listed", path)
}
}
// assertRestoreSkipsLostFiles restores backup with SkipErrors and returns
// what it printed. Every file stored in the lost blob must be left
// absent, every other file must be restored intact, and Restore must
// still return an error.
func assertRestoreSkipsLostFiles( func assertRestoreSkipsLostFiles(
ctx context.Context, t *testing.T, backup *missingBlobBackup, ctx context.Context, t *testing.T, backup *missingBlobBackup,
) string { ) {
t.Helper() t.Helper()
var out bytes.Buffer var out bytes.Buffer
@@ -138,6 +93,8 @@ func assertRestoreSkipsLostFiles(
restored := filepath.Join(backup.restoreDir, path) restored := filepath.Join(backup.restoreDir, path)
if backup.lost[path] { if backup.lost[path] {
assert.Containsf(t, out.String(), path,
"%s needs the lost blob and must be reported", path)
assert.NoFileExists(t, restored) assert.NoFileExists(t, restored)
continue continue
@@ -147,8 +104,6 @@ func assertRestoreSkipsLostFiles(
require.NoErrorf(t, err, "%s does not need the lost blob", path) require.NoErrorf(t, err, "%s does not need the lost blob", path)
assert.Equalf(t, content, got, "%s restored with wrong content", path) assert.Equalf(t, content, got, "%s restored with wrong content", path)
} }
return out.String()
} }
// TestRestoreSkipErrorsVerifyChecksOnlyRestoredFiles restores with // TestRestoreSkipErrorsVerifyChecksOnlyRestoredFiles restores with
@@ -189,9 +144,9 @@ func TestRestoreSkipErrorsVerifyChecksOnlyRestoredFiles(t *testing.T) {
// TestRestoreSkipErrorsVerifyReportsFailuresApart restores with // TestRestoreSkipErrorsVerifyReportsFailuresApart restores with
// SkipErrors and Verify after one blob of a two-blob snapshot was // SkipErrors and Verify after one blob of a two-blob snapshot was
// deleted, while one file that does restore is written one byte too long. // deleted, while one file that does restore is written one byte too long.
// The restore warning must count the files of the deleted blob without // The restore warning must list only the files of the deleted blob, the
// listing them, the verification summary must list only the long file, // verification summary only the long file, each path once, and Restore
// once, and Restore must return an error. // must return an error.
func TestRestoreSkipErrorsVerifyReportsFailuresApart(t *testing.T) { func TestRestoreSkipErrorsVerifyReportsFailuresApart(t *testing.T) {
log.Initialize(log.Config{}) log.Initialize(log.Config{})
t.Parallel() t.Parallel()
@@ -231,9 +186,9 @@ func TestRestoreSkipErrorsVerifyReportsFailuresApart(t *testing.T) {
"Verification failed: 1 files did not match") "Verification failed: 1 files did not match")
require.Truef(t, found, "no verification summary in:\n%s", out.String()) require.Truef(t, found, "no verification summary in:\n%s", out.String())
assertLostFilesCountedNotListed(t, backup, warning)
for path := range backup.lost { for path := range backup.lost {
assert.Equalf(t, 1, strings.Count(warning, path),
"%s must be listed once as failed to restore", path)
assert.NotContainsf(t, summary, path, assert.NotContainsf(t, summary, path,
"%s was not restored and must not fail verification", path) "%s was not restored and must not fail verification", path)
} }
@@ -337,35 +292,6 @@ func backupThenDeleteOneBlob(
return backup return backup
} }
// overwriteBlobPattern repeats, each time followed by a newline, through
// the bytes that backupThenOverwriteOneBlob writes over a blob. age's own
// error quotes the data's first line only when it reads a whole line.
const overwriteBlobPattern = "overwritten-blob-bytes-"
// backupThenOverwriteOneBlob backs up missingBlobFileCount single-chunk
// files, then overwrites the blob holding the first of them with as many
// bytes of overwriteBlobPattern lines. It also returns that blob's hash.
func backupThenOverwriteOneBlob(
ctx context.Context, t *testing.T,
) (*missingBlobBackup, string) {
t.Helper()
backup, db, lostBlob := backupSingleChunkFiles(ctx, t)
require.NoError(t, db.Close())
key := fmt.Sprintf("blobs/%s/%s/%s", lostBlob[:2], lostBlob[2:4], lostBlob)
info, err := backup.storer.Stat(ctx, key)
require.NoError(t, err)
line := overwriteBlobPattern + "\n"
pattern := bytes.Repeat([]byte(line), int(info.Size)/len(line)+1)
require.NoError(t, backup.storer.Put(
ctx, key, bytes.NewReader(pattern[:info.Size])))
return backup, lostBlob
}
// backupThenDropOneBlobFromMetadata backs up missingBlobFileCount // backupThenDropOneBlobFromMetadata backs up missingBlobFileCount
// single-chunk files, then removes the blob holding the first of them from // single-chunk files, then removes the blob holding the first of them from
// the snapshot and exports the snapshot metadata again. The exported // the snapshot and exports the snapshot metadata again. The exported
@@ -27,10 +27,10 @@ import (
// TestRestoreSweeperEvictsBlobs exercises the reference-counted blob // TestRestoreSweeperEvictsBlobs exercises the reference-counted blob
// disk cache eviction during restore. // disk cache eviction during restore.
// //
// The scenario: 30 unique 128 KiB random files plus 10 duplicates of // The scenario: 30 unique 1 MB random files plus 10 duplicates of those
// those (40 files total, 3.75 MiB of unique content) get backed up with // (40 files total, 30 MB of unique content) get backed up with a 10 MB
// a 1 MiB blob_size_limit. After backup the snapshot's encrypted blobs // blob_size_limit. After backup the snapshot's encrypted blobs are
// are restored through Vaultik.Restore, and per-key Get counts on the // restored through Vaultik.Restore, and per-key Get counts on the
// storage layer are recorded. Each blob in the snapshot MUST be // storage layer are recorded. Each blob in the snapshot MUST be
// downloaded exactly once — re-downloads would mean the sweeper either // downloaded exactly once — re-downloads would mean the sweeper either
// evicted a blob that was still needed (LRU regression) or that the // evicted a blob that was still needed (LRU regression) or that the
@@ -41,7 +41,7 @@ import (
// original AND every duplicate referencing its chunks have been // original AND every duplicate referencing its chunks have been
// restored. // restored.
// buildSweeperFixture writes the source layout for the sweeper test: 30 // buildSweeperFixture writes the source layout for the sweeper test: 30
// unique 128 KiB random files plus 10 duplicates of a random subset. The // unique 1 MB random files plus 10 duplicates of a random subset. The
// PRNG seed is fixed so failures are reproducible; the entropy is what // PRNG seed is fixed so failures are reproducible; the entropy is what
// matters here — the FastCDC chunker needs realistic-looking data to // matters here — the FastCDC chunker needs realistic-looking data to
// pick chunk boundaries naturally. Returns the expected content by // pick chunk boundaries naturally. Returns the expected content by
@@ -51,7 +51,7 @@ func buildSweeperFixture(
) map[string][]byte { ) map[string][]byte {
t.Helper() t.Helper()
const fileSize = 128 * 1024 const fileSize = 1 * 1024 * 1024
rng := rand.New(rand.NewSource(42)) //nolint:gosec // G404: fixture data only rng := rand.New(rand.NewSource(42)) //nolint:gosec // G404: fixture data only
@@ -159,7 +159,7 @@ func runSweeperBackup(
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, uniqueFiles+duplicateFiles, scanResult.FilesScanned) require.Equal(t, uniqueFiles+duplicateFiles, scanResult.FilesScanned)
require.Greater(t, scanResult.BlobsCreated, 1, require.Greater(t, scanResult.BlobsCreated, 1,
"3.75 MiB of unique data at 1 MiB blob size should yield multiple blobs") "30 MB of unique data at 10 MB blob size should yield multiple blobs")
require.NoError(t, sm.CompleteSnapshot(ctx, snapshotID)) require.NoError(t, sm.CompleteSnapshot(ctx, snapshotID))
require.NoError(t, sm.ExportSnapshotMetadata(ctx, dbPath, snapshotID)) require.NoError(t, sm.ExportSnapshotMetadata(ctx, dbPath, snapshotID))
@@ -199,8 +199,8 @@ func TestRestoreSweeperEvictsBlobs(t *testing.T) {
expected := buildSweeperFixture(t, fs, dataDir, uniqueFiles, duplicateFiles) expected := buildSweeperFixture(t, fs, dataDir, uniqueFiles, duplicateFiles)
chunkSize := int64(16 * 1024) chunkSize := int64(64 * 1024)
maxBlobSize := int64(1024 * 1024) maxBlobSize := int64(10 * 1024 * 1024)
storer, err := storage.NewFileStorer(storeDir) storer, err := storage.NewFileStorer(storeDir)
require.NoError(t, err) require.NoError(t, err)
+9 -4
View File
@@ -881,12 +881,17 @@ func (v *Vaultik) confirmAndExecutePurge(
var confirm string var confirm string
// A failure to read the answer means no confirmation.
_, err := v.scanStdin(&confirm) _, err := v.scanStdin(&confirm)
if err != nil || strings.ToLower(confirm) != "y" { if err != nil {
v.printlnStdout("Cancelled") v.printlnStdout("Cancelled")
return ErrNotConfirmed return nil //nolint:nilerr // treat EOF or read error as "no"
}
if strings.ToLower(confirm) != "y" {
v.printlnStdout("Cancelled")
return nil
} }
} else if !quiet { } else if !quiet {
v.UI.Infof("The following snapshots will be deleted:") v.UI.Infof("The following snapshots will be deleted:")
@@ -1485,7 +1490,7 @@ func (v *Vaultik) RemoveSnapshot(
} }
if !v.confirmRemoveSnapshot(snapshotID, opts) { if !v.confirmRemoveSnapshot(snapshotID, opts) {
return result, ErrNotConfirmed return result, nil
} }
// Checked before the local record is deleted: it may be the last // Checked before the local record is deleted: it may be the last
+7 -33
View File
@@ -187,17 +187,6 @@ func (e *listEnv) addRemoteRawTimestamp(
t.Helper() t.Helper()
remoteKey := snapshot.RemoteSnapshotKey(snapshotID) remoteKey := snapshot.RemoteSnapshotKey(snapshotID)
e.putRemoteManifest(t, remoteKey,
encodeRemoteManifest(t, remoteKey, timestamp))
return remoteKey
}
// encodeRemoteManifest returns the encoded manifest of a one-blob
// snapshot whose remote key is remoteKey.
func encodeRemoteManifest(t *testing.T, remoteKey, timestamp string) []byte {
t.Helper()
manifest := &snapshot.Manifest{ manifest := &snapshot.Manifest{
// Note: the hashed key, never the human ID. That is precisely // Note: the hashed key, never the human ID. That is precisely
// why a remote-only snapshot cannot be named. // why a remote-only snapshot cannot be named.
@@ -213,17 +202,11 @@ func encodeRemoteManifest(t *testing.T, remoteKey, timestamp string) []byte {
data, err := snapshot.EncodeManifest(manifest, 3) data, err := snapshot.EncodeManifest(manifest, 3)
require.NoError(t, err) require.NoError(t, err)
return data err = e.store.Put(context.Background(),
}
// putRemoteManifest stores an encoded manifest on the destination store
// under remoteKey.
func (e *listEnv) putRemoteManifest(t *testing.T, remoteKey string, data []byte) {
t.Helper()
err := e.store.Put(context.Background(),
"metadata/"+remoteKey+"/manifest.json.zst", bytes.NewReader(data)) "metadata/"+remoteKey+"/manifest.json.zst", bytes.NewReader(data))
require.NoError(t, err) require.NoError(t, err)
return remoteKey
} }
// Fixtures shared across the listing tests. // Fixtures shared across the listing tests.
@@ -362,9 +345,8 @@ func TestSnapshotListAndPurgePrintSizesLikeRemoteInfo(t *testing.T) {
// Without --force and with nothing on stdin the purge lists what it // Without --force and with nothing on stdin the purge lists what it
// would delete, reads no confirmation, and deletes nothing. // would delete, reads no confirmation, and deletes nothing.
env.stdout.Reset() env.stdout.Reset()
require.ErrorIs(t, env.v.PurgeSnapshotsWithOptions( require.NoError(t, env.v.PurgeSnapshotsWithOptions(
&vaultik.SnapshotPurgeOptions{OlderThan: "1d"}), &vaultik.SnapshotPurgeOptions{OlderThan: "1d"}))
vaultik.ErrNotConfirmed)
assert.Contains(t, findTableRow(t, env.stdout.String(), listLocalID), assert.Contains(t, findTableRow(t, env.stdout.String(), listLocalID),
wantSize, "snapshot purge") wantSize, "snapshot purge")
} }
@@ -809,17 +791,9 @@ func TestListSnapshots_JSONReportsTruncation(t *testing.T) {
timestamp := time.Date(2026, 3, 2, 11, 22, 33, 0, time.UTC) timestamp := time.Date(2026, 3, 2, 11, 22, 33, 0, time.UTC)
// One past the cap, so exactly one snapshot is omitted. The listing // One past the cap, so exactly one snapshot is omitted.
// names a snapshot by the key its manifest is stored under, never by
// the manifest's own ID. So all of them share one encoded manifest:
// encoding 1001 took seconds.
data := encodeRemoteManifest(t,
snapshot.RemoteSnapshotKey("otherhost_bulk"),
timestamp.Format(time.RFC3339))
for i := range maxRemoteOnlyRowsForTest + 1 { for i := range maxRemoteOnlyRowsForTest + 1 {
env.putRemoteManifest(t, snapshot.RemoteSnapshotKey( env.addRemote(t, fmt.Sprintf("otherhost_bulk_%04d", i), timestamp)
fmt.Sprintf("otherhost_bulk_%04d", i)), data)
} }
stderr := captureProcessStderr(t, func() { stderr := captureProcessStderr(t, func() {
-7
View File
@@ -5,7 +5,6 @@ package vaultik
import ( import (
"bytes" "bytes"
"context" "context"
"errors"
"fmt" "fmt"
"io" "io"
"os" "os"
@@ -185,12 +184,6 @@ func (v *Vaultik) printlnStdout(args ...any) {
_, _ = fmt.Fprintln(v.Stdout, args...) _, _ = fmt.Fprintln(v.Stdout, args...)
} }
// ErrNotConfirmed is returned when a confirmation prompt gets any answer
// but the one it asks for (`y` or `Y`, or `yes` for `database delete`), or
// stdin ends first. Nothing the prompt asked about is deleted, and the
// command exits non-zero.
var ErrNotConfirmed = errors.New("deletion not confirmed")
// scanStdin reads a line of input from stdin. // scanStdin reads a line of input from stdin.
func (v *Vaultik) scanStdin(a ...any) (int, error) { func (v *Vaultik) scanStdin(a ...any) (int, error) {
return fmt.Fscanln(v.Stdin, a...) return fmt.Fscanln(v.Stdin, a...)