Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8dee0179d8 |
No files matched your search
@@ -831,7 +831,8 @@ Work planned after 1.0. Loosely ordered by priority.
|
|||||||
|
|
||||||
Every command's user-facing output is governed by `internal/ui`, in one of two
|
Every command's user-facing output is governed by `internal/ui`, in one of two
|
||||||
ways. Color is enabled when the stream written to is a TTY and the `NO_COLOR`
|
ways. Color is enabled when the stream written to is a TTY and the `NO_COLOR`
|
||||||
environment variable is unset (https://no-color.org/).
|
environment variable is unset (https://no-color.org/). The same rule governs the
|
||||||
|
colorized log lines `internal/log` writes to a terminal.
|
||||||
|
|
||||||
- **Status, progress, warnings, and errors** go through the `internal/ui`
|
- **Status, progress, warnings, and errors** go through the `internal/ui`
|
||||||
message methods below: marker-prefixed, colored on a TTY, and — except
|
message methods below: marker-prefixed, colored on a TTY, and — except
|
||||||
|
|||||||
+1205
-3
File diff suppressed because it is too large.
Load diff
+32
-11
@@ -5,6 +5,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
|
"os"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
@@ -60,6 +61,12 @@ type TTYHandler struct {
|
|||||||
mu *sync.Mutex
|
mu *sync.Mutex
|
||||||
out io.Writer
|
out io.Writer
|
||||||
|
|
||||||
|
// color reports whether records are written with ANSI color. It is
|
||||||
|
// false when NO_COLOR is set, matching internal/ui's shouldColor
|
||||||
|
// test; the caller already chose this handler for a terminal, so
|
||||||
|
// TTY-ness is not re-checked here.
|
||||||
|
color bool
|
||||||
|
|
||||||
// attrs are the attributes accumulated through WithAttrs, emitted
|
// attrs are the attributes accumulated through WithAttrs, emitted
|
||||||
// ahead of each record's own attributes. Their keys already carry
|
// ahead of each record's own attributes. Their keys already carry
|
||||||
// the group path that was open when they were added, so no
|
// the group path that was open when they were added, so no
|
||||||
@@ -72,16 +79,19 @@ type TTYHandler struct {
|
|||||||
groups []string
|
groups []string
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewTTYHandler creates a new TTY handler with colored output.
|
// NewTTYHandler creates a new TTY handler. Color is enabled unless the
|
||||||
|
// NO_COLOR environment variable is set to a non-empty value, the same
|
||||||
|
// test internal/ui applies to status lines.
|
||||||
func NewTTYHandler(out io.Writer, opts *slog.HandlerOptions) *TTYHandler {
|
func NewTTYHandler(out io.Writer, opts *slog.HandlerOptions) *TTYHandler {
|
||||||
if opts == nil {
|
if opts == nil {
|
||||||
opts = &slog.HandlerOptions{}
|
opts = &slog.HandlerOptions{}
|
||||||
}
|
}
|
||||||
|
|
||||||
return &TTYHandler{
|
return &TTYHandler{
|
||||||
out: out,
|
out: out,
|
||||||
opts: *opts,
|
opts: *opts,
|
||||||
mu: &sync.Mutex{},
|
mu: &sync.Mutex{},
|
||||||
|
color: os.Getenv("NO_COLOR") == "",
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -124,10 +134,10 @@ func (h *TTYHandler) Handle(_ context.Context, r slog.Record) error {
|
|||||||
// are written around it: it can carry text from an untrusted source
|
// are written around it: it can carry text from an untrusted source
|
||||||
// (a storage error, for one), and a raw control character would
|
// (a storage error, for one), and a raw control character would
|
||||||
// otherwise reach the terminal.
|
// otherwise reach the terminal.
|
||||||
_, _ = fmt.Fprintf(h.out, "%s%s%s %s%s%s %s%s%s",
|
_, _ = fmt.Fprintf(h.out, "%s %s %s",
|
||||||
colorGray, timestamp, colorReset,
|
h.paint(colorGray, timestamp),
|
||||||
levelColor, level, colorReset,
|
h.paint(levelColor, level),
|
||||||
colorBold, sanitize(r.Message), colorReset)
|
h.paint(colorBold, sanitize(r.Message)))
|
||||||
|
|
||||||
// Attributes carried by the handler come first, then the record's
|
// Attributes carried by the handler come first, then the record's
|
||||||
// own. Handler attributes were qualified when they were added; the
|
// own. Handler attributes were qualified when they were added; the
|
||||||
@@ -236,6 +246,7 @@ func (h *TTYHandler) clone() *TTYHandler {
|
|||||||
opts: h.opts,
|
opts: h.opts,
|
||||||
mu: h.mu,
|
mu: h.mu,
|
||||||
out: h.out,
|
out: h.out,
|
||||||
|
color: h.color,
|
||||||
attrs: make([]slog.Attr, len(h.attrs), len(h.attrs)+1),
|
attrs: make([]slog.Attr, len(h.attrs), len(h.attrs)+1),
|
||||||
groups: make([]string, len(h.groups), len(h.groups)+1),
|
groups: make([]string, len(h.groups), len(h.groups)+1),
|
||||||
}
|
}
|
||||||
@@ -273,9 +284,19 @@ func (h *TTYHandler) writeAttr(a slog.Attr) {
|
|||||||
// the snapshot database — so a control character in one of them must
|
// the snapshot database — so a control character in one of them must
|
||||||
// be rendered as an escape sequence rather than reaching the terminal,
|
// be rendered as an escape sequence rather than reaching the terminal,
|
||||||
// where it could move the cursor or inject its own colours.
|
// where it could move the cursor or inject its own colours.
|
||||||
_, _ = fmt.Fprintf(h.out, " %s%s%s=%s%s%s",
|
_, _ = fmt.Fprintf(h.out, " %s=%s",
|
||||||
colorCyan, sanitize(a.Key), colorReset,
|
h.paint(colorCyan, sanitize(a.Key)),
|
||||||
colorBlue, sanitize(value), colorReset)
|
h.paint(colorBlue, sanitize(value)))
|
||||||
|
}
|
||||||
|
|
||||||
|
// paint wraps s in the given ANSI color when color is enabled, and
|
||||||
|
// returns s unchanged otherwise.
|
||||||
|
func (h *TTYHandler) paint(color, s string) string {
|
||||||
|
if !h.color {
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
return color + s + colorReset
|
||||||
}
|
}
|
||||||
|
|
||||||
// sanitize returns s unchanged when every rune in it is printable, and a
|
// sanitize returns s unchanged when every rune in it is printable, and a
|
||||||
|
|||||||
@@ -421,3 +421,44 @@ func TestTTYHandlerEmptyGroupAndAttrsAreNoOps(t *testing.T) {
|
|||||||
assert.Contains(t, plain, "kept=yes")
|
assert.Contains(t, plain, "kept=yes")
|
||||||
assert.NotContains(t, plain, " =")
|
assert.NotContains(t, plain, " =")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestTTYHandlerRespectsNoColor checks that only a non-empty NO_COLOR
|
||||||
|
// turns color off, matching internal/ui's shouldColor test. The handler
|
||||||
|
// is only ever chosen for a terminal, so this is the one remaining
|
||||||
|
// condition that can still leave color in log output.
|
||||||
|
func TestTTYHandlerRespectsNoColor(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
noColor string
|
||||||
|
colored bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "NO_COLOR turns color off",
|
||||||
|
noColor: "1",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "empty NO_COLOR keeps color on",
|
||||||
|
noColor: "",
|
||||||
|
colored: true,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, testCase := range cases {
|
||||||
|
t.Run(testCase.name, func(t *testing.T) {
|
||||||
|
t.Setenv("NO_COLOR", testCase.noColor)
|
||||||
|
|
||||||
|
var buf bytes.Buffer
|
||||||
|
|
||||||
|
logger := slog.New(log.NewTTYHandler(&buf, debugHandlerOptions()))
|
||||||
|
logger.Info("hello", "key", "value")
|
||||||
|
|
||||||
|
if testCase.colored {
|
||||||
|
assert.Contains(t, buf.String(), "\x1b[",
|
||||||
|
"an empty NO_COLOR must leave ANSI color in log output")
|
||||||
|
} else {
|
||||||
|
assert.NotContains(t, buf.String(), "\x1b[",
|
||||||
|
"NO_COLOR must suppress every ANSI escape in log output")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -22,10 +22,6 @@ var errBlobHashMismatch = errors.New("blob hash mismatch")
|
|||||||
var errBlobNotFullyRead = errors.New(
|
var errBlobNotFullyRead = errors.New(
|
||||||
"blob closed before fully read; hash not verified")
|
"blob closed before fully read; hash not verified")
|
||||||
|
|
||||||
// errWrongPrivateKeyForBlob is returned when the configured key parses
|
|
||||||
// but is not one of the keys a blob was encrypted to.
|
|
||||||
var errWrongPrivateKeyForBlob = errors.New("cannot decrypt this blob")
|
|
||||||
|
|
||||||
// hashVerifyReader wraps a blobgen.Reader and verifies the double-SHA-256 hash
|
// hashVerifyReader wraps a blobgen.Reader and verifies the double-SHA-256 hash
|
||||||
// of decrypted plaintext when Close is called. It reuses the hash that
|
// of decrypted plaintext when Close is called. It reuses the hash that
|
||||||
// blobgen.Reader already computes internally via its TeeReader, avoiding
|
// blobgen.Reader already computes internally via its TeeReader, avoiding
|
||||||
@@ -91,14 +87,10 @@ func (v *Vaultik) FetchAndDecryptBlob(
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
reader, err := v.newBlobReader(rc, identities)
|
reader, err := blobgen.NewReader(rc, identities...)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
_ = rc.Close()
|
_ = rc.Close()
|
||||||
|
|
||||||
if errors.Is(err, errWrongPrivateKeyForBlob) {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil, fmt.Errorf("creating blob reader: %w", err)
|
return nil, fmt.Errorf("creating blob reader: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -110,28 +102,6 @@ func (v *Vaultik) FetchAndDecryptBlob(
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// newBlobReader opens the decrypting reader for a blob. When none of the
|
|
||||||
// identities can decrypt it, the error says in plain words that the
|
|
||||||
// configured private key is not one of the keys the blob was encrypted
|
|
||||||
// to and names where the key was configured.
|
|
||||||
func (v *Vaultik) newBlobReader(
|
|
||||||
r io.Reader, identities []age.Identity,
|
|
||||||
) (*blobgen.Reader, error) {
|
|
||||||
blobReader, err := blobgen.NewReader(r, identities...)
|
|
||||||
|
|
||||||
if _, ok := errors.AsType[*age.NoIdentityMatchError](err); ok {
|
|
||||||
return nil, fmt.Errorf(
|
|
||||||
"%w: the private key in %s is not one of the keys the blob was encrypted to",
|
|
||||||
errWrongPrivateKeyForBlob, v.Config.AgeSecretKeySourceName())
|
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return blobReader, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// FetchBlob downloads a blob and returns a reader for the encrypted data.
|
// FetchBlob downloads a blob and returns a reader for the encrypted data.
|
||||||
// The reader is closed as soon as ctx is cancelled, which ends a read
|
// The reader is closed as soon as ctx is cancelled, which ends a read
|
||||||
// blocked inside it. A storage error is returned as is: restore and deep
|
// blocked inside it. A storage error is returned as is: restore and deep
|
||||||
|
|||||||
@@ -7,7 +7,6 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/spf13/afero"
|
"github.com/spf13/afero"
|
||||||
@@ -25,28 +24,19 @@ import (
|
|||||||
// that is not plugged in, cannot be listed. It is not an empty store, so
|
// that is not plugged in, cannot be listed. It is not an empty store, so
|
||||||
// no command may conclude from it that the local snapshots are gone.
|
// no command may conclude from it that the local snapshots are gone.
|
||||||
|
|
||||||
// newFileDestinationVaultik writes the fault source tree and returns a
|
// newFileDestinationVaultik returns a Vaultik that backs up the
|
||||||
// Vaultik that backs it up to a file:// destination at storeDir.
|
// snapshot named "first" to a file:// destination at storeDir.
|
||||||
func newFileDestinationVaultik(
|
func newFileDestinationVaultik(
|
||||||
ctx context.Context, t *testing.T, storeDir string,
|
ctx context.Context, t *testing.T, storeDir string,
|
||||||
) (*vaultik.Vaultik, *database.Repositories) {
|
) (*vaultik.Vaultik, *database.Repositories) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
|
osFs := afero.NewOsFs()
|
||||||
tempDir := t.TempDir()
|
tempDir := t.TempDir()
|
||||||
dataDir := filepath.Join(tempDir, "src")
|
dataDir := filepath.Join(tempDir, "src")
|
||||||
writeFaultSourceTree(t, afero.NewOsFs(), dataDir)
|
dbPath := filepath.Join(tempDir, "index.sqlite")
|
||||||
|
|
||||||
return openFileDestinationVaultik(ctx, t, storeDir, dataDir,
|
writeFaultSourceTree(t, osFs, dataDir)
|
||||||
filepath.Join(tempDir, "index.sqlite"))
|
|
||||||
}
|
|
||||||
|
|
||||||
// openFileDestinationVaultik returns a Vaultik that backs up dataDir to a
|
|
||||||
// file:// destination at storeDir, with its local index at dbPath. The
|
|
||||||
// index is closed when the test ends.
|
|
||||||
func openFileDestinationVaultik(
|
|
||||||
ctx context.Context, t *testing.T, storeDir, dataDir, dbPath string,
|
|
||||||
) (*vaultik.Vaultik, *database.Repositories) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
store, err := storage.NewFileStorer(storeDir)
|
store, err := storage.NewFileStorer(storeDir)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@@ -58,84 +48,22 @@ func openFileDestinationVaultik(
|
|||||||
repos := database.NewRepositories(db)
|
repos := database.NewRepositories(db)
|
||||||
cfg := changedFileConfig(dataDir, dbPath)
|
cfg := changedFileConfig(dataDir, dbPath)
|
||||||
|
|
||||||
return newBackupVaultik(ctx, cfg, store, repos, db, afero.NewOsFs()), repos
|
return newBackupVaultik(ctx, cfg, store, repos, db, osFs), repos
|
||||||
}
|
|
||||||
|
|
||||||
// firstBackup is one backup of the snapshot named "first" to a file://
|
|
||||||
// destination. dir holds its source tree in src, its store in store and
|
|
||||||
// its closed local index in index/index.sqlite. The first test that
|
|
||||||
// needs it makes it, and TestMain removes it. Every test that copies it
|
|
||||||
// backs up the same source tree, so no test may change that tree.
|
|
||||||
//
|
|
||||||
//nolint:gochecknoglobals // made once and shared by the tests that copy it
|
|
||||||
var firstBackup struct {
|
|
||||||
once sync.Once
|
|
||||||
dir string
|
|
||||||
ready bool
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestMain removes firstBackup once every test has run.
|
|
||||||
func TestMain(m *testing.M) {
|
|
||||||
code := m.Run()
|
|
||||||
|
|
||||||
_ = os.RemoveAll(firstBackup.dir)
|
|
||||||
|
|
||||||
os.Exit(code)
|
|
||||||
}
|
|
||||||
|
|
||||||
// makeFirstBackup writes the fault source tree and backs it up as the
|
|
||||||
// snapshot named "first", all in a new directory that it records in
|
|
||||||
// firstBackup.
|
|
||||||
func makeFirstBackup(ctx context.Context, t *testing.T) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
dir, err := os.MkdirTemp("", "vaultik-first-backup-")
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
firstBackup.dir = dir
|
|
||||||
storeDir := filepath.Join(dir, "store")
|
|
||||||
dataDir := filepath.Join(dir, "src")
|
|
||||||
indexDir := filepath.Join(dir, "index")
|
|
||||||
|
|
||||||
require.NoError(t, os.Mkdir(storeDir, 0o750))
|
|
||||||
require.NoError(t, os.Mkdir(indexDir, 0o750))
|
|
||||||
writeFaultSourceTree(t, afero.NewOsFs(), dataDir)
|
|
||||||
|
|
||||||
v, _ := openFileDestinationVaultik(ctx, t, storeDir, dataDir,
|
|
||||||
filepath.Join(indexDir, "index.sqlite"))
|
|
||||||
require.NoError(t, backUp(v, "first"))
|
|
||||||
require.NoError(t, v.DB.Close())
|
|
||||||
|
|
||||||
firstBackup.ready = true
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// backUpToFileDestination creates the directory storeDir, as a user must
|
// backUpToFileDestination creates the directory storeDir, as a user must
|
||||||
// before the first backup, and returns a Vaultik whose file://
|
// before the first backup, and backs up the snapshot named "first" to a
|
||||||
// destination there holds a backup of the snapshot named "first". The
|
// file:// destination there. Everything the returned Vaultik prints
|
||||||
// store and the local index are copies of firstBackup's, which is faster
|
// after the backup goes to the returned buffer.
|
||||||
// than backing up in every test. The returned Vaultik backs up a source
|
|
||||||
// tree shared with every other caller, so no test may change that tree.
|
|
||||||
// Everything the returned Vaultik prints after the backup goes to the
|
|
||||||
// returned buffer.
|
|
||||||
func backUpToFileDestination(
|
func backUpToFileDestination(
|
||||||
ctx context.Context, t *testing.T, storeDir string,
|
ctx context.Context, t *testing.T, storeDir string,
|
||||||
) (*vaultik.Vaultik, *database.Repositories, *bytes.Buffer) {
|
) (*vaultik.Vaultik, *database.Repositories, *bytes.Buffer) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
firstBackup.once.Do(func() { makeFirstBackup(ctx, t) })
|
|
||||||
require.True(t, firstBackup.ready, "another test failed to make firstBackup")
|
|
||||||
|
|
||||||
require.NoError(t, os.MkdirAll(storeDir, 0o750))
|
require.NoError(t, os.MkdirAll(storeDir, 0o750))
|
||||||
require.NoError(t, os.CopyFS(storeDir,
|
|
||||||
os.DirFS(filepath.Join(firstBackup.dir, "store"))))
|
|
||||||
|
|
||||||
indexDir := t.TempDir()
|
v, repos := newFileDestinationVaultik(ctx, t, storeDir)
|
||||||
require.NoError(t, os.CopyFS(indexDir,
|
require.NoError(t, backUp(v, "first"))
|
||||||
os.DirFS(filepath.Join(firstBackup.dir, "index"))))
|
|
||||||
|
|
||||||
v, repos := openFileDestinationVaultik(ctx, t, storeDir,
|
|
||||||
filepath.Join(firstBackup.dir, "src"),
|
|
||||||
filepath.Join(indexDir, "index.sqlite"))
|
|
||||||
|
|
||||||
out := &bytes.Buffer{}
|
out := &bytes.Buffer{}
|
||||||
v.Stdout = out
|
v.Stdout = out
|
||||||
|
|||||||
@@ -40,9 +40,6 @@ var (
|
|||||||
// which is secret, so the message is safe to print and log.
|
// which is secret, so the message is safe to print and log.
|
||||||
errInvalidAgeSecretKey = errors.New(
|
errInvalidAgeSecretKey = errors.New(
|
||||||
"configured age secret key holds no usable age identity")
|
"configured age secret key holds no usable age identity")
|
||||||
// errWrongPrivateKey is returned when the configured key parses but
|
|
||||||
// is not one of the keys the snapshot was encrypted to.
|
|
||||||
errWrongPrivateKey = errors.New("cannot decrypt this snapshot")
|
|
||||||
errBlobMissingFromIndex = errors.New("blob hash missing from blob index")
|
errBlobMissingFromIndex = errors.New("blob hash missing from blob index")
|
||||||
errChunkNotInAnyBlob = errors.New("chunk not found in any blob")
|
errChunkNotInAnyBlob = errors.New("chunk not found in any blob")
|
||||||
errBlobIDNotInHashIndex = errors.New("blob id missing from hash index")
|
errBlobIDNotInHashIndex = errors.New("blob id missing from hash index")
|
||||||
@@ -135,10 +132,6 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error {
|
|||||||
log.Info("Downloading snapshot metadata...")
|
log.Info("Downloading snapshot metadata...")
|
||||||
|
|
||||||
tempDB, tempDir, err := v.downloadSnapshotDB(opts.SnapshotID, identities)
|
tempDB, tempDir, err := v.downloadSnapshotDB(opts.SnapshotID, identities)
|
||||||
if errors.Is(err, errWrongPrivateKey) {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("downloading snapshot database: %w", err)
|
return fmt.Errorf("downloading snapshot database: %w", err)
|
||||||
}
|
}
|
||||||
@@ -330,28 +323,6 @@ func (v *Vaultik) restoreIdentities() ([]age.Identity, error) {
|
|||||||
return identities, nil
|
return identities, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// newSnapshotDBReader opens the decrypting reader for a snapshot
|
|
||||||
// database. When none of the identities can decrypt it, the error says
|
|
||||||
// in plain words that the configured private key is the wrong one and
|
|
||||||
// names where it was configured.
|
|
||||||
func (v *Vaultik) newSnapshotDBReader(
|
|
||||||
r io.Reader, identities []age.Identity,
|
|
||||||
) (*blobgen.Reader, error) {
|
|
||||||
blobReader, err := blobgen.NewReader(r, identities...)
|
|
||||||
|
|
||||||
if _, ok := errors.AsType[*age.NoIdentityMatchError](err); ok {
|
|
||||||
return nil, fmt.Errorf(
|
|
||||||
"%w: the private key in %s is not one of the keys the backup was encrypted to",
|
|
||||||
errWrongPrivateKey, v.Config.AgeSecretKeySourceName())
|
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return blobReader, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// restoreAllFiles processes files in blob-locality order: drain every
|
// restoreAllFiles processes files in blob-locality order: drain every
|
||||||
// file whose blob set is on disk, download the missing blobs for the
|
// file whose blob set is on disk, download the missing blobs for the
|
||||||
// pending file with the smallest uncached count, repeat. This keeps
|
// pending file with the smallest uncached count, repeat. This keeps
|
||||||
@@ -812,9 +783,9 @@ func (v *Vaultik) downloadSnapshotDB(
|
|||||||
// the plaintext to a temp file. Neither the encrypted bytes nor the
|
// the plaintext to a temp file. Neither the encrypted bytes nor the
|
||||||
// decrypted database is ever held whole in memory; a snapshot database
|
// decrypted database is ever held whole in memory; a snapshot database
|
||||||
// can be large.
|
// can be large.
|
||||||
blobReader, err := v.newSnapshotDBReader(reader, identities)
|
blobReader, err := blobgen.NewReader(reader, identities...)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, "", err
|
return nil, "", fmt.Errorf("creating decryption reader: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
defer func() { _ = blobReader.Close() }()
|
defer func() { _ = blobReader.Close() }()
|
||||||
|
|||||||
@@ -46,11 +46,11 @@ import (
|
|||||||
// slice out a few KB — currently the dominant cost in restore.
|
// slice out a few KB — currently the dominant cost in restore.
|
||||||
//
|
//
|
||||||
// The test deliberately constructs an adversarial scenario: three
|
// The test deliberately constructs an adversarial scenario: three
|
||||||
// blobs A/B/C of five 64 KiB chunks each, nine files distributed across
|
// blobs A/B/C of ~6 MB each, nine files distributed across them, and
|
||||||
// them, and path-ordered names that interleave the blobs (a1, b1, c1,
|
// path-ordered names that interleave the blobs (a1, b1, c1, a2, b2,
|
||||||
// a2, b2, c2, …) so naive path-order processing would touch every blob
|
// c2, …) so naive path-order processing would touch every blob before
|
||||||
// before finishing any of them.
|
// finishing any of them.
|
||||||
// localitySource is one 64 KiB fixture file used by the locality test.
|
// localitySource is one 1 MiB fixture file used by the locality test.
|
||||||
type localitySource struct {
|
type localitySource struct {
|
||||||
path string
|
path string
|
||||||
data []byte
|
data []byte
|
||||||
@@ -65,10 +65,10 @@ type localityCopy struct {
|
|||||||
|
|
||||||
// localitySourceBytes is the size of each source file of the locality
|
// localitySourceBytes is the size of each source file of the locality
|
||||||
// fixture.
|
// fixture.
|
||||||
const localitySourceBytes = 64 * 1024
|
const localitySourceBytes = 1024 * 1024
|
||||||
|
|
||||||
// buildLocalityFixture writes the adversarial source layout described
|
// buildLocalityFixture writes the adversarial source layout described
|
||||||
// in TestRestoreLocalityAndReadAt: 15 sources of 64 KiB each (which the
|
// in TestRestoreLocalityAndReadAt: 15 sources of 1 MiB each (which the
|
||||||
// backup packs into 3 blobs of 5 chunks) plus 9 interleaved-name copies
|
// backup packs into 3 blobs of 5 chunks) plus 9 interleaved-name copies
|
||||||
// (3 per blob group).
|
// (3 per blob group).
|
||||||
func buildLocalityFixture(
|
func buildLocalityFixture(
|
||||||
|
|||||||
@@ -27,10 +27,10 @@ import (
|
|||||||
// TestRestoreSweeperEvictsBlobs exercises the reference-counted blob
|
// TestRestoreSweeperEvictsBlobs exercises the reference-counted blob
|
||||||
// disk cache eviction during restore.
|
// disk cache eviction during restore.
|
||||||
//
|
//
|
||||||
// The scenario: 30 unique 128 KiB random files plus 10 duplicates of
|
// The scenario: 30 unique 1 MB random files plus 10 duplicates of those
|
||||||
// those (40 files total, 3.75 MiB of unique content) get backed up with
|
// (40 files total, 30 MB of unique content) get backed up with a 10 MB
|
||||||
// a 1 MiB blob_size_limit. After backup the snapshot's encrypted blobs
|
// blob_size_limit. After backup the snapshot's encrypted blobs are
|
||||||
// are restored through Vaultik.Restore, and per-key Get counts on the
|
// restored through Vaultik.Restore, and per-key Get counts on the
|
||||||
// storage layer are recorded. Each blob in the snapshot MUST be
|
// storage layer are recorded. Each blob in the snapshot MUST be
|
||||||
// downloaded exactly once — re-downloads would mean the sweeper either
|
// downloaded exactly once — re-downloads would mean the sweeper either
|
||||||
// evicted a blob that was still needed (LRU regression) or that the
|
// evicted a blob that was still needed (LRU regression) or that the
|
||||||
@@ -41,7 +41,7 @@ import (
|
|||||||
// original AND every duplicate referencing its chunks have been
|
// original AND every duplicate referencing its chunks have been
|
||||||
// restored.
|
// restored.
|
||||||
// buildSweeperFixture writes the source layout for the sweeper test: 30
|
// buildSweeperFixture writes the source layout for the sweeper test: 30
|
||||||
// unique 128 KiB random files plus 10 duplicates of a random subset. The
|
// unique 1 MB random files plus 10 duplicates of a random subset. The
|
||||||
// PRNG seed is fixed so failures are reproducible; the entropy is what
|
// PRNG seed is fixed so failures are reproducible; the entropy is what
|
||||||
// matters here — the FastCDC chunker needs realistic-looking data to
|
// matters here — the FastCDC chunker needs realistic-looking data to
|
||||||
// pick chunk boundaries naturally. Returns the expected content by
|
// pick chunk boundaries naturally. Returns the expected content by
|
||||||
@@ -51,7 +51,7 @@ func buildSweeperFixture(
|
|||||||
) map[string][]byte {
|
) map[string][]byte {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
const fileSize = 128 * 1024
|
const fileSize = 1 * 1024 * 1024
|
||||||
|
|
||||||
rng := rand.New(rand.NewSource(42)) //nolint:gosec // G404: fixture data only
|
rng := rand.New(rand.NewSource(42)) //nolint:gosec // G404: fixture data only
|
||||||
|
|
||||||
@@ -159,7 +159,7 @@ func runSweeperBackup(
|
|||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.Equal(t, uniqueFiles+duplicateFiles, scanResult.FilesScanned)
|
require.Equal(t, uniqueFiles+duplicateFiles, scanResult.FilesScanned)
|
||||||
require.Greater(t, scanResult.BlobsCreated, 1,
|
require.Greater(t, scanResult.BlobsCreated, 1,
|
||||||
"3.75 MiB of unique data at 1 MiB blob size should yield multiple blobs")
|
"30 MB of unique data at 10 MB blob size should yield multiple blobs")
|
||||||
|
|
||||||
require.NoError(t, sm.CompleteSnapshot(ctx, snapshotID))
|
require.NoError(t, sm.CompleteSnapshot(ctx, snapshotID))
|
||||||
require.NoError(t, sm.ExportSnapshotMetadata(ctx, dbPath, snapshotID))
|
require.NoError(t, sm.ExportSnapshotMetadata(ctx, dbPath, snapshotID))
|
||||||
@@ -199,8 +199,8 @@ func TestRestoreSweeperEvictsBlobs(t *testing.T) {
|
|||||||
|
|
||||||
expected := buildSweeperFixture(t, fs, dataDir, uniqueFiles, duplicateFiles)
|
expected := buildSweeperFixture(t, fs, dataDir, uniqueFiles, duplicateFiles)
|
||||||
|
|
||||||
chunkSize := int64(16 * 1024)
|
chunkSize := int64(64 * 1024)
|
||||||
maxBlobSize := int64(1024 * 1024)
|
maxBlobSize := int64(10 * 1024 * 1024)
|
||||||
|
|
||||||
storer, err := storage.NewFileStorer(storeDir)
|
storer, err := storage.NewFileStorer(storeDir)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|||||||
@@ -1,161 +0,0 @@
|
|||||||
package vaultik_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"io"
|
|
||||||
"path"
|
|
||||||
"path/filepath"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"filippo.io/age"
|
|
||||||
"github.com/spf13/afero"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"sneak.berlin/go/vaultik/internal/blobgen"
|
|
||||||
"sneak.berlin/go/vaultik/internal/config"
|
|
||||||
"sneak.berlin/go/vaultik/internal/log"
|
|
||||||
"sneak.berlin/go/vaultik/internal/snapshot"
|
|
||||||
"sneak.berlin/go/vaultik/internal/ui"
|
|
||||||
"sneak.berlin/go/vaultik/internal/vaultik"
|
|
||||||
)
|
|
||||||
|
|
||||||
// TestWrongPrivateKeyError checks that restore and deep verify, given a
|
|
||||||
// valid private key the backup was not encrypted to, fail with one plain
|
|
||||||
// sentence that names where the key was configured.
|
|
||||||
func TestWrongPrivateKeyError(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
store := NewMockStorer()
|
|
||||||
|
|
||||||
const snapshotID = "test-host_2026-01-01T00:00:00Z"
|
|
||||||
|
|
||||||
remoteKey := snapshot.RemoteSnapshotKey(snapshotID)
|
|
||||||
|
|
||||||
manifest, err := snapshot.EncodeManifest(
|
|
||||||
&snapshot.Manifest{SnapshotID: remoteKey}, 3)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NoError(t, store.Put(ctx,
|
|
||||||
"metadata/"+remoteKey+"/manifest.json.zst", bytes.NewReader(manifest)))
|
|
||||||
|
|
||||||
// The database is encrypted to the test recipient only.
|
|
||||||
var db bytes.Buffer
|
|
||||||
|
|
||||||
writer, err := blobgen.NewWriter(&db, 1, []string{testAgePublicKey})
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
_, err = writer.Write([]byte("snapshot database"))
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NoError(t, writer.Close())
|
|
||||||
require.NoError(t, store.Put(ctx,
|
|
||||||
"metadata/"+remoteKey+"/db.zst.age", &db))
|
|
||||||
|
|
||||||
otherKey, err := age.GenerateX25519Identity()
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
v := &vaultik.Vaultik{
|
|
||||||
Config: &config.Config{AgeSecretKey: otherKey.String()},
|
|
||||||
Storage: store,
|
|
||||||
Fs: afero.NewMemMapFs(),
|
|
||||||
Stdout: io.Discard,
|
|
||||||
Stderr: io.Discard,
|
|
||||||
UI: ui.NewWithColor(io.Discard, false),
|
|
||||||
}
|
|
||||||
v.SetContext(ctx)
|
|
||||||
|
|
||||||
// A Config built directly has no recorded source, so the error names
|
|
||||||
// the config-file field.
|
|
||||||
const want = "cannot decrypt this snapshot: the private key in " +
|
|
||||||
"age_secret_key is not one of the keys the backup was encrypted to"
|
|
||||||
|
|
||||||
err = v.Restore(&vaultik.RestoreOptions{
|
|
||||||
SnapshotID: snapshotID,
|
|
||||||
TargetDir: t.TempDir(),
|
|
||||||
})
|
|
||||||
require.EqualError(t, err, want)
|
|
||||||
|
|
||||||
err = v.RunDeepVerify(snapshotID, &vaultik.VerifyOptions{Deep: true})
|
|
||||||
require.EqualError(t, err, want)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestWrongPrivateKeyForBlobError backs up a file, then backs it up again
|
|
||||||
// from an empty local index with a second recipient added. The second
|
|
||||||
// snapshot reuses the stored blob, which stays encrypted to the first
|
|
||||||
// recipient only. With only the added key configured, restore and deep
|
|
||||||
// verify read the database but fail on the blob with one plain sentence
|
|
||||||
// that names where the key was configured.
|
|
||||||
func TestWrongPrivateKeyForBlobError(t *testing.T) {
|
|
||||||
log.Initialize(log.Config{})
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
fs := afero.NewOsFs()
|
|
||||||
tempDir := t.TempDir()
|
|
||||||
dataDir := filepath.Join(tempDir, "source")
|
|
||||||
dataPath := filepath.Join(dataDir, "data.bin")
|
|
||||||
storeDir := t.TempDir()
|
|
||||||
secondDBPath := filepath.Join(tempDir, "second.sqlite")
|
|
||||||
|
|
||||||
chunkSize := int64(64 * 1024)
|
|
||||||
maxBlobSize := int64(512 * 1024)
|
|
||||||
|
|
||||||
require.NoError(t, fs.MkdirAll(dataDir, 0o755))
|
|
||||||
require.NoError(t, afero.WriteFile(fs, dataPath,
|
|
||||||
bytesPattern("key-", int(chunkSize)), 0o644))
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
|
|
||||||
runFileStorageBackup(ctx, t, fs, dataDir, storeDir,
|
|
||||||
filepath.Join(tempDir, "first.sqlite"), chunkSize, maxBlobSize)
|
|
||||||
|
|
||||||
env := setupDedupBackupEnv(
|
|
||||||
ctx, t, fs, storeDir, secondDBPath, chunkSize, maxBlobSize)
|
|
||||||
|
|
||||||
defer func() { _ = env.db.Close() }()
|
|
||||||
|
|
||||||
addedKey, err := age.GenerateX25519Identity()
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
env.cfg.AgeRecipients = append(env.cfg.AgeRecipients,
|
|
||||||
addedKey.Recipient().String())
|
|
||||||
|
|
||||||
snapshotID, result := runDedupSnapshot(ctx, t, env.sm, env.makeScanner(),
|
|
||||||
env.cfg.Hostname, "second", dataDir, secondDBPath)
|
|
||||||
require.Zero(t, result.BlobsUploaded,
|
|
||||||
"the second backup should reuse the stored blob")
|
|
||||||
|
|
||||||
require.NoError(t, env.db.Close())
|
|
||||||
|
|
||||||
blobKeys, err := env.storer.List(ctx, "blobs/")
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Len(t, blobKeys, 1)
|
|
||||||
|
|
||||||
blobHash := path.Base(blobKeys[0])
|
|
||||||
|
|
||||||
env.cfg.AgeSecretKey = addedKey.String()
|
|
||||||
|
|
||||||
v := &vaultik.Vaultik{
|
|
||||||
Config: env.cfg,
|
|
||||||
Storage: env.storer,
|
|
||||||
Fs: fs,
|
|
||||||
Stdout: io.Discard,
|
|
||||||
Stderr: io.Discard,
|
|
||||||
UI: ui.NewWithColor(io.Discard, false),
|
|
||||||
}
|
|
||||||
v.SetContext(ctx)
|
|
||||||
|
|
||||||
const sentence = "cannot decrypt this blob: the private key in " +
|
|
||||||
"age_secret_key is not one of the keys the blob was encrypted to"
|
|
||||||
|
|
||||||
// Restore names the blob by the first 16 characters of its hash.
|
|
||||||
err = v.Restore(&vaultik.RestoreOptions{
|
|
||||||
SnapshotID: snapshotID,
|
|
||||||
TargetDir: t.TempDir(),
|
|
||||||
})
|
|
||||||
require.EqualError(t, err, "restoring "+dataPath+": downloading blob "+
|
|
||||||
blobHash[:16]+": "+sentence+
|
|
||||||
" (pass --skip-errors to continue past restore failures)")
|
|
||||||
|
|
||||||
err = v.RunDeepVerify(snapshotID, &vaultik.VerifyOptions{Deep: true})
|
|
||||||
require.EqualError(t, err,
|
|
||||||
"blob "+blobHash+" verification failed: "+sentence)
|
|
||||||
}
|
|
||||||
@@ -187,17 +187,6 @@ func (e *listEnv) addRemoteRawTimestamp(
|
|||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
remoteKey := snapshot.RemoteSnapshotKey(snapshotID)
|
remoteKey := snapshot.RemoteSnapshotKey(snapshotID)
|
||||||
e.putRemoteManifest(t, remoteKey,
|
|
||||||
encodeRemoteManifest(t, remoteKey, timestamp))
|
|
||||||
|
|
||||||
return remoteKey
|
|
||||||
}
|
|
||||||
|
|
||||||
// encodeRemoteManifest returns the encoded manifest of a one-blob
|
|
||||||
// snapshot whose remote key is remoteKey.
|
|
||||||
func encodeRemoteManifest(t *testing.T, remoteKey, timestamp string) []byte {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
manifest := &snapshot.Manifest{
|
manifest := &snapshot.Manifest{
|
||||||
// Note: the hashed key, never the human ID. That is precisely
|
// Note: the hashed key, never the human ID. That is precisely
|
||||||
// why a remote-only snapshot cannot be named.
|
// why a remote-only snapshot cannot be named.
|
||||||
@@ -213,17 +202,11 @@ func encodeRemoteManifest(t *testing.T, remoteKey, timestamp string) []byte {
|
|||||||
data, err := snapshot.EncodeManifest(manifest, 3)
|
data, err := snapshot.EncodeManifest(manifest, 3)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
return data
|
err = e.store.Put(context.Background(),
|
||||||
}
|
|
||||||
|
|
||||||
// putRemoteManifest stores an encoded manifest on the destination store
|
|
||||||
// under remoteKey.
|
|
||||||
func (e *listEnv) putRemoteManifest(t *testing.T, remoteKey string, data []byte) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
err := e.store.Put(context.Background(),
|
|
||||||
"metadata/"+remoteKey+"/manifest.json.zst", bytes.NewReader(data))
|
"metadata/"+remoteKey+"/manifest.json.zst", bytes.NewReader(data))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
return remoteKey
|
||||||
}
|
}
|
||||||
|
|
||||||
// Fixtures shared across the listing tests.
|
// Fixtures shared across the listing tests.
|
||||||
@@ -808,17 +791,9 @@ func TestListSnapshots_JSONReportsTruncation(t *testing.T) {
|
|||||||
|
|
||||||
timestamp := time.Date(2026, 3, 2, 11, 22, 33, 0, time.UTC)
|
timestamp := time.Date(2026, 3, 2, 11, 22, 33, 0, time.UTC)
|
||||||
|
|
||||||
// One past the cap, so exactly one snapshot is omitted. The listing
|
// One past the cap, so exactly one snapshot is omitted.
|
||||||
// names a snapshot by the key its manifest is stored under, never by
|
|
||||||
// the manifest's own ID. So all of them share one encoded manifest:
|
|
||||||
// encoding 1001 took seconds.
|
|
||||||
data := encodeRemoteManifest(t,
|
|
||||||
snapshot.RemoteSnapshotKey("otherhost_bulk"),
|
|
||||||
timestamp.Format(time.RFC3339))
|
|
||||||
|
|
||||||
for i := range maxRemoteOnlyRowsForTest + 1 {
|
for i := range maxRemoteOnlyRowsForTest + 1 {
|
||||||
env.putRemoteManifest(t, snapshot.RemoteSnapshotKey(
|
env.addRemote(t, fmt.Sprintf("otherhost_bulk_%04d", i), timestamp)
|
||||||
fmt.Sprintf("otherhost_bulk_%04d", i)), data)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
stderr := captureProcessStderr(t, func() {
|
stderr := captureProcessStderr(t, func() {
|
||||||
|
|||||||
@@ -267,10 +267,6 @@ func (v *Vaultik) downloadVerifiedSnapshotDB(
|
|||||||
defer func() { _ = dbReader.Close() }()
|
defer func() { _ = dbReader.Close() }()
|
||||||
|
|
||||||
tdb, err := v.decryptAndLoadDatabase(dbReader, identities)
|
tdb, err := v.decryptAndLoadDatabase(dbReader, identities)
|
||||||
if errors.Is(err, errWrongPrivateKey) {
|
|
||||||
return nil, v.deepVerifyFailure(result, opts, err.Error(), err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, v.deepVerifyFailure(result, opts,
|
return nil, v.deepVerifyFailure(result, opts,
|
||||||
fmt.Sprintf("failed to decrypt database: %v", err),
|
fmt.Sprintf("failed to decrypt database: %v", err),
|
||||||
@@ -355,9 +351,9 @@ func (v *Vaultik) decryptAndLoadDatabase(
|
|||||||
reader io.ReadCloser, identities []age.Identity,
|
reader io.ReadCloser, identities []age.Identity,
|
||||||
) (*tempDB, error) {
|
) (*tempDB, error) {
|
||||||
// Decrypt and decompress through the shared blobgen reader.
|
// Decrypt and decompress through the shared blobgen reader.
|
||||||
blobReader, err := v.newSnapshotDBReader(reader, identities)
|
blobReader, err := blobgen.NewReader(reader, identities...)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, fmt.Errorf("failed to create decryption reader: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
defer func() { _ = blobReader.Close() }()
|
defer func() { _ = blobReader.Close() }()
|
||||||
@@ -430,11 +426,7 @@ func (v *Vaultik) verifyBlob(
|
|||||||
// the plaintext as it is read. A blob's hash — its remote name — is the
|
// the plaintext as it is read. A blob's hash — its remote name — is the
|
||||||
// double SHA-256 of that plaintext (see blobgen.DoubleSHA256), not of the
|
// double SHA-256 of that plaintext (see blobgen.DoubleSHA256), not of the
|
||||||
// encrypted bytes.
|
// encrypted bytes.
|
||||||
blobReader, err := v.newBlobReader(reader, identities)
|
blobReader, err := blobgen.NewReader(reader, identities...)
|
||||||
if errors.Is(err, errWrongPrivateKeyForBlob) {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to create blob reader: %w", err)
|
return fmt.Errorf("failed to create blob reader: %w", err)
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in new issue
Block a user