Compare commits

..
Author SHA1 Message Date
sneak c574229a01 Make config set create a list for an index under a missing key (closes #308)
check / check (push) Waiting to run
`config set age_recipients.0 age1...` on a file without `age_recipients`
wrote a map keyed `0`, after which no command could load the config. A
missing key is now created as a list when the next path element is a
number.

`config set` checks its result with the config loader before writing and
refuses a failure the change brings in: a config that loaded must still
load, and one still being filled in (failing validation until complete)
must still decode into the config type. The new `config.Parse`, split out
of `config.Load`, decodes config data, filling in `${...}` references,
without validating.

Rule suppressed: gosec G304 where `config.Load` and the new tests read a
file.

Model: opus-5-5
2026-10-13 14:09:02 +00:00
17 changed files with 387 additions and 525 deletions
+4 -4
View File
@@ -279,8 +279,8 @@ Key methods:
- `Scan(ctx, path, snapshotID)` → Main scan operation
Scan phases:
1. **Phase 1**: Walk directory, identify files needing processing
2. **Phase 1b**: Count as deleted the files that the previous snapshot of the same name lists and the walk did not find; a file that is excluded, or inside an excluded directory, is not counted
1. **Phase 0**: Detect deleted files from previous snapshots
2. **Phase 1**: Walk directory, identify files needing processing
3. **Phase 2**: Process files (chunk → pack → upload)
#### SnapshotManager
@@ -320,13 +320,13 @@ CreateSnapshot(opts)
│ │
│ ├─► scanner.Scan(ctx, path, snapshotID)
│ │ │
│ │ ├─► Phase 0: detectDeletedFiles()
│ │ │
│ │ ├─► Phase 1: scanPhase()
│ │ │ Walk directory
│ │ │ Check file metadata changes
│ │ │ Build list of files to process
│ │ │
│ │ ├─► Phase 1b: detectDeletedFiles()
│ │ │
│ │ └─► Phase 2: processPhase()
│ │ For each file:
│ │ chunker.ChunkReaderStreaming()
+10 -2
View File
@@ -271,8 +271,16 @@ existing file. Created with mode `0600` since it will contain credentials.
**`config set`**: Set a scalar config value by dotted YAML path
(e.g. `vaultik config set compression_level 9`,
`vaultik config set storage_url "file:///mnt/backups"`). Comments and
formatting in the file are preserved; intermediate maps are created as
needed.
formatting in the file are preserved. Missing keys along the path are
created: as a list when the next path element is a number (so
`vaultik config set age_recipients.0 age1...` works on a file without
`age_recipients`), otherwise as a map. If the config loads, a change
after which it would not, such as `vaultik config set compression_level 99`,
is refused and the file is left unchanged. A config that does not load
yet, such as the one `config init` writes before `storage_url` is set,
can be filled in one setting at a time: only a change that gives a
setting the wrong type, such as a single value where a list belongs or a
word where a number belongs, is refused.
**`snapshot create`**: Perform incremental backup of configured snapshots.
* Optional snapshot names argument to create specific snapshots (default: all)
+74 -24
View File
@@ -44,6 +44,9 @@ var (
errNeedNumericIndex = errors.New("key is a list; use a numeric index")
errIndexOutOfRange = errors.New("index out of range")
errNotMapOrList = errors.New("key is not a map or list")
errWouldNotLoad = errors.New(
"vaultik would not load the config after this change, " +
"so the file was not changed")
)
const defaultConfigTemplate = `# vaultik configuration
@@ -391,8 +394,15 @@ func newConfigSetCommand() *cobra.Command {
Use: "set <key> <value>",
Short: "Set a config value by dotted path (e.g. compression_level 5)",
Long: `Sets a scalar config value addressed by dotted YAML path and writes
the file back, preserving comments and formatting. Intermediate maps
are created as needed.
the file back, preserving comments and formatting. Missing keys along
the path are created: as a list when the next path element is a
number, otherwise as a map.
If the config loads, a change after which it would not, such as
compression_level 99, is refused and the file is left unchanged. A
config that does not load yet, such as the one config init writes
before storage_url is set, can be filled in one setting at a time:
only a change that gives a setting the wrong type is refused.
Examples:
vaultik config set storage_url "file:///mnt/backups"
@@ -422,6 +432,11 @@ func writeConfigSet(out *ui.Writer, path, key, value string) error {
return err
}
oldData, err := marshalConfigYAML(root)
if err != nil {
return fmt.Errorf("marshaling config: %w", err)
}
err = yamlPathSet(root, strings.Split(key, "."), value)
if err != nil {
return err
@@ -432,6 +447,11 @@ func writeConfigSet(out *ui.Writer, path, key, value string) error {
return fmt.Errorf("marshaling config: %w", err)
}
err = checkConfigChange(oldData, data)
if err != nil {
return fmt.Errorf("%w: %w", errWouldNotLoad, err)
}
err = os.WriteFile(path, data, configFileMode)
if err != nil {
return fmt.Errorf("writing config file: %w", err)
@@ -453,6 +473,28 @@ func writeConfigSet(out *ui.Writer, path, key, value string) error {
return nil
}
// checkConfigChange returns the reason vaultik would not load newData, the
// config after a change, if the change brings that failure in. If oldData,
// the config before it, decodes into the config type, the new one must
// too; if the old one also passes validation, the new one must pass it. A
// config still being filled in fails validation until it is complete, so
// only decoding is checked for it.
func checkConfigChange(oldData, newData []byte) error {
oldCfg, oldErr := config.Parse(oldData)
if oldErr == nil {
newCfg, err := config.Parse(newData)
if err != nil {
return err
}
if oldCfg.Validate() == nil {
return newCfg.Validate()
}
}
return nil
}
// marshalConfigYAML renders a config document tree with 2-space indentation,
// matching defaultConfigTemplate. yaml.Marshal defaults to 4 spaces, which
// would reindent the whole file on the first `config set` despite the promise
@@ -559,8 +601,8 @@ func yamlPathGet(root *yaml.Node, keys []string) (*yaml.Node, error) {
return node, nil
}
// yamlPathSet navigates a dotted key path, creating intermediate maps as
// needed, and sets the final key to the given scalar value. Numeric path
// yamlPathSet navigates a dotted key path, creating missing keys as needed,
// and sets the final key to the given scalar value. Numeric path
// components index into sequences; an index equal to the sequence length
// appends a new element (e.g. "age_recipients.1" on a 1-element list).
func yamlPathSet(root *yaml.Node, keys []string, value string) error {
@@ -573,12 +615,12 @@ func yamlPathSet(root *yaml.Node, keys []string, value string) error {
node = node.Content[0]
}
for i, key := range keys {
for i := range keys {
last := i == len(keys)-1
switch node.Kind {
case yaml.MappingNode:
node = yamlSetInMapping(node, key, value, last)
node = yamlSetInMapping(node, keys, i, value, last)
case yaml.SequenceNode:
next, err := yamlSetInSequence(node, keys, i, value, last)
if err != nil {
@@ -647,14 +689,16 @@ func yamlField(typ reflect.Type, key string) (reflect.StructField, bool) {
return reflect.StructField{}, false
}
// yamlSetInMapping resolves (creating if needed) the value node for key
// within a mapping node, setting it to value when it is the final path
// element, and returns the node to descend into.
func yamlSetInMapping(node *yaml.Node, key, value string, last bool) *yaml.Node {
// yamlSetInMapping resolves (creating if needed) the value node for the
// path element keys[i] within a mapping node, setting it to value when it
// is the final path element, and returns the node to descend into.
func yamlSetInMapping(
node *yaml.Node, keys []string, i int, value string, last bool,
) *yaml.Node {
var valueNode *yaml.Node
for j := 0; j+1 < len(node.Content); j += 2 {
if node.Content[j].Value == key {
if node.Content[j].Value == keys[i] {
valueNode = node.Content[j+1]
break
@@ -662,13 +706,8 @@ func yamlSetInMapping(node *yaml.Node, key, value string, last bool) *yaml.Node
}
if valueNode == nil {
keyNode := &yaml.Node{Kind: yaml.ScalarNode, Value: key}
valueNode = &yaml.Node{Kind: yaml.MappingNode}
if last {
valueNode = &yaml.Node{Kind: yaml.ScalarNode, Value: value}
}
keyNode := &yaml.Node{Kind: yaml.ScalarNode, Value: keys[i]}
valueNode = newPathNode(keys, i, value)
node.Content = append(node.Content, keyNode, valueNode)
} else if last {
setScalar(valueNode, value)
@@ -696,12 +735,7 @@ func yamlSetInSequence(
}
if idx == len(node.Content) {
newNode := &yaml.Node{Kind: yaml.MappingNode}
if last {
newNode = &yaml.Node{Kind: yaml.ScalarNode, Value: value}
}
node.Content = append(node.Content, newNode)
node.Content = append(node.Content, newPathNode(keys, i, value))
} else if last {
setScalar(node.Content[idx], value)
}
@@ -709,6 +743,22 @@ func yamlSetInSequence(
return node.Content[idx], nil
}
// newPathNode returns the node to create for the missing path element
// keys[i]: the value when it is the final path element, otherwise an empty
// list when the next path element is a number and an empty map when not.
func newPathNode(keys []string, i int, value string) *yaml.Node {
if i == len(keys)-1 {
return &yaml.Node{Kind: yaml.ScalarNode, Value: value}
}
_, err := strconv.Atoi(keys[i+1])
if err == nil {
return &yaml.Node{Kind: yaml.SequenceNode}
}
return &yaml.Node{Kind: yaml.MappingNode}
}
// setScalar overwrites a node in place with a plain scalar value.
func setScalar(n *yaml.Node, value string) {
n.Kind = yaml.ScalarNode
+176 -5
View File
@@ -58,14 +58,16 @@ func TestDefaultConfigTemplateParses(t *testing.T) {
}
}
// testRecipient is a valid age public key.
const testRecipient = "age1278m9q7dp3chsh2dcy82qk27v047zywyvtxwnj4cvt0z65jw6a7q5dqhfj"
// TestConfigSetRecipientOnFreshConfig follows the README quickstart: on the
// file `config init` writes, `config set age_recipients.0` and
// `config set storage_url` give a config that loads with that recipient.
// Neither is refused, although the config does not load until both are set.
func TestConfigSetRecipientOnFreshConfig(t *testing.T) {
t.Parallel()
const recipient = "age1278m9q7dp3chsh2dcy82qk27v047zywyvtxwnj4cvt0z65jw6a7q5dqhfj"
path := filepath.Join(t.TempDir(), "config.yml")
err := os.WriteFile(path, []byte(defaultConfigTemplate), configFileMode)
@@ -75,7 +77,7 @@ func TestConfigSetRecipientOnFreshConfig(t *testing.T) {
out := ui.NewWithColor(&bytes.Buffer{}, false)
err = writeConfigSet(out, path, "age_recipients.0", recipient)
err = writeConfigSet(out, path, "age_recipients.0", testRecipient)
if err != nil {
t.Fatalf("config set age_recipients.0: %v", err)
}
@@ -90,8 +92,177 @@ func TestConfigSetRecipientOnFreshConfig(t *testing.T) {
t.Fatalf("config.Load: %v", err)
}
if len(cfg.AgeRecipients) != 1 || cfg.AgeRecipients[0] != recipient {
t.Errorf("age_recipients = %v, want [%s]", cfg.AgeRecipients, recipient)
if len(cfg.AgeRecipients) != 1 || cfg.AgeRecipients[0] != testRecipient {
t.Errorf("age_recipients = %v, want [%s]", cfg.AgeRecipients, testRecipient)
}
}
// configWithoutRecipients loads, but has no age_recipients key and no
// snapshot named work.
const configWithoutRecipients = `storage_url: file:///mnt/backups
snapshots:
home:
paths:
- "~"
`
// TestConfigSetIndexUnderMissingKey checks that a list index under a key
// the file does not have creates a list, and that the result loads.
func TestConfigSetIndexUnderMissingKey(t *testing.T) {
t.Parallel()
tests := []struct {
key string
value string
field func(cfg *config.Config) []string
}{
{"age_recipients.0", testRecipient,
func(cfg *config.Config) []string { return cfg.AgeRecipients }},
{"snapshots.work.paths.0", "/srv/work",
func(cfg *config.Config) []string { return cfg.Snapshots["work"].Paths }},
}
for _, tt := range tests {
t.Run(tt.key, func(t *testing.T) {
t.Parallel()
path := filepath.Join(t.TempDir(), "config.yml")
err := os.WriteFile(path, []byte(configWithoutRecipients), configFileMode)
if err != nil {
t.Fatalf("write config: %v", err)
}
err = writeConfigSet(ui.NewWithColor(&bytes.Buffer{}, false),
path, tt.key, tt.value)
if err != nil {
t.Fatalf("config set %s: %v", tt.key, err)
}
cfg, err := config.Load(path)
if err != nil {
t.Fatalf("config.Load: %v", err)
}
got := tt.field(cfg)
if len(got) != 1 || got[0] != tt.value {
t.Errorf("%s gives %v, want [%s]", tt.key, got, tt.value)
}
})
}
}
// TestConfigSetKeepsFileThatWouldNotLoad checks that config set refuses a
// change after which the config would not load, with an error giving the
// reason, and leaves the file as it was. On the file `config init` writes,
// which does not load until storage_url is set, only a value of the wrong
// type is refused.
func TestConfigSetKeepsFileThatWouldNotLoad(t *testing.T) {
t.Parallel()
tests := []struct {
config string
key string
value string
reason string
}{
// Index 1 of the empty list the missing key gives.
{configWithoutRecipients, "age_recipients.1", testRecipient,
"index out of range"},
// A list set to a single value.
{configWithoutRecipients, "snapshots.home.paths", "/srv",
"cannot unmarshal"},
{configWithoutRecipients, "compression_level", "fast",
"cannot unmarshal"},
// Values of the right type that fail validation.
{configWithoutRecipients, "compression_level", "99",
"compression_level must be between 1 and 19"},
{configWithoutRecipients, "chunk_size", "1KB",
"chunk_size must be at least 1MB"},
{configWithoutRecipients, "age_recipients.0", "notakey",
"not a valid recipient"},
// The type check on a config that does not load yet.
{defaultConfigTemplate, "snapshots.apps.paths", "/srv",
"cannot unmarshal"},
}
for _, tt := range tests {
t.Run(tt.key+" "+tt.value, func(t *testing.T) {
t.Parallel()
path := filepath.Join(t.TempDir(), "config.yml")
err := os.WriteFile(path, []byte(tt.config), configFileMode)
if err != nil {
t.Fatalf("write config: %v", err)
}
err = writeConfigSet(ui.NewWithColor(&bytes.Buffer{}, false),
path, tt.key, tt.value)
if err == nil || !strings.Contains(err.Error(), tt.reason) {
t.Errorf("config set %s %s gives error %v, want one saying %q",
tt.key, tt.value, err, tt.reason)
}
data, err := os.ReadFile(path) //nolint:gosec // G304: test-controlled path
if err != nil {
t.Fatalf("read config: %v", err)
}
if string(data) != tt.config {
t.Errorf("config set %s changed the file to:\n%s",
tt.key, data)
}
})
}
}
// TestConfigSetRefusesOnlyItsOwnFailure checks that setting another key
// works on a config with a ${...} reference on compression_level, which
// loads because loading fills the reference in, and on one whose
// compression_level already fails to load.
func TestConfigSetRefusesOnlyItsOwnFailure(t *testing.T) {
t.Parallel()
levelPath := filepath.Join(t.TempDir(), "level")
err := os.WriteFile(levelPath, []byte("9"), configFileMode)
if err != nil {
t.Fatalf("write level: %v", err)
}
configs := map[string]string{
"reference": configWithoutRecipients +
"compression_level: ${FILE:" + levelPath + "}\n",
"failing": configWithoutRecipients + "compression_level: fast\n",
}
for name, configYAML := range configs {
t.Run(name, func(t *testing.T) {
t.Parallel()
path := filepath.Join(t.TempDir(), "config.yml")
err := os.WriteFile(path, []byte(configYAML), configFileMode)
if err != nil {
t.Fatalf("write config: %v", err)
}
err = writeConfigSet(ui.NewWithColor(&bytes.Buffer{}, false),
path, "hostname", "myhost")
if err != nil {
t.Fatalf("config set hostname: %v", err)
}
data, err := os.ReadFile(path) //nolint:gosec // G304: test-controlled path
if err != nil {
t.Fatalf("read config: %v", err)
}
if !strings.Contains(string(data), "hostname: myhost") {
t.Errorf("config set hostname wrote:\n%s", data)
}
})
}
}
+41 -24
View File
@@ -4,6 +4,7 @@
package config
import (
"bytes"
"errors"
"fmt"
"os"
@@ -233,14 +234,48 @@ func New(path Path) (*Config, error) {
return cfg, nil
}
// Load reads and parses the configuration file from the specified path.
// It applies default values for optional fields, performs environment variable
// substitution using smartconfig, and validates the configuration.
// The configuration file should be in YAML format. Returns an error if the file
// cannot be read, parsed, or if validation fails.
// Load reads the configuration file at path, parses it with Parse, warns
// when others can read the file, and validates the configuration.
// Returns an error if the file cannot be read, parsed, or if validation
// fails.
func Load(path string) (*Config, error) {
data, err := os.ReadFile(path) //nolint:gosec // G304: config path is operator-supplied
if err != nil {
return nil, fmt.Errorf("failed to load config file: %w", err)
}
cfg, err := Parse(data)
if err != nil {
return nil, err
}
// Check config file permissions (warn if world or group readable)
//nolint:gosec // G703: config path is operator-supplied by design
info, statErr := os.Stat(path)
if statErr == nil {
mode := info.Mode().Perm()
if mode&0044 != 0 { // group or world readable
log.Warn(cfg.readableByOthersWarning(),
"path", path,
"mode", fmt.Sprintf("%04o", mode),
"recommendation", "chmod 600 "+path)
}
}
err = cfg.Validate()
if err != nil {
return nil, fmt.Errorf("invalid config: %w", err)
}
return cfg, nil
}
// Parse reads YAML config data as Load does, without validating it. It
// applies default values for optional fields, fills in ${...} references
// using smartconfig, and expands ~ in paths.
func Parse(data []byte) (*Config, error) {
// Load config using smartconfig for interpolation
sc, err := smartconfig.NewFromConfigPath(path)
sc, err := smartconfig.NewFromReader(bytes.NewReader(data))
if err != nil {
return nil, fmt.Errorf("failed to load config file: %w", err)
}
@@ -302,24 +337,6 @@ func Load(path string) (*Config, error) {
cfg.S3.Region = "us-east-1"
}
// Check config file permissions (warn if world or group readable)
//nolint:gosec // G703: config path is operator-supplied by design
info, statErr := os.Stat(path)
if statErr == nil {
mode := info.Mode().Perm()
if mode&0044 != 0 { // group or world readable
log.Warn(cfg.readableByOthersWarning(),
"path", path,
"mode", fmt.Sprintf("%04o", mode),
"recommendation", "chmod 600 "+path)
}
}
err = cfg.Validate()
if err != nil {
return nil, fmt.Errorf("invalid config: %w", err)
}
return cfg, nil
}
-44
View File
@@ -277,50 +277,6 @@ func (r *FileRepository) ListUnderPath(
return files, rows.Err()
}
// ListUnderPathInSnapshot returns the files that ListUnderPath returns
// for path and that the snapshot lists, in no particular order.
func (r *FileRepository) ListUnderPathInSnapshot(
ctx context.Context, path, snapshotID string,
) ([]*File, error) {
path = strings.TrimRight(path, "/")
dirPrefix := path + "/"
query := `
SELECT f.id, f.path, f.source_path, f.mtime, f.mtime_nsec, f.size,
f.mode, f.uid, f.gid, f.link_target
FROM files f
JOIN snapshot_files sf ON sf.file_id = f.id
WHERE sf.snapshot_id = ?
AND (f.path = ? OR substr(f.path, 1, length(?)) = ?)
`
rows, err := r.db.conn.QueryContext(ctx, query,
snapshotID, path, dirPrefix, dirPrefix)
if err != nil {
return nil, fmt.Errorf("querying files: %w", err)
}
defer func() {
err := rows.Close()
if err != nil {
Fatalf("failed to close rows: %v", err)
}
}()
var files []*File
for rows.Next() {
file, err := r.scanFileRows(rows)
if err != nil {
return nil, fmt.Errorf("scanning file: %w", err)
}
files = append(files, file)
}
return files, rows.Err()
}
// ListIDsWithChunksNotInUploadedBlobs returns the IDs of the files whose
// path starts with prefix and that list at least one chunk held by no
// blob whose upload has completed (uploaded_ts set). A new snapshot
-59
View File
@@ -212,65 +212,6 @@ func TestFileRepositoryListUnderPath(t *testing.T) {
}
}
// ListUnderPathInSnapshot leaves out files outside path and files the
// snapshot does not list, though another snapshot lists them.
func TestFileRepositoryListUnderPathInSnapshot(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
files := database.NewFileRepository(db)
snapshots := database.NewSnapshotRepository(db)
listed := map[string][]string{
"one": {"/src/a.txt", "/src2/c.txt"},
"two": {"/src/b.txt"},
}
for snapshotID, paths := range listed {
err := snapshots.Create(ctx, nil, &database.Snapshot{
ID: types.SnapshotID(snapshotID),
Hostname: "host",
StartedAt: time.Now(),
})
if err != nil {
t.Fatalf("failed to create snapshot %s: %v", snapshotID, err)
}
for _, path := range paths {
err := files.Create(ctx, nil, &database.File{
Path: types.FilePath(path),
MTime: time.Now().Truncate(time.Second),
Mode: 0644,
})
if err != nil {
t.Fatalf("failed to create %s: %v", path, err)
}
err = snapshots.AddFile(ctx, nil, snapshotID, path)
if err != nil {
t.Fatalf("failed to add %s to %s: %v", path, snapshotID, err)
}
}
}
found, err := files.ListUnderPathInSnapshot(ctx, "/src", "one")
if err != nil {
t.Fatalf("failed to list files: %v", err)
}
got := make([]string, 0, len(found))
for _, f := range found {
got = append(got, f.Path.String())
}
if !slices.Equal(got, []string{"/src/a.txt"}) {
t.Errorf("got %q, want only /src/a.txt", got)
}
}
func TestFileRepositorySymlink(t *testing.T) {
t.Parallel()
+10 -7
View File
@@ -10,8 +10,6 @@ import (
"sync"
"time"
"unicode"
"github.com/dustin/go-humanize"
)
// groupSeparator joins an open group path to an attribute key. This
@@ -306,13 +304,18 @@ func formatDuration(d time.Duration) string {
}
}
// formatBytes formats a byte count with humanize.Bytes, as every command
// prints one. humanize.Bytes takes no negative count, so one is printed
// as it is.
// formatBytes formats bytes in a human-readable way
func formatBytes(b int64) string {
if b < 0 {
const unit = 1024
if b < unit {
return fmt.Sprintf("%d B", b)
}
return humanize.Bytes(uint64(b))
div, exp := int64(unit), 0
for n := b / unit; n >= unit; n /= unit {
div *= unit
exp++
}
return fmt.Sprintf("%.1f %cB", float64(b)/float64(div), "KMGTPE"[exp])
}
+4 -5
View File
@@ -200,8 +200,7 @@ func TestTTYHandlerWithGroupQualifiesKeys(t *testing.T) {
func TestTTYHandlerByteFormattingSurvivesGrouping(t *testing.T) {
t.Parallel()
// Printed as "1.0 MB", in powers of 1000 like every other byte count.
const oneMegabyte = 1_000_000
const oneAndAHalfKiB = 1536
for name, testCase := range map[string]struct {
derive func(*slog.Logger) *slog.Logger
@@ -224,13 +223,13 @@ func TestTTYHandlerByteFormattingSurvivesGrouping(t *testing.T) {
var buf bytes.Buffer
logger := slog.New(log.NewTTYHandler(&buf, debugHandlerOptions()))
testCase.derive(logger).Info("uploaded", "bytes", oneMegabyte)
testCase.derive(logger).Info("uploaded", "bytes", oneAndAHalfKiB)
line := ansiEscape.ReplaceAllString(buf.String(), "")
assert.Contains(t, line, testCase.key+"=1.0 MB",
assert.Contains(t, line, testCase.key+"=1.5 KB",
"a byte count must be human-readable however it is qualified")
assert.NotContains(t, line, strconv.Itoa(oneMegabyte),
assert.NotContains(t, line, strconv.Itoa(oneAndAHalfKiB),
"the raw number must not survive the formatting")
})
}
+11 -15
View File
@@ -16,9 +16,6 @@ type ScannerParams struct {
Fs afero.Fs
Exclude []string // Exclude patterns (combined global + snapshot-specific)
SkipErrors bool // Skip file read errors (log loudly but continue)
// PreviousSnapshotID is the most recent completed snapshot of the
// same name, "" for none.
PreviousSnapshotID string
}
// Module exports backup functionality as an fx module.
@@ -47,18 +44,17 @@ func provideScannerFactory(
}
return NewScanner(ScannerConfig{
FS: params.Fs,
ChunkSize: cfg.ChunkSize.Int64(),
Repositories: repos,
Storage: storer,
MaxBlobSize: cfg.BlobSizeLimit.Int64(),
CompressionLevel: cfg.CompressionLevel,
AgeRecipients: cfg.AgeRecipients,
EnableProgress: params.EnableProgress,
UI: params.UI,
Exclude: excludes,
SkipErrors: params.SkipErrors,
PreviousSnapshotID: params.PreviousSnapshotID,
FS: params.Fs,
ChunkSize: cfg.ChunkSize.Int64(),
Repositories: repos,
Storage: storer,
MaxBlobSize: cfg.BlobSizeLimit.Int64(),
CompressionLevel: cfg.CompressionLevel,
AgeRecipients: cfg.AgeRecipients,
EnableProgress: params.EnableProgress,
UI: params.UI,
Exclude: excludes,
SkipErrors: params.SkipErrors,
})
}
}
+19 -51
View File
@@ -58,10 +58,6 @@ type Scanner struct {
compressionLevel int
ageRecipient string
snapshotID string // Current snapshot being processed
// previousSnapshotID is the most recent completed snapshot of the
// same name, "" for none. Only a file it lists can be counted as
// deleted.
previousSnapshotID string
// currentSourcePath is the source directory being scanned, stored with
// each file record.
currentSourcePath string
@@ -133,9 +129,6 @@ type ScannerConfig struct {
// SkipErrors skips files that cannot be opened or read (log loudly but
// continue); packer, database, encryption, and upload errors still abort.
SkipErrors bool
// PreviousSnapshotID is the most recent completed snapshot of the
// same name, "" for none.
PreviousSnapshotID string
}
// ScanResult contains the results of a scan operation. Files and bytes
@@ -211,7 +204,6 @@ func NewScanner(cfg ScannerConfig) *Scanner {
compiledExclude: compiledExclude,
progress: progress,
skipErrors: cfg.SkipErrors,
previousSnapshotID: cfg.PreviousSnapshotID,
ui: uiw,
pendingChunkHashes: make(map[string]struct{}),
}
@@ -263,9 +255,8 @@ func (s *Scanner) Scan(
filesToProcess := scanResult.FilesToProcess
// Phase 1b: Detect deleted files by comparing the previous snapshot
// of this name against scanned files
err = s.detectDeletedFiles(ctx, path, existingFiles, result)
// Phase 1b: Detect deleted files by comparing DB against scanned files
err = s.detectDeletedFilesFromMap(ctx, knownFiles, existingFiles, result)
if err != nil {
return nil, fmt.Errorf("detecting deleted files: %w", err)
}
@@ -445,8 +436,9 @@ func (s *Scanner) summarizeScanPhase(
}
// loadKnownFiles loads the known files at and beneath path from the
// database into a map for fast lookup. This avoids per-file database
// queries during the scan phase.
// database into a map for fast lookup. Every loaded file the scan does
// not find is counted as deleted. This avoids per-file database queries
// during the scan phase.
func (s *Scanner) loadKnownFiles(
ctx context.Context, path string,
) (map[string]*database.File, error) {
@@ -1946,28 +1938,21 @@ func (s *Scanner) queueFileForBatchInsert(
})
}
// detectDeletedFiles counts as deleted each file at or beneath path that
// the previous snapshot of this name lists and the walk did not find.
// A file that is excluded, or inside an excluded directory, is not
// counted: the walk skips it whether or not it is still on disk.
func (s *Scanner) detectDeletedFiles(
// detectDeletedFilesFromMap finds files that existed in previous snapshots
// but no longer exist. Uses pre-loaded maps to avoid any filesystem or
// database access.
func (s *Scanner) detectDeletedFilesFromMap(
ctx context.Context,
path string,
knownFiles map[string]*database.File,
existingFiles map[string]struct{},
result *ScanResult,
) error {
if s.previousSnapshotID == "" {
if len(knownFiles) == 0 {
return nil
}
previousFiles, err := s.repos.Files.ListUnderPathInSnapshot(
ctx, path, s.previousSnapshotID)
if err != nil {
return fmt.Errorf("listing files of snapshot %s: %w",
s.previousSnapshotID, err)
}
for _, file := range previousFiles {
// Check each known file against the enumerated set (no filesystem access needed)
for path, file := range knownFiles {
// Check context cancellation periodically
select {
case <-ctx.Done():
@@ -1975,16 +1960,13 @@ func (s *Scanner) detectDeletedFiles(
default:
}
filePath := file.Path.String()
_, exists := existingFiles[filePath]
if exists || s.excludedOrInExcludedDir(filePath, path) {
continue
// Check if the file exists in our enumerated set
if _, exists := existingFiles[path]; !exists {
// File has been deleted
result.FilesDeleted++
result.BytesDeleted += file.Size
log.Debug("Detected deleted file", "path", path, "size", file.Size)
}
result.FilesDeleted++
result.BytesDeleted += file.Size
log.Debug("Detected deleted file", "path", filePath, "size", file.Size)
}
if result.FilesDeleted > 0 {
@@ -1995,20 +1977,6 @@ func (s *Scanner) detectDeletedFiles(
return nil
}
// excludedOrInExcludedDir reports whether filePath, which is at or
// beneath rootPath, or any directory between rootPath and it, matches an
// exclude pattern. A pattern such as */.local/share/Trash matches the
// directory but not the files inside it, which the walk skips too.
func (s *Scanner) excludedOrInExcludedDir(filePath, rootPath string) bool {
for p := filePath; len(p) > len(rootPath); p = filepath.Dir(p) {
if s.shouldExclude(p, rootPath) {
return true
}
}
return false
}
// wrapPermissionError augments permission errors with platform-specific
// remediation instructions. On macOS, TCC-protected directories (Calendars,
// Reminders, Photos, etc.) return EPERM unless the running application has
+11 -12
View File
@@ -661,18 +661,17 @@ func faultScannerFactory(
) snapshot.ScannerFactory {
return func(params snapshot.ScannerParams) *snapshot.Scanner {
return snapshot.NewScanner(snapshot.ScannerConfig{
FS: params.Fs,
Storage: storer,
ChunkSize: faultChunkSize,
MaxBlobSize: faultMaxBlobSize,
CompressionLevel: cfg.CompressionLevel,
AgeRecipients: cfg.AgeRecipients,
Repositories: repos,
EnableProgress: params.EnableProgress,
UI: params.UI,
Exclude: params.Exclude,
SkipErrors: params.SkipErrors,
PreviousSnapshotID: params.PreviousSnapshotID,
FS: params.Fs,
Storage: storer,
ChunkSize: faultChunkSize,
MaxBlobSize: faultMaxBlobSize,
CompressionLevel: cfg.CompressionLevel,
AgeRecipients: cfg.AgeRecipients,
Repositories: repos,
EnableProgress: params.EnableProgress,
UI: params.UI,
Exclude: params.Exclude,
SkipErrors: params.SkipErrors,
})
}
}
+16
View File
@@ -89,6 +89,22 @@ type SnapshotInfo struct {
RemotePresent *bool `json:"remote_present"`
}
// formatBytes formats bytes in a human-readable format
func formatBytes(bytes int64) string {
const unit = 1024
if bytes < unit {
return fmt.Sprintf("%d B", bytes)
}
div, exp := int64(unit), 0
for n := bytes / unit; n >= unit; n /= unit {
div *= unit
exp++
}
return fmt.Sprintf("%.1f %cB", float64(bytes)/float64(div), "KMGTPE"[exp])
}
// parseSnapshotTimestamp extracts the timestamp from a snapshot ID
// Format: hostname_snapshotname_2026-01-12T14:41:15Z
func parseSnapshotTimestamp(snapshotID string) (time.Time, error) {
+6 -30
View File
@@ -240,18 +240,12 @@ func (v *Vaultik) createNamedSnapshot(
return err
}
previousSnapshotID, err := v.previousSnapshotID(snapName)
if err != nil {
return err
}
scanner := v.ScannerFactory(snapshot.ScannerParams{
EnableProgress: !opts.Cron,
UI: v.UI,
Fs: v.Fs,
Exclude: v.Config.GetExcludes(snapName),
SkipErrors: opts.SkipErrors,
PreviousSnapshotID: previousSnapshotID,
EnableProgress: !opts.Cron,
UI: v.UI,
Fs: v.Fs,
Exclude: v.Config.GetExcludes(snapName),
SkipErrors: opts.SkipErrors,
})
snapshotID, err := v.SnapshotManager.CreateSnapshotWithName(
@@ -286,24 +280,6 @@ func (v *Vaultik) createNamedSnapshot(
return nil
}
// previousSnapshotID returns the ID of the most recent completed snapshot
// named snapName in the local index, or "" when there is none.
func (v *Vaultik) previousSnapshotID(snapName string) (string, error) {
snapshots, err := v.Repositories.Snapshots.ListRecent(v.ctx, listRecentLimit)
if err != nil {
return "", fmt.Errorf("listing snapshots: %w", err)
}
for _, s := range snapshots {
if s.CompletedAt != nil &&
parseSnapshotName(s.ID.String(), s.Hostname.String()) == snapName {
return s.ID.String(), nil
}
}
return "", nil
}
// resolveSnapshotPaths resolves source directories to absolute paths
// with symlink resolution.
func (v *Vaultik) resolveSnapshotPaths(snapName string) ([]string, error) {
@@ -657,7 +633,7 @@ func (v *Vaultik) confirmAndExecutePurge(
v.stdoutf(" %s (%s, %s)\n",
snap.ID,
snap.Timestamp.Format("2006-01-02 15:04:05"),
ubytes(snap.CompressedSize))
formatBytes(snap.CompressedSize))
}
}
@@ -1,196 +0,0 @@
package vaultik_test
import (
"bytes"
"context"
"fmt"
"os"
"path/filepath"
"testing"
"time"
"github.com/spf13/afero"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"sneak.berlin/go/vaultik/internal/config"
"sneak.berlin/go/vaultik/internal/database"
"sneak.berlin/go/vaultik/internal/log"
"sneak.berlin/go/vaultik/internal/storage"
"sneak.berlin/go/vaultik/internal/ui"
"sneak.berlin/go/vaultik/internal/vaultik"
)
// These tests cover https://git.eeqj.de/sneak/vaultik/issues/304: a
// backup counts a file as deleted only when the previous snapshot of
// the same name had it and it is gone, and never counts an excluded
// file.
// deletedCountEnv is a backup setup over the source directory src whose
// user-facing output is kept in out. Each test configures its snapshots
// with setSnapshot.
type deletedCountEnv struct {
v *vaultik.Vaultik
out *bytes.Buffer
src string
}
// newDeletedCountEnv writes files, keyed by path relative to src.
func newDeletedCountEnv(t *testing.T, files map[string][]byte) *deletedCountEnv {
t.Helper()
ctx := context.Background()
fs := afero.NewOsFs()
tempDir := t.TempDir()
srcDir := filepath.Join(tempDir, "src")
for rel, content := range files {
path := filepath.Join(srcDir, rel)
require.NoError(t, fs.MkdirAll(filepath.Dir(path), 0o755))
require.NoError(t, afero.WriteFile(fs, path, content, 0o644))
}
cfg := faultTestConfig()
cfg.IndexPath = filepath.Join(tempDir, "index.sqlite")
cfg.Snapshots = map[string]config.SnapshotConfig{}
store, err := storage.NewFileStorer(filepath.Join(tempDir, "remote"))
require.NoError(t, err)
db, err := database.New(ctx, cfg.IndexPath)
require.NoError(t, err)
t.Cleanup(func() { _ = db.Close() })
repos := database.NewRepositories(db)
out := &bytes.Buffer{}
v := newBackupVaultik(ctx, cfg, store, repos, db, fs)
v.UI = ui.NewWithColor(out, false)
return &deletedCountEnv{v: v, out: out, src: srcDir}
}
// setSnapshot configures the snapshot name to back up the path rel
// beneath src, with the given excludes.
func (e *deletedCountEnv) setSnapshot(name, rel string, exclude ...string) {
e.v.Config.Snapshots[name] = config.SnapshotConfig{
Paths: []string{filepath.Join(e.src, rel)},
Exclude: exclude,
}
}
// backUp runs a backup of the named snapshot and returns its output.
func (e *deletedCountEnv) backUp(t *testing.T, name string) string {
t.Helper()
e.out.Reset()
require.NoError(t, e.v.CreateSnapshot(&vaultik.SnapshotCreateOptions{
Snapshots: []string{name},
}))
return e.out.String()
}
// A file this snapshot excludes is not counted as deleted: neither one
// that another snapshot backs up, nor one that the previous snapshot of
// this name had before the exclude was added.
func TestBackupDoesNotCountExcludedFileAsDeleted(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
env := newDeletedCountEnv(t, map[string][]byte{
"top.txt": []byte("top"),
"cache/tmp.bin": []byte("cache"),
"photos/a.jpg": []byte("photo a"),
"photos/b.jpg": []byte("photo b"),
"photos/c.jpg": []byte("photo c"),
})
env.setSnapshot("photos", "photos")
env.setSnapshot("home", "", "/photos")
env.backUp(t, "photos")
assert.Contains(t, env.backUp(t, "home"), filesLine(2, 2, 0))
env.setSnapshot("home", "", "/photos", "/cache")
assert.Contains(t, env.backUp(t, "home"), filesLine(1, 0, 1))
}
// A file inside a directory that an exclude added after a backup
// matches is not counted as deleted, though the pattern matches only the
// directory.
func TestBackupDoesNotCountFileInExcludedDirectoryAsDeleted(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
env := newDeletedCountEnv(t, map[string][]byte{
"top.txt": []byte("top"),
"user/.local/share/Trash/old.txt": []byte("old"),
})
env.setSnapshot("home", "")
assert.Contains(t, env.backUp(t, "home"), filesLine(2, 2, 0))
env.setSnapshot("home", "", "*/.local/share/Trash")
assert.Contains(t, env.backUp(t, "home"), filesLine(1, 0, 1))
}
// A backup counts only files that the previous snapshot of its own name
// had, not one that a later snapshot of another name over the same path
// had and that is since removed.
func TestBackupIgnoresSnapshotOfAnotherName(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
env := newDeletedCountEnv(t, map[string][]byte{
"kept.txt": []byte("kept"),
})
env.setSnapshot("home", "")
env.setSnapshot("other", "")
env.backUp(t, "home")
// Snapshot start times are kept in whole seconds: wait so that other's
// snapshot is strictly the most recent.
time.Sleep(time.Second)
extra := filepath.Join(env.src, "extra.txt")
require.NoError(t, os.WriteFile(extra, []byte("extra"), 0o600))
env.backUp(t, "other")
require.NoError(t, os.Remove(extra))
assert.Contains(t, env.backUp(t, "home"), filesLine(1, 0, 1))
}
// A file removed from disk is counted as deleted by the next backup of
// the snapshot that had it, and not by any later backup of that
// snapshot, nor by the first backup of another name over the same path.
func TestBackupCountsDeletedFileOnce(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
aContent := bytes.Repeat([]byte("a"), 1000)
bContent := bytes.Repeat([]byte("b"), 2000)
keptContent := bytes.Repeat([]byte("k"), 500)
env := newDeletedCountEnv(t, map[string][]byte{
"a.txt": aContent,
"b.txt": bContent,
"kept.txt": keptContent,
})
env.setSnapshot("home", "")
env.setSnapshot("other", "")
assert.Contains(t, env.backUp(t, "home"), filesLine(3, 3, 0))
require.NoError(t, os.Remove(filepath.Join(env.src, "a.txt")))
require.NoError(t, os.Remove(filepath.Join(env.src, "b.txt")))
summary := env.backUp(t, "home")
assert.Contains(t, summary,
"Files: 1 examined, 0 backed up, 1 unchanged, 2 deleted.")
assert.Contains(t, summary,
fmt.Sprintf("Data: %s total (%s backed up), %s deleted.",
env.v.UI.Size(int64(len(keptContent))), env.v.UI.Size(0),
env.v.UI.Size(int64(len(aContent)+len(bContent)))))
assert.Contains(t, env.backUp(t, "home"), filesLine(1, 0, 1))
assert.Contains(t, env.backUp(t, "other"), filesLine(1, 0, 1))
}
+3 -3
View File
@@ -553,8 +553,8 @@ func (v *Vaultik) printSnapshotTable(snapshots []SnapshotInfo) error {
if snap.LocallyTracked {
id = snap.ID.String()
uncompressed = ubytes(snap.UncompressedSize)
newChunks = ubytes(snap.NewChunkSize)
uncompressed = formatBytes(snap.UncompressedSize)
newChunks = formatBytes(snap.NewChunkSize)
} else {
id = formatRemoteOnlyID(snap.RemoteKey)
uncompressed = remoteOnlyCell
@@ -564,7 +564,7 @@ func (v *Vaultik) printSnapshotTable(snapshots []SnapshotInfo) error {
_, err = fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n",
id,
snap.Timestamp.Format("2006-01-02 15:04:05"),
ubytes(snap.CompressedSize),
formatBytes(snap.CompressedSize),
uncompressed,
newChunks)
if err != nil {
+2 -44
View File
@@ -212,8 +212,8 @@ const (
listConfiguredName = "home"
listLocalID = "testhost_home_2026-03-01T10:00:00Z"
listRemoteID = "otherhost_media_2026-03-02T11:22:33Z"
// fiveMegabytes is printed as "5.0 MB" by every command.
fiveMegabytes = 5_000_000
// fiveMegabytes formats as "5.0 MB" through formatBytes.
fiveMegabytes = 5 * 1024 * 1024
)
// TestListSnapshots_RemoteWithoutSecretKey is the regression guard for
@@ -306,48 +306,6 @@ func TestListSnapshots_RemoteOnlyRowRendering(t *testing.T) {
"are not in the local index")
}
// TestSnapshotListAndPurgePrintSizesLikeRemoteInfo checks that the
// `snapshot list` table and the purge listing print a byte count the way
// `remote info` does: in powers of 1000, as the MB label says.
func TestSnapshotListAndPurgePrintSizesLikeRemoteInfo(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
const wantSize = "5.0 MB" // fiveMegabytes
env := newListEnv(t)
// A remote-only snapshot: `snapshot list` and `remote info` both take
// its size from the manifest.
remoteKey := env.addRemote(t, listRemoteID,
time.Date(2026, 3, 2, 11, 22, 33, 0, time.UTC))
// A locally tracked snapshot of the same size for the purge to list.
// The purge prints the snapshot's blob size, the last argument here.
start := time.Date(2026, 3, 1, 10, 0, 0, 0, time.UTC)
env.addLocal(t, listLocalID, start)
env.addRemote(t, listLocalID, start)
require.NoError(t, env.v.Repositories.Snapshots.UpdateCounts(
context.Background(), nil, listLocalID, 0, 0, 0, 0, fiveMegabytes))
require.NoError(t, env.v.RemoteInfo(false))
assert.Contains(t, findTableRow(t, env.stdout.String(), remoteKey[:16]),
wantSize, "remote info")
env.stdout.Reset()
require.NoError(t, env.v.ListSnapshots(false))
assert.Contains(t, findTableRow(t, env.stdout.String(),
"<remote only:"+remoteKey[:12]+">"), wantSize, "snapshot list")
// Without --force and with nothing on stdin the purge lists what it
// would delete, reads no confirmation, and deletes nothing.
env.stdout.Reset()
require.NoError(t, env.v.PurgeSnapshotsWithOptions(
&vaultik.SnapshotPurgeOptions{OlderThan: "1d"}))
assert.Contains(t, findTableRow(t, env.stdout.String(), listLocalID),
wantSize, "snapshot purge")
}
// findTableRow returns the single output line containing needle.
func findTableRow(t *testing.T, out, needle string) string {
t.Helper()