Compare commits

..
Author SHA1 Message Date
sneak 2013d5655a Count a file as deleted only if the previous snapshot of its name had it (closes #304)
check / check (push) Waiting to run
A backup counted as deleted every file in the local index under the
source path that the walk did not find. That took in files the
snapshot excludes but another snapshot backs up, and files removed long
ago, which every later backup, and the first backup of another name
over the same path, reported as deleted again.

The backup now looks up the most recent completed snapshot of the same
name and counts only the files it lists that the walk did not find. A
file that is excluded, or inside an excluded directory, is not counted.

The snapshots row stores no deleted figure; only the backup's output
changes.

Model: opus-5-5
2026-10-13 14:23:18 +00:00
clawbot 005927a7fc Print snapshot list and purge sizes like every other command (closes #312)
check / check (push) Waiting to run
`snapshot list`, the `snapshot purge` listing and the `bytes` value in
terminal log lines divided by 1024 while labelling the result KB or MB,
so 6,329,198 bytes read 6.0 MB there and 6.3 MB in every other command.
All three now use humanize.Bytes, as the rest of the program does, and
the 1024-based helper in internal/vaultik is removed.

The size shared by the list tests is now 5,000,000 bytes, which the two
ways of counting print differently.

Judgement call: the log handler's copy of the formatter is changed too,
since the issue asks that every command print a byte count the same way.

Model: opus-5-5
2026-10-13 16:22:03 +02:00
17 changed files with 525 additions and 387 deletions
+4 -4
View File
@@ -279,8 +279,8 @@ Key methods:
- `Scan(ctx, path, snapshotID)` → Main scan operation
Scan phases:
1. **Phase 0**: Detect deleted files from previous snapshots
2. **Phase 1**: Walk directory, identify files needing processing
1. **Phase 1**: Walk directory, identify files needing processing
2. **Phase 1b**: Count as deleted the files that the previous snapshot of the same name lists and the walk did not find; a file that is excluded, or inside an excluded directory, is not counted
3. **Phase 2**: Process files (chunk → pack → upload)
#### SnapshotManager
@@ -320,13 +320,13 @@ CreateSnapshot(opts)
│ │
│ ├─► scanner.Scan(ctx, path, snapshotID)
│ │ │
│ │ ├─► Phase 0: detectDeletedFiles()
│ │ │
│ │ ├─► Phase 1: scanPhase()
│ │ │ Walk directory
│ │ │ Check file metadata changes
│ │ │ Build list of files to process
│ │ │
│ │ ├─► Phase 1b: detectDeletedFiles()
│ │ │
│ │ └─► Phase 2: processPhase()
│ │ For each file:
│ │ chunker.ChunkReaderStreaming()
+2 -10
View File
@@ -271,16 +271,8 @@ existing file. Created with mode `0600` since it will contain credentials.
**`config set`**: Set a scalar config value by dotted YAML path
(e.g. `vaultik config set compression_level 9`,
`vaultik config set storage_url "file:///mnt/backups"`). Comments and
formatting in the file are preserved. Missing keys along the path are
created: as a list when the next path element is a number (so
`vaultik config set age_recipients.0 age1...` works on a file without
`age_recipients`), otherwise as a map. If the config loads, a change
after which it would not, such as `vaultik config set compression_level 99`,
is refused and the file is left unchanged. A config that does not load
yet, such as the one `config init` writes before `storage_url` is set,
can be filled in one setting at a time: only a change that gives a
setting the wrong type, such as a single value where a list belongs or a
word where a number belongs, is refused.
formatting in the file are preserved; intermediate maps are created as
needed.
**`snapshot create`**: Perform incremental backup of configured snapshots.
* Optional snapshot names argument to create specific snapshots (default: all)
+24 -74
View File
@@ -44,9 +44,6 @@ var (
errNeedNumericIndex = errors.New("key is a list; use a numeric index")
errIndexOutOfRange = errors.New("index out of range")
errNotMapOrList = errors.New("key is not a map or list")
errWouldNotLoad = errors.New(
"vaultik would not load the config after this change, " +
"so the file was not changed")
)
const defaultConfigTemplate = `# vaultik configuration
@@ -394,15 +391,8 @@ func newConfigSetCommand() *cobra.Command {
Use: "set <key> <value>",
Short: "Set a config value by dotted path (e.g. compression_level 5)",
Long: `Sets a scalar config value addressed by dotted YAML path and writes
the file back, preserving comments and formatting. Missing keys along
the path are created: as a list when the next path element is a
number, otherwise as a map.
If the config loads, a change after which it would not, such as
compression_level 99, is refused and the file is left unchanged. A
config that does not load yet, such as the one config init writes
before storage_url is set, can be filled in one setting at a time:
only a change that gives a setting the wrong type is refused.
the file back, preserving comments and formatting. Intermediate maps
are created as needed.
Examples:
vaultik config set storage_url "file:///mnt/backups"
@@ -432,11 +422,6 @@ func writeConfigSet(out *ui.Writer, path, key, value string) error {
return err
}
oldData, err := marshalConfigYAML(root)
if err != nil {
return fmt.Errorf("marshaling config: %w", err)
}
err = yamlPathSet(root, strings.Split(key, "."), value)
if err != nil {
return err
@@ -447,11 +432,6 @@ func writeConfigSet(out *ui.Writer, path, key, value string) error {
return fmt.Errorf("marshaling config: %w", err)
}
err = checkConfigChange(oldData, data)
if err != nil {
return fmt.Errorf("%w: %w", errWouldNotLoad, err)
}
err = os.WriteFile(path, data, configFileMode)
if err != nil {
return fmt.Errorf("writing config file: %w", err)
@@ -473,28 +453,6 @@ func writeConfigSet(out *ui.Writer, path, key, value string) error {
return nil
}
// checkConfigChange returns the reason vaultik would not load newData, the
// config after a change, if the change brings that failure in. If oldData,
// the config before it, decodes into the config type, the new one must
// too; if the old one also passes validation, the new one must pass it. A
// config still being filled in fails validation until it is complete, so
// only decoding is checked for it.
func checkConfigChange(oldData, newData []byte) error {
oldCfg, oldErr := config.Parse(oldData)
if oldErr == nil {
newCfg, err := config.Parse(newData)
if err != nil {
return err
}
if oldCfg.Validate() == nil {
return newCfg.Validate()
}
}
return nil
}
// marshalConfigYAML renders a config document tree with 2-space indentation,
// matching defaultConfigTemplate. yaml.Marshal defaults to 4 spaces, which
// would reindent the whole file on the first `config set` despite the promise
@@ -601,8 +559,8 @@ func yamlPathGet(root *yaml.Node, keys []string) (*yaml.Node, error) {
return node, nil
}
// yamlPathSet navigates a dotted key path, creating missing keys as needed,
// and sets the final key to the given scalar value. Numeric path
// yamlPathSet navigates a dotted key path, creating intermediate maps as
// needed, and sets the final key to the given scalar value. Numeric path
// components index into sequences; an index equal to the sequence length
// appends a new element (e.g. "age_recipients.1" on a 1-element list).
func yamlPathSet(root *yaml.Node, keys []string, value string) error {
@@ -615,12 +573,12 @@ func yamlPathSet(root *yaml.Node, keys []string, value string) error {
node = node.Content[0]
}
for i := range keys {
for i, key := range keys {
last := i == len(keys)-1
switch node.Kind {
case yaml.MappingNode:
node = yamlSetInMapping(node, keys, i, value, last)
node = yamlSetInMapping(node, key, value, last)
case yaml.SequenceNode:
next, err := yamlSetInSequence(node, keys, i, value, last)
if err != nil {
@@ -689,16 +647,14 @@ func yamlField(typ reflect.Type, key string) (reflect.StructField, bool) {
return reflect.StructField{}, false
}
// yamlSetInMapping resolves (creating if needed) the value node for the
// path element keys[i] within a mapping node, setting it to value when it
// is the final path element, and returns the node to descend into.
func yamlSetInMapping(
node *yaml.Node, keys []string, i int, value string, last bool,
) *yaml.Node {
// yamlSetInMapping resolves (creating if needed) the value node for key
// within a mapping node, setting it to value when it is the final path
// element, and returns the node to descend into.
func yamlSetInMapping(node *yaml.Node, key, value string, last bool) *yaml.Node {
var valueNode *yaml.Node
for j := 0; j+1 < len(node.Content); j += 2 {
if node.Content[j].Value == keys[i] {
if node.Content[j].Value == key {
valueNode = node.Content[j+1]
break
@@ -706,8 +662,13 @@ func yamlSetInMapping(
}
if valueNode == nil {
keyNode := &yaml.Node{Kind: yaml.ScalarNode, Value: keys[i]}
valueNode = newPathNode(keys, i, value)
keyNode := &yaml.Node{Kind: yaml.ScalarNode, Value: key}
valueNode = &yaml.Node{Kind: yaml.MappingNode}
if last {
valueNode = &yaml.Node{Kind: yaml.ScalarNode, Value: value}
}
node.Content = append(node.Content, keyNode, valueNode)
} else if last {
setScalar(valueNode, value)
@@ -735,7 +696,12 @@ func yamlSetInSequence(
}
if idx == len(node.Content) {
node.Content = append(node.Content, newPathNode(keys, i, value))
newNode := &yaml.Node{Kind: yaml.MappingNode}
if last {
newNode = &yaml.Node{Kind: yaml.ScalarNode, Value: value}
}
node.Content = append(node.Content, newNode)
} else if last {
setScalar(node.Content[idx], value)
}
@@ -743,22 +709,6 @@ func yamlSetInSequence(
return node.Content[idx], nil
}
// newPathNode returns the node to create for the missing path element
// keys[i]: the value when it is the final path element, otherwise an empty
// list when the next path element is a number and an empty map when not.
func newPathNode(keys []string, i int, value string) *yaml.Node {
if i == len(keys)-1 {
return &yaml.Node{Kind: yaml.ScalarNode, Value: value}
}
_, err := strconv.Atoi(keys[i+1])
if err == nil {
return &yaml.Node{Kind: yaml.SequenceNode}
}
return &yaml.Node{Kind: yaml.MappingNode}
}
// setScalar overwrites a node in place with a plain scalar value.
func setScalar(n *yaml.Node, value string) {
n.Kind = yaml.ScalarNode
+5 -176
View File
@@ -58,16 +58,14 @@ func TestDefaultConfigTemplateParses(t *testing.T) {
}
}
// testRecipient is a valid age public key.
const testRecipient = "age1278m9q7dp3chsh2dcy82qk27v047zywyvtxwnj4cvt0z65jw6a7q5dqhfj"
// TestConfigSetRecipientOnFreshConfig follows the README quickstart: on the
// file `config init` writes, `config set age_recipients.0` and
// `config set storage_url` give a config that loads with that recipient.
// Neither is refused, although the config does not load until both are set.
func TestConfigSetRecipientOnFreshConfig(t *testing.T) {
t.Parallel()
const recipient = "age1278m9q7dp3chsh2dcy82qk27v047zywyvtxwnj4cvt0z65jw6a7q5dqhfj"
path := filepath.Join(t.TempDir(), "config.yml")
err := os.WriteFile(path, []byte(defaultConfigTemplate), configFileMode)
@@ -77,7 +75,7 @@ func TestConfigSetRecipientOnFreshConfig(t *testing.T) {
out := ui.NewWithColor(&bytes.Buffer{}, false)
err = writeConfigSet(out, path, "age_recipients.0", testRecipient)
err = writeConfigSet(out, path, "age_recipients.0", recipient)
if err != nil {
t.Fatalf("config set age_recipients.0: %v", err)
}
@@ -92,177 +90,8 @@ func TestConfigSetRecipientOnFreshConfig(t *testing.T) {
t.Fatalf("config.Load: %v", err)
}
if len(cfg.AgeRecipients) != 1 || cfg.AgeRecipients[0] != testRecipient {
t.Errorf("age_recipients = %v, want [%s]", cfg.AgeRecipients, testRecipient)
}
}
// configWithoutRecipients loads, but has no age_recipients key and no
// snapshot named work.
const configWithoutRecipients = `storage_url: file:///mnt/backups
snapshots:
home:
paths:
- "~"
`
// TestConfigSetIndexUnderMissingKey checks that a list index under a key
// the file does not have creates a list, and that the result loads.
func TestConfigSetIndexUnderMissingKey(t *testing.T) {
t.Parallel()
tests := []struct {
key string
value string
field func(cfg *config.Config) []string
}{
{"age_recipients.0", testRecipient,
func(cfg *config.Config) []string { return cfg.AgeRecipients }},
{"snapshots.work.paths.0", "/srv/work",
func(cfg *config.Config) []string { return cfg.Snapshots["work"].Paths }},
}
for _, tt := range tests {
t.Run(tt.key, func(t *testing.T) {
t.Parallel()
path := filepath.Join(t.TempDir(), "config.yml")
err := os.WriteFile(path, []byte(configWithoutRecipients), configFileMode)
if err != nil {
t.Fatalf("write config: %v", err)
}
err = writeConfigSet(ui.NewWithColor(&bytes.Buffer{}, false),
path, tt.key, tt.value)
if err != nil {
t.Fatalf("config set %s: %v", tt.key, err)
}
cfg, err := config.Load(path)
if err != nil {
t.Fatalf("config.Load: %v", err)
}
got := tt.field(cfg)
if len(got) != 1 || got[0] != tt.value {
t.Errorf("%s gives %v, want [%s]", tt.key, got, tt.value)
}
})
}
}
// TestConfigSetKeepsFileThatWouldNotLoad checks that config set refuses a
// change after which the config would not load, with an error giving the
// reason, and leaves the file as it was. On the file `config init` writes,
// which does not load until storage_url is set, only a value of the wrong
// type is refused.
func TestConfigSetKeepsFileThatWouldNotLoad(t *testing.T) {
t.Parallel()
tests := []struct {
config string
key string
value string
reason string
}{
// Index 1 of the empty list the missing key gives.
{configWithoutRecipients, "age_recipients.1", testRecipient,
"index out of range"},
// A list set to a single value.
{configWithoutRecipients, "snapshots.home.paths", "/srv",
"cannot unmarshal"},
{configWithoutRecipients, "compression_level", "fast",
"cannot unmarshal"},
// Values of the right type that fail validation.
{configWithoutRecipients, "compression_level", "99",
"compression_level must be between 1 and 19"},
{configWithoutRecipients, "chunk_size", "1KB",
"chunk_size must be at least 1MB"},
{configWithoutRecipients, "age_recipients.0", "notakey",
"not a valid recipient"},
// The type check on a config that does not load yet.
{defaultConfigTemplate, "snapshots.apps.paths", "/srv",
"cannot unmarshal"},
}
for _, tt := range tests {
t.Run(tt.key+" "+tt.value, func(t *testing.T) {
t.Parallel()
path := filepath.Join(t.TempDir(), "config.yml")
err := os.WriteFile(path, []byte(tt.config), configFileMode)
if err != nil {
t.Fatalf("write config: %v", err)
}
err = writeConfigSet(ui.NewWithColor(&bytes.Buffer{}, false),
path, tt.key, tt.value)
if err == nil || !strings.Contains(err.Error(), tt.reason) {
t.Errorf("config set %s %s gives error %v, want one saying %q",
tt.key, tt.value, err, tt.reason)
}
data, err := os.ReadFile(path) //nolint:gosec // G304: test-controlled path
if err != nil {
t.Fatalf("read config: %v", err)
}
if string(data) != tt.config {
t.Errorf("config set %s changed the file to:\n%s",
tt.key, data)
}
})
}
}
// TestConfigSetRefusesOnlyItsOwnFailure checks that setting another key
// works on a config with a ${...} reference on compression_level, which
// loads because loading fills the reference in, and on one whose
// compression_level already fails to load.
func TestConfigSetRefusesOnlyItsOwnFailure(t *testing.T) {
t.Parallel()
levelPath := filepath.Join(t.TempDir(), "level")
err := os.WriteFile(levelPath, []byte("9"), configFileMode)
if err != nil {
t.Fatalf("write level: %v", err)
}
configs := map[string]string{
"reference": configWithoutRecipients +
"compression_level: ${FILE:" + levelPath + "}\n",
"failing": configWithoutRecipients + "compression_level: fast\n",
}
for name, configYAML := range configs {
t.Run(name, func(t *testing.T) {
t.Parallel()
path := filepath.Join(t.TempDir(), "config.yml")
err := os.WriteFile(path, []byte(configYAML), configFileMode)
if err != nil {
t.Fatalf("write config: %v", err)
}
err = writeConfigSet(ui.NewWithColor(&bytes.Buffer{}, false),
path, "hostname", "myhost")
if err != nil {
t.Fatalf("config set hostname: %v", err)
}
data, err := os.ReadFile(path) //nolint:gosec // G304: test-controlled path
if err != nil {
t.Fatalf("read config: %v", err)
}
if !strings.Contains(string(data), "hostname: myhost") {
t.Errorf("config set hostname wrote:\n%s", data)
}
})
if len(cfg.AgeRecipients) != 1 || cfg.AgeRecipients[0] != recipient {
t.Errorf("age_recipients = %v, want [%s]", cfg.AgeRecipients, recipient)
}
}
+24 -41
View File
@@ -4,7 +4,6 @@
package config
import (
"bytes"
"errors"
"fmt"
"os"
@@ -234,48 +233,14 @@ func New(path Path) (*Config, error) {
return cfg, nil
}
// Load reads the configuration file at path, parses it with Parse, warns
// when others can read the file, and validates the configuration.
// Returns an error if the file cannot be read, parsed, or if validation
// fails.
// Load reads and parses the configuration file from the specified path.
// It applies default values for optional fields, performs environment variable
// substitution using smartconfig, and validates the configuration.
// The configuration file should be in YAML format. Returns an error if the file
// cannot be read, parsed, or if validation fails.
func Load(path string) (*Config, error) {
data, err := os.ReadFile(path) //nolint:gosec // G304: config path is operator-supplied
if err != nil {
return nil, fmt.Errorf("failed to load config file: %w", err)
}
cfg, err := Parse(data)
if err != nil {
return nil, err
}
// Check config file permissions (warn if world or group readable)
//nolint:gosec // G703: config path is operator-supplied by design
info, statErr := os.Stat(path)
if statErr == nil {
mode := info.Mode().Perm()
if mode&0044 != 0 { // group or world readable
log.Warn(cfg.readableByOthersWarning(),
"path", path,
"mode", fmt.Sprintf("%04o", mode),
"recommendation", "chmod 600 "+path)
}
}
err = cfg.Validate()
if err != nil {
return nil, fmt.Errorf("invalid config: %w", err)
}
return cfg, nil
}
// Parse reads YAML config data as Load does, without validating it. It
// applies default values for optional fields, fills in ${...} references
// using smartconfig, and expands ~ in paths.
func Parse(data []byte) (*Config, error) {
// Load config using smartconfig for interpolation
sc, err := smartconfig.NewFromReader(bytes.NewReader(data))
sc, err := smartconfig.NewFromConfigPath(path)
if err != nil {
return nil, fmt.Errorf("failed to load config file: %w", err)
}
@@ -337,6 +302,24 @@ func Parse(data []byte) (*Config, error) {
cfg.S3.Region = "us-east-1"
}
// Check config file permissions (warn if world or group readable)
//nolint:gosec // G703: config path is operator-supplied by design
info, statErr := os.Stat(path)
if statErr == nil {
mode := info.Mode().Perm()
if mode&0044 != 0 { // group or world readable
log.Warn(cfg.readableByOthersWarning(),
"path", path,
"mode", fmt.Sprintf("%04o", mode),
"recommendation", "chmod 600 "+path)
}
}
err = cfg.Validate()
if err != nil {
return nil, fmt.Errorf("invalid config: %w", err)
}
return cfg, nil
}
+44
View File
@@ -277,6 +277,50 @@ func (r *FileRepository) ListUnderPath(
return files, rows.Err()
}
// ListUnderPathInSnapshot returns the files that ListUnderPath returns
// for path and that the snapshot lists, in no particular order.
func (r *FileRepository) ListUnderPathInSnapshot(
ctx context.Context, path, snapshotID string,
) ([]*File, error) {
path = strings.TrimRight(path, "/")
dirPrefix := path + "/"
query := `
SELECT f.id, f.path, f.source_path, f.mtime, f.mtime_nsec, f.size,
f.mode, f.uid, f.gid, f.link_target
FROM files f
JOIN snapshot_files sf ON sf.file_id = f.id
WHERE sf.snapshot_id = ?
AND (f.path = ? OR substr(f.path, 1, length(?)) = ?)
`
rows, err := r.db.conn.QueryContext(ctx, query,
snapshotID, path, dirPrefix, dirPrefix)
if err != nil {
return nil, fmt.Errorf("querying files: %w", err)
}
defer func() {
err := rows.Close()
if err != nil {
Fatalf("failed to close rows: %v", err)
}
}()
var files []*File
for rows.Next() {
file, err := r.scanFileRows(rows)
if err != nil {
return nil, fmt.Errorf("scanning file: %w", err)
}
files = append(files, file)
}
return files, rows.Err()
}
// ListIDsWithChunksNotInUploadedBlobs returns the IDs of the files whose
// path starts with prefix and that list at least one chunk held by no
// blob whose upload has completed (uploaded_ts set). A new snapshot
+59
View File
@@ -212,6 +212,65 @@ func TestFileRepositoryListUnderPath(t *testing.T) {
}
}
// ListUnderPathInSnapshot leaves out files outside path and files the
// snapshot does not list, though another snapshot lists them.
func TestFileRepositoryListUnderPathInSnapshot(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
files := database.NewFileRepository(db)
snapshots := database.NewSnapshotRepository(db)
listed := map[string][]string{
"one": {"/src/a.txt", "/src2/c.txt"},
"two": {"/src/b.txt"},
}
for snapshotID, paths := range listed {
err := snapshots.Create(ctx, nil, &database.Snapshot{
ID: types.SnapshotID(snapshotID),
Hostname: "host",
StartedAt: time.Now(),
})
if err != nil {
t.Fatalf("failed to create snapshot %s: %v", snapshotID, err)
}
for _, path := range paths {
err := files.Create(ctx, nil, &database.File{
Path: types.FilePath(path),
MTime: time.Now().Truncate(time.Second),
Mode: 0644,
})
if err != nil {
t.Fatalf("failed to create %s: %v", path, err)
}
err = snapshots.AddFile(ctx, nil, snapshotID, path)
if err != nil {
t.Fatalf("failed to add %s to %s: %v", path, snapshotID, err)
}
}
}
found, err := files.ListUnderPathInSnapshot(ctx, "/src", "one")
if err != nil {
t.Fatalf("failed to list files: %v", err)
}
got := make([]string, 0, len(found))
for _, f := range found {
got = append(got, f.Path.String())
}
if !slices.Equal(got, []string{"/src/a.txt"}) {
t.Errorf("got %q, want only /src/a.txt", got)
}
}
func TestFileRepositorySymlink(t *testing.T) {
t.Parallel()
+7 -10
View File
@@ -10,6 +10,8 @@ import (
"sync"
"time"
"unicode"
"github.com/dustin/go-humanize"
)
// groupSeparator joins an open group path to an attribute key. This
@@ -304,18 +306,13 @@ func formatDuration(d time.Duration) string {
}
}
// formatBytes formats bytes in a human-readable way
// formatBytes formats a byte count with humanize.Bytes, as every command
// prints one. humanize.Bytes takes no negative count, so one is printed
// as it is.
func formatBytes(b int64) string {
const unit = 1024
if b < unit {
if b < 0 {
return fmt.Sprintf("%d B", b)
}
div, exp := int64(unit), 0
for n := b / unit; n >= unit; n /= unit {
div *= unit
exp++
}
return fmt.Sprintf("%.1f %cB", float64(b)/float64(div), "KMGTPE"[exp])
return humanize.Bytes(uint64(b))
}
+5 -4
View File
@@ -200,7 +200,8 @@ func TestTTYHandlerWithGroupQualifiesKeys(t *testing.T) {
func TestTTYHandlerByteFormattingSurvivesGrouping(t *testing.T) {
t.Parallel()
const oneAndAHalfKiB = 1536
// Printed as "1.0 MB", in powers of 1000 like every other byte count.
const oneMegabyte = 1_000_000
for name, testCase := range map[string]struct {
derive func(*slog.Logger) *slog.Logger
@@ -223,13 +224,13 @@ func TestTTYHandlerByteFormattingSurvivesGrouping(t *testing.T) {
var buf bytes.Buffer
logger := slog.New(log.NewTTYHandler(&buf, debugHandlerOptions()))
testCase.derive(logger).Info("uploaded", "bytes", oneAndAHalfKiB)
testCase.derive(logger).Info("uploaded", "bytes", oneMegabyte)
line := ansiEscape.ReplaceAllString(buf.String(), "")
assert.Contains(t, line, testCase.key+"=1.5 KB",
assert.Contains(t, line, testCase.key+"=1.0 MB",
"a byte count must be human-readable however it is qualified")
assert.NotContains(t, line, strconv.Itoa(oneAndAHalfKiB),
assert.NotContains(t, line, strconv.Itoa(oneMegabyte),
"the raw number must not survive the formatting")
})
}
+15 -11
View File
@@ -16,6 +16,9 @@ type ScannerParams struct {
Fs afero.Fs
Exclude []string // Exclude patterns (combined global + snapshot-specific)
SkipErrors bool // Skip file read errors (log loudly but continue)
// PreviousSnapshotID is the most recent completed snapshot of the
// same name, "" for none.
PreviousSnapshotID string
}
// Module exports backup functionality as an fx module.
@@ -44,17 +47,18 @@ func provideScannerFactory(
}
return NewScanner(ScannerConfig{
FS: params.Fs,
ChunkSize: cfg.ChunkSize.Int64(),
Repositories: repos,
Storage: storer,
MaxBlobSize: cfg.BlobSizeLimit.Int64(),
CompressionLevel: cfg.CompressionLevel,
AgeRecipients: cfg.AgeRecipients,
EnableProgress: params.EnableProgress,
UI: params.UI,
Exclude: excludes,
SkipErrors: params.SkipErrors,
FS: params.Fs,
ChunkSize: cfg.ChunkSize.Int64(),
Repositories: repos,
Storage: storer,
MaxBlobSize: cfg.BlobSizeLimit.Int64(),
CompressionLevel: cfg.CompressionLevel,
AgeRecipients: cfg.AgeRecipients,
EnableProgress: params.EnableProgress,
UI: params.UI,
Exclude: excludes,
SkipErrors: params.SkipErrors,
PreviousSnapshotID: params.PreviousSnapshotID,
})
}
}
+51 -19
View File
@@ -58,6 +58,10 @@ type Scanner struct {
compressionLevel int
ageRecipient string
snapshotID string // Current snapshot being processed
// previousSnapshotID is the most recent completed snapshot of the
// same name, "" for none. Only a file it lists can be counted as
// deleted.
previousSnapshotID string
// currentSourcePath is the source directory being scanned, stored with
// each file record.
currentSourcePath string
@@ -129,6 +133,9 @@ type ScannerConfig struct {
// SkipErrors skips files that cannot be opened or read (log loudly but
// continue); packer, database, encryption, and upload errors still abort.
SkipErrors bool
// PreviousSnapshotID is the most recent completed snapshot of the
// same name, "" for none.
PreviousSnapshotID string
}
// ScanResult contains the results of a scan operation. Files and bytes
@@ -204,6 +211,7 @@ func NewScanner(cfg ScannerConfig) *Scanner {
compiledExclude: compiledExclude,
progress: progress,
skipErrors: cfg.SkipErrors,
previousSnapshotID: cfg.PreviousSnapshotID,
ui: uiw,
pendingChunkHashes: make(map[string]struct{}),
}
@@ -255,8 +263,9 @@ func (s *Scanner) Scan(
filesToProcess := scanResult.FilesToProcess
// Phase 1b: Detect deleted files by comparing DB against scanned files
err = s.detectDeletedFilesFromMap(ctx, knownFiles, existingFiles, result)
// Phase 1b: Detect deleted files by comparing the previous snapshot
// of this name against scanned files
err = s.detectDeletedFiles(ctx, path, existingFiles, result)
if err != nil {
return nil, fmt.Errorf("detecting deleted files: %w", err)
}
@@ -436,9 +445,8 @@ func (s *Scanner) summarizeScanPhase(
}
// loadKnownFiles loads the known files at and beneath path from the
// database into a map for fast lookup. Every loaded file the scan does
// not find is counted as deleted. This avoids per-file database queries
// during the scan phase.
// database into a map for fast lookup. This avoids per-file database
// queries during the scan phase.
func (s *Scanner) loadKnownFiles(
ctx context.Context, path string,
) (map[string]*database.File, error) {
@@ -1938,21 +1946,28 @@ func (s *Scanner) queueFileForBatchInsert(
})
}
// detectDeletedFilesFromMap finds files that existed in previous snapshots
// but no longer exist. Uses pre-loaded maps to avoid any filesystem or
// database access.
func (s *Scanner) detectDeletedFilesFromMap(
// detectDeletedFiles counts as deleted each file at or beneath path that
// the previous snapshot of this name lists and the walk did not find.
// A file that is excluded, or inside an excluded directory, is not
// counted: the walk skips it whether or not it is still on disk.
func (s *Scanner) detectDeletedFiles(
ctx context.Context,
knownFiles map[string]*database.File,
path string,
existingFiles map[string]struct{},
result *ScanResult,
) error {
if len(knownFiles) == 0 {
if s.previousSnapshotID == "" {
return nil
}
// Check each known file against the enumerated set (no filesystem access needed)
for path, file := range knownFiles {
previousFiles, err := s.repos.Files.ListUnderPathInSnapshot(
ctx, path, s.previousSnapshotID)
if err != nil {
return fmt.Errorf("listing files of snapshot %s: %w",
s.previousSnapshotID, err)
}
for _, file := range previousFiles {
// Check context cancellation periodically
select {
case <-ctx.Done():
@@ -1960,13 +1975,16 @@ func (s *Scanner) detectDeletedFilesFromMap(
default:
}
// Check if the file exists in our enumerated set
if _, exists := existingFiles[path]; !exists {
// File has been deleted
result.FilesDeleted++
result.BytesDeleted += file.Size
log.Debug("Detected deleted file", "path", path, "size", file.Size)
filePath := file.Path.String()
_, exists := existingFiles[filePath]
if exists || s.excludedOrInExcludedDir(filePath, path) {
continue
}
result.FilesDeleted++
result.BytesDeleted += file.Size
log.Debug("Detected deleted file", "path", filePath, "size", file.Size)
}
if result.FilesDeleted > 0 {
@@ -1977,6 +1995,20 @@ func (s *Scanner) detectDeletedFilesFromMap(
return nil
}
// excludedOrInExcludedDir reports whether filePath, which is at or
// beneath rootPath, or any directory between rootPath and it, matches an
// exclude pattern. A pattern such as */.local/share/Trash matches the
// directory but not the files inside it, which the walk skips too.
func (s *Scanner) excludedOrInExcludedDir(filePath, rootPath string) bool {
for p := filePath; len(p) > len(rootPath); p = filepath.Dir(p) {
if s.shouldExclude(p, rootPath) {
return true
}
}
return false
}
// wrapPermissionError augments permission errors with platform-specific
// remediation instructions. On macOS, TCC-protected directories (Calendars,
// Reminders, Photos, etc.) return EPERM unless the running application has
+12 -11
View File
@@ -661,17 +661,18 @@ func faultScannerFactory(
) snapshot.ScannerFactory {
return func(params snapshot.ScannerParams) *snapshot.Scanner {
return snapshot.NewScanner(snapshot.ScannerConfig{
FS: params.Fs,
Storage: storer,
ChunkSize: faultChunkSize,
MaxBlobSize: faultMaxBlobSize,
CompressionLevel: cfg.CompressionLevel,
AgeRecipients: cfg.AgeRecipients,
Repositories: repos,
EnableProgress: params.EnableProgress,
UI: params.UI,
Exclude: params.Exclude,
SkipErrors: params.SkipErrors,
FS: params.Fs,
Storage: storer,
ChunkSize: faultChunkSize,
MaxBlobSize: faultMaxBlobSize,
CompressionLevel: cfg.CompressionLevel,
AgeRecipients: cfg.AgeRecipients,
Repositories: repos,
EnableProgress: params.EnableProgress,
UI: params.UI,
Exclude: params.Exclude,
SkipErrors: params.SkipErrors,
PreviousSnapshotID: params.PreviousSnapshotID,
})
}
}
-16
View File
@@ -89,22 +89,6 @@ type SnapshotInfo struct {
RemotePresent *bool `json:"remote_present"`
}
// formatBytes formats bytes in a human-readable format
func formatBytes(bytes int64) string {
const unit = 1024
if bytes < unit {
return fmt.Sprintf("%d B", bytes)
}
div, exp := int64(unit), 0
for n := bytes / unit; n >= unit; n /= unit {
div *= unit
exp++
}
return fmt.Sprintf("%.1f %cB", float64(bytes)/float64(div), "KMGTPE"[exp])
}
// parseSnapshotTimestamp extracts the timestamp from a snapshot ID
// Format: hostname_snapshotname_2026-01-12T14:41:15Z
func parseSnapshotTimestamp(snapshotID string) (time.Time, error) {
+30 -6
View File
@@ -240,12 +240,18 @@ func (v *Vaultik) createNamedSnapshot(
return err
}
previousSnapshotID, err := v.previousSnapshotID(snapName)
if err != nil {
return err
}
scanner := v.ScannerFactory(snapshot.ScannerParams{
EnableProgress: !opts.Cron,
UI: v.UI,
Fs: v.Fs,
Exclude: v.Config.GetExcludes(snapName),
SkipErrors: opts.SkipErrors,
EnableProgress: !opts.Cron,
UI: v.UI,
Fs: v.Fs,
Exclude: v.Config.GetExcludes(snapName),
SkipErrors: opts.SkipErrors,
PreviousSnapshotID: previousSnapshotID,
})
snapshotID, err := v.SnapshotManager.CreateSnapshotWithName(
@@ -280,6 +286,24 @@ func (v *Vaultik) createNamedSnapshot(
return nil
}
// previousSnapshotID returns the ID of the most recent completed snapshot
// named snapName in the local index, or "" when there is none.
func (v *Vaultik) previousSnapshotID(snapName string) (string, error) {
snapshots, err := v.Repositories.Snapshots.ListRecent(v.ctx, listRecentLimit)
if err != nil {
return "", fmt.Errorf("listing snapshots: %w", err)
}
for _, s := range snapshots {
if s.CompletedAt != nil &&
parseSnapshotName(s.ID.String(), s.Hostname.String()) == snapName {
return s.ID.String(), nil
}
}
return "", nil
}
// resolveSnapshotPaths resolves source directories to absolute paths
// with symlink resolution.
func (v *Vaultik) resolveSnapshotPaths(snapName string) ([]string, error) {
@@ -633,7 +657,7 @@ func (v *Vaultik) confirmAndExecutePurge(
v.stdoutf(" %s (%s, %s)\n",
snap.ID,
snap.Timestamp.Format("2006-01-02 15:04:05"),
formatBytes(snap.CompressedSize))
ubytes(snap.CompressedSize))
}
}
@@ -0,0 +1,196 @@
package vaultik_test
import (
"bytes"
"context"
"fmt"
"os"
"path/filepath"
"testing"
"time"
"github.com/spf13/afero"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"sneak.berlin/go/vaultik/internal/config"
"sneak.berlin/go/vaultik/internal/database"
"sneak.berlin/go/vaultik/internal/log"
"sneak.berlin/go/vaultik/internal/storage"
"sneak.berlin/go/vaultik/internal/ui"
"sneak.berlin/go/vaultik/internal/vaultik"
)
// These tests cover https://git.eeqj.de/sneak/vaultik/issues/304: a
// backup counts a file as deleted only when the previous snapshot of
// the same name had it and it is gone, and never counts an excluded
// file.
// deletedCountEnv is a backup setup over the source directory src whose
// user-facing output is kept in out. Each test configures its snapshots
// with setSnapshot.
type deletedCountEnv struct {
v *vaultik.Vaultik
out *bytes.Buffer
src string
}
// newDeletedCountEnv writes files, keyed by path relative to src.
func newDeletedCountEnv(t *testing.T, files map[string][]byte) *deletedCountEnv {
t.Helper()
ctx := context.Background()
fs := afero.NewOsFs()
tempDir := t.TempDir()
srcDir := filepath.Join(tempDir, "src")
for rel, content := range files {
path := filepath.Join(srcDir, rel)
require.NoError(t, fs.MkdirAll(filepath.Dir(path), 0o755))
require.NoError(t, afero.WriteFile(fs, path, content, 0o644))
}
cfg := faultTestConfig()
cfg.IndexPath = filepath.Join(tempDir, "index.sqlite")
cfg.Snapshots = map[string]config.SnapshotConfig{}
store, err := storage.NewFileStorer(filepath.Join(tempDir, "remote"))
require.NoError(t, err)
db, err := database.New(ctx, cfg.IndexPath)
require.NoError(t, err)
t.Cleanup(func() { _ = db.Close() })
repos := database.NewRepositories(db)
out := &bytes.Buffer{}
v := newBackupVaultik(ctx, cfg, store, repos, db, fs)
v.UI = ui.NewWithColor(out, false)
return &deletedCountEnv{v: v, out: out, src: srcDir}
}
// setSnapshot configures the snapshot name to back up the path rel
// beneath src, with the given excludes.
func (e *deletedCountEnv) setSnapshot(name, rel string, exclude ...string) {
e.v.Config.Snapshots[name] = config.SnapshotConfig{
Paths: []string{filepath.Join(e.src, rel)},
Exclude: exclude,
}
}
// backUp runs a backup of the named snapshot and returns its output.
func (e *deletedCountEnv) backUp(t *testing.T, name string) string {
t.Helper()
e.out.Reset()
require.NoError(t, e.v.CreateSnapshot(&vaultik.SnapshotCreateOptions{
Snapshots: []string{name},
}))
return e.out.String()
}
// A file this snapshot excludes is not counted as deleted: neither one
// that another snapshot backs up, nor one that the previous snapshot of
// this name had before the exclude was added.
func TestBackupDoesNotCountExcludedFileAsDeleted(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
env := newDeletedCountEnv(t, map[string][]byte{
"top.txt": []byte("top"),
"cache/tmp.bin": []byte("cache"),
"photos/a.jpg": []byte("photo a"),
"photos/b.jpg": []byte("photo b"),
"photos/c.jpg": []byte("photo c"),
})
env.setSnapshot("photos", "photos")
env.setSnapshot("home", "", "/photos")
env.backUp(t, "photos")
assert.Contains(t, env.backUp(t, "home"), filesLine(2, 2, 0))
env.setSnapshot("home", "", "/photos", "/cache")
assert.Contains(t, env.backUp(t, "home"), filesLine(1, 0, 1))
}
// A file inside a directory that an exclude added after a backup
// matches is not counted as deleted, though the pattern matches only the
// directory.
func TestBackupDoesNotCountFileInExcludedDirectoryAsDeleted(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
env := newDeletedCountEnv(t, map[string][]byte{
"top.txt": []byte("top"),
"user/.local/share/Trash/old.txt": []byte("old"),
})
env.setSnapshot("home", "")
assert.Contains(t, env.backUp(t, "home"), filesLine(2, 2, 0))
env.setSnapshot("home", "", "*/.local/share/Trash")
assert.Contains(t, env.backUp(t, "home"), filesLine(1, 0, 1))
}
// A backup counts only files that the previous snapshot of its own name
// had, not one that a later snapshot of another name over the same path
// had and that is since removed.
func TestBackupIgnoresSnapshotOfAnotherName(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
env := newDeletedCountEnv(t, map[string][]byte{
"kept.txt": []byte("kept"),
})
env.setSnapshot("home", "")
env.setSnapshot("other", "")
env.backUp(t, "home")
// Snapshot start times are kept in whole seconds: wait so that other's
// snapshot is strictly the most recent.
time.Sleep(time.Second)
extra := filepath.Join(env.src, "extra.txt")
require.NoError(t, os.WriteFile(extra, []byte("extra"), 0o600))
env.backUp(t, "other")
require.NoError(t, os.Remove(extra))
assert.Contains(t, env.backUp(t, "home"), filesLine(1, 0, 1))
}
// A file removed from disk is counted as deleted by the next backup of
// the snapshot that had it, and not by any later backup of that
// snapshot, nor by the first backup of another name over the same path.
func TestBackupCountsDeletedFileOnce(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
aContent := bytes.Repeat([]byte("a"), 1000)
bContent := bytes.Repeat([]byte("b"), 2000)
keptContent := bytes.Repeat([]byte("k"), 500)
env := newDeletedCountEnv(t, map[string][]byte{
"a.txt": aContent,
"b.txt": bContent,
"kept.txt": keptContent,
})
env.setSnapshot("home", "")
env.setSnapshot("other", "")
assert.Contains(t, env.backUp(t, "home"), filesLine(3, 3, 0))
require.NoError(t, os.Remove(filepath.Join(env.src, "a.txt")))
require.NoError(t, os.Remove(filepath.Join(env.src, "b.txt")))
summary := env.backUp(t, "home")
assert.Contains(t, summary,
"Files: 1 examined, 0 backed up, 1 unchanged, 2 deleted.")
assert.Contains(t, summary,
fmt.Sprintf("Data: %s total (%s backed up), %s deleted.",
env.v.UI.Size(int64(len(keptContent))), env.v.UI.Size(0),
env.v.UI.Size(int64(len(aContent)+len(bContent)))))
assert.Contains(t, env.backUp(t, "home"), filesLine(1, 0, 1))
assert.Contains(t, env.backUp(t, "other"), filesLine(1, 0, 1))
}
+3 -3
View File
@@ -553,8 +553,8 @@ func (v *Vaultik) printSnapshotTable(snapshots []SnapshotInfo) error {
if snap.LocallyTracked {
id = snap.ID.String()
uncompressed = formatBytes(snap.UncompressedSize)
newChunks = formatBytes(snap.NewChunkSize)
uncompressed = ubytes(snap.UncompressedSize)
newChunks = ubytes(snap.NewChunkSize)
} else {
id = formatRemoteOnlyID(snap.RemoteKey)
uncompressed = remoteOnlyCell
@@ -564,7 +564,7 @@ func (v *Vaultik) printSnapshotTable(snapshots []SnapshotInfo) error {
_, err = fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n",
id,
snap.Timestamp.Format("2006-01-02 15:04:05"),
formatBytes(snap.CompressedSize),
ubytes(snap.CompressedSize),
uncompressed,
newChunks)
if err != nil {
+44 -2
View File
@@ -212,8 +212,8 @@ const (
listConfiguredName = "home"
listLocalID = "testhost_home_2026-03-01T10:00:00Z"
listRemoteID = "otherhost_media_2026-03-02T11:22:33Z"
// fiveMegabytes formats as "5.0 MB" through formatBytes.
fiveMegabytes = 5 * 1024 * 1024
// fiveMegabytes is printed as "5.0 MB" by every command.
fiveMegabytes = 5_000_000
)
// TestListSnapshots_RemoteWithoutSecretKey is the regression guard for
@@ -306,6 +306,48 @@ func TestListSnapshots_RemoteOnlyRowRendering(t *testing.T) {
"are not in the local index")
}
// TestSnapshotListAndPurgePrintSizesLikeRemoteInfo checks that the
// `snapshot list` table and the purge listing print a byte count the way
// `remote info` does: in powers of 1000, as the MB label says.
func TestSnapshotListAndPurgePrintSizesLikeRemoteInfo(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
const wantSize = "5.0 MB" // fiveMegabytes
env := newListEnv(t)
// A remote-only snapshot: `snapshot list` and `remote info` both take
// its size from the manifest.
remoteKey := env.addRemote(t, listRemoteID,
time.Date(2026, 3, 2, 11, 22, 33, 0, time.UTC))
// A locally tracked snapshot of the same size for the purge to list.
// The purge prints the snapshot's blob size, the last argument here.
start := time.Date(2026, 3, 1, 10, 0, 0, 0, time.UTC)
env.addLocal(t, listLocalID, start)
env.addRemote(t, listLocalID, start)
require.NoError(t, env.v.Repositories.Snapshots.UpdateCounts(
context.Background(), nil, listLocalID, 0, 0, 0, 0, fiveMegabytes))
require.NoError(t, env.v.RemoteInfo(false))
assert.Contains(t, findTableRow(t, env.stdout.String(), remoteKey[:16]),
wantSize, "remote info")
env.stdout.Reset()
require.NoError(t, env.v.ListSnapshots(false))
assert.Contains(t, findTableRow(t, env.stdout.String(),
"<remote only:"+remoteKey[:12]+">"), wantSize, "snapshot list")
// Without --force and with nothing on stdin the purge lists what it
// would delete, reads no confirmation, and deletes nothing.
env.stdout.Reset()
require.NoError(t, env.v.PurgeSnapshotsWithOptions(
&vaultik.SnapshotPurgeOptions{OlderThan: "1d"}))
assert.Contains(t, findTableRow(t, env.stdout.String(), listLocalID),
wantSize, "snapshot purge")
}
// findTableRow returns the single output line containing needle.
func findTableRow(t *testing.T, out, needle string) string {
t.Helper()