Compare commits

..
1 Commits
Author SHA1 Message Date
sneak 55f2cb9cff Leave remote info orphan figures unknown when a manifest is unreadable (closes #228)
check / check (push) Canceled after 0s
When a manifest could not be read, remote info skipped it, counted that
snapshot's blobs as orphaned and advised running prune. The orphan
figures are now unknown in that case, with no prune advice; --json gives
them as null and lists the unreadable remote keys in
unreadable_manifests. Only a listed manifest.json.zst is read, so a
directory without one, as an interrupted backup leaves, keeps the
figures known.

Names under metadata/ were used unchecked and printed raw. A name that
is not a remote key is now skipped with a warning. A manifest under it
is not read either, so it also leaves the figures unknown; --json counts
such manifests in skipped_manifests.

The closing log line carries the unreadable manifest count in place of
the orphan count.

Model: opus-5-5
2026-10-07 06:54:51 +00:00
6 changed files with 14 additions and 191 deletions
+1 -1
View File
@@ -396,7 +396,7 @@ key is skipped with a warning and is not printed. If a listed
orphaned blob figures are reported as unknown; `--json` gives them as orphaned blob figures are reported as unknown; `--json` gives them as
`null`, lists the remote key of each unreadable manifest in `null`, lists the remote key of each unreadable manifest in
`unreadable_manifests` and counts the manifests under skipped names in `unreadable_manifests` and counts the manifests under skipped names in
`skipped_manifest_count`. `skipped_manifests`.
* `--json`: Output as JSON * `--json`: Output as JSON
**`remote nuke`**: Delete every snapshot's metadata and every blob from the **`remote nuke`**: Delete every snapshot's metadata and every blob from the
+2 -10
View File
@@ -35,16 +35,8 @@ the tag exists and is exercised; what is left is merging `next` to
being printed, control characters included. A manifest under a being printed, control characters included. A manifest under a
skipped name is then not read either, so it also leaves the orphan skipped name is then not read either, so it also leaves the orphan
figures unknown, and `--json` counts such manifests in figures unknown, and `--json` counts such manifests in
`skipped_manifest_count`. A directory with no manifest in it, as left `skipped_manifests`. A directory with no manifest in it, as left by an
by an interrupted backup, leaves the figures known. interrupted backup, leaves the figures known.
- 2026-10-07: Made `config set` keep a string that looks like a number
([issue #229](https://git.eeqj.de/sneak/vaultik/issues/229)). It wrote
every value unquoted, and `config.Load` reads the file through untyped
YAML, so an access key `00112233` loaded as `38043` and a hostname `007`
as `7`. A value for a string setting in `config.Config` is now tagged as
a YAML string, which the file quotes wherever YAML would read a number or
a boolean; other settings are still written unquoted.
- 2026-10-07: Made a backup notice a file rewritten with its size - 2026-10-07: Made a backup notice a file rewritten with its size
unchanged and a new mtime in the same second as the one in the index unchanged and a new mtime in the same second as the one in the index
-55
View File
@@ -7,14 +7,11 @@ import (
"os" "os"
"os/exec" "os/exec"
"path/filepath" "path/filepath"
"reflect"
"strconv" "strconv"
"strings" "strings"
"unicode/utf8"
"github.com/spf13/cobra" "github.com/spf13/cobra"
"gopkg.in/yaml.v3" "gopkg.in/yaml.v3"
"sneak.berlin/go/vaultik/internal/config"
"sneak.berlin/go/vaultik/internal/ui" "sneak.berlin/go/vaultik/internal/ui"
) )
@@ -34,9 +31,6 @@ const configDirMode = 0o755
// yaml.Marshal's 4-space default. // yaml.Marshal's 4-space default.
const configYAMLIndent = 2 const configYAMLIndent = 2
// yamlStringTag is YAML's tag for a string scalar.
const yamlStringTag = "!!str"
var ( var (
errConfigExists = errors.New("config file already exists") errConfigExists = errors.New("config file already exists")
errEmptyConfig = errors.New("empty config file") errEmptyConfig = errors.New("empty config file")
@@ -589,58 +583,9 @@ func yamlPathSet(root *yaml.Node, keys []string, value string) error {
} }
} }
// config.Load reads the file through untyped YAML, which turns an
// unquoted 00112233 into the number 38043 and 1e5 into 100000. Tagging
// a string setting as a string makes the encoder quote such a value.
// Other settings stay unquoted, so compression_level 9 is a number.
// The encoder refuses to write a value that is not valid UTF-8 as a
// string. Left untagged, such a value is written as base64 !!binary and
// loads back unchanged.
if configKeyIsString(keys) && utf8.ValidString(value) {
node.Tag = yamlStringTag
}
return nil return nil
} }
// configKeyIsString reports whether the dotted key names a string in
// config.Config, following the fields' yaml tags, as s3.access_key_id and
// snapshots.home.exclude.0 do.
func configKeyIsString(keys []string) bool {
typ := reflect.TypeFor[config.Config]()
for _, key := range keys {
switch {
case typ.Kind() == reflect.Map || typ.Kind() == reflect.Slice:
// The key is a snapshot name or a list index.
typ = typ.Elem()
case typ.Kind() == reflect.Struct:
field, ok := yamlField(typ, key)
if !ok {
return false
}
typ = field.Type
default:
return false
}
}
return typ.Kind() == reflect.String
}
// yamlField returns the field of struct type typ whose yaml tag names key.
func yamlField(typ reflect.Type, key string) (reflect.StructField, bool) {
for field := range typ.Fields() {
name, _, _ := strings.Cut(field.Tag.Get("yaml"), ",")
if name == key {
return field, true
}
}
return reflect.StructField{}, false
}
// yamlSetInMapping resolves (creating if needed) the value node for key // yamlSetInMapping resolves (creating if needed) the value node for key
// within a mapping node, setting it to value when it is the final path // within a mapping node, setting it to value when it is the final path
// element, and returns the node to descend into. // element, and returns the node to descend into.
-105
View File
@@ -4,7 +4,6 @@ import (
"bytes" "bytes"
"os" "os"
"path/filepath" "path/filepath"
"strconv"
"strings" "strings"
"testing" "testing"
@@ -95,110 +94,6 @@ func TestConfigSetRecipientOnFreshConfig(t *testing.T) {
} }
} }
// TestConfigSetStringLooksLikeNumber sets string settings to values that
// YAML reads as numbers or booleans when they are unquoted, and checks that
// config.Load returns each one unchanged.
func TestConfigSetStringLooksLikeNumber(t *testing.T) {
t.Parallel()
tests := []struct {
key string
value string
field func(cfg *config.Config) string
}{
{"s3.access_key_id", "00112233",
func(cfg *config.Config) string { return cfg.S3.AccessKeyID }},
{"s3.secret_access_key", "12345678901234567890123456789012",
func(cfg *config.Config) string { return cfg.S3.SecretAccessKey }},
{"hostname", "007",
func(cfg *config.Config) string { return cfg.Hostname }},
{"s3.prefix", "1e5",
func(cfg *config.Config) string { return cfg.S3.Prefix }},
{"s3.bucket", "true",
func(cfg *config.Config) string { return cfg.S3.Bucket }},
{"s3.region", "FALSE",
func(cfg *config.Config) string { return cfg.S3.Region }},
{"snapshots.home.exclude.0", "1.10",
func(cfg *config.Config) string { return cfg.Snapshots["home"].Exclude[0] }},
}
for _, tt := range tests {
t.Run(tt.key+"="+tt.value, func(t *testing.T) {
t.Parallel()
cfg := loadAfterConfigSet(t, tt.key, tt.value)
got := tt.field(cfg)
if got != tt.value {
t.Errorf("%s = %q after config set %q", tt.key, got, tt.value)
}
})
}
}
// TestConfigSetNonUTF8Path checks that config set still accepts a value that
// is not valid UTF-8, such as a path with a Latin-1 file name, and that
// config.Load returns it unchanged.
func TestConfigSetNonUTF8Path(t *testing.T) {
t.Parallel()
const dir = "/srv/caf\xe9"
cfg := loadAfterConfigSet(t, "snapshots.home.paths.0", dir)
got := cfg.Snapshots["home"].Paths[0]
if got != dir {
t.Errorf("snapshots.home.paths.0 = %q, want %q", got, dir)
}
}
// TestConfigSetNumberStaysNumber checks that a number set for an integer
// setting is still read as a number, not as a quoted string.
func TestConfigSetNumberStaysNumber(t *testing.T) {
t.Parallel()
const level = 9
cfg := loadAfterConfigSet(t, "compression_level", strconv.Itoa(level))
if cfg.CompressionLevel != level {
t.Errorf("compression_level = %d, want %d", cfg.CompressionLevel, level)
}
}
// loadAfterConfigSet writes the file `config init` writes, sets storage_url
// to a local directory so that the file passes validation, applies
// `config set key value` and returns what config.Load reads back.
func loadAfterConfigSet(t *testing.T, key, value string) *config.Config {
t.Helper()
path := filepath.Join(t.TempDir(), "config.yml")
err := os.WriteFile(path, []byte(defaultConfigTemplate), configFileMode)
if err != nil {
t.Fatalf("write config: %v", err)
}
out := ui.NewWithColor(&bytes.Buffer{}, false)
err = writeConfigSet(out, path, "storage_url", "file:///mnt/backups")
if err != nil {
t.Fatalf("config set storage_url: %v", err)
}
err = writeConfigSet(out, path, key, value)
if err != nil {
t.Fatalf("config set %s: %v", key, err)
}
cfg, err := config.Load(path)
if err != nil {
t.Fatalf("config.Load: %v", err)
}
return cfg
}
const testYAML = `# top comment const testYAML = `# top comment
compression_level: 3 compression_level: 3
age_recipients: age_recipients:
+9 -17
View File
@@ -224,7 +224,7 @@ type RemoteInfoResult struct {
// metadata/ is not a remote key. The names themselves are not // metadata/ is not a remote key. The names themselves are not
// reported: they come from the destination store and may hold // reported: they come from the destination store and may hold
// control characters. // control characters.
SkippedManifestCount int `json:"skipped_manifest_count,omitempty"` SkippedManifests int `json:"skipped_manifests,omitempty"`
} }
// RemoteInfo displays information about remote storage // RemoteInfo displays information about remote storage
@@ -250,23 +250,15 @@ func (v *Vaultik) RemoteInfo(jsonOutput bool) error {
v.stdoutf("Scanning snapshot metadata...\n") v.stdoutf("Scanning snapshot metadata...\n")
} }
snapshotMetadata, snapshotIDs, skippedManifestCount, err := v.collectSnapshotMetadata() snapshotMetadata, snapshotIDs, skippedManifests, err := v.collectSnapshotMetadata()
if err != nil { if err != nil {
return err return err
} }
result.SkippedManifestCount = skippedManifestCount result.SkippedManifests = skippedManifests
if showText { if showText {
manifestCount := 0 v.stdoutf("Downloading %d manifest(s)...\n", len(snapshotIDs))
for _, info := range snapshotMetadata {
if info.hasManifest {
manifestCount++
}
}
v.stdoutf("Downloading %d manifest(s)...\n", manifestCount)
} }
referencedBlobs, unreadableManifests := v.collectReferencedBlobsFromManifests( referencedBlobs, unreadableManifests := v.collectReferencedBlobsFromManifests(
@@ -307,7 +299,7 @@ func (v *Vaultik) collectSnapshotMetadata() (
map[string]*SnapshotMetadataInfo, []string, int, error, map[string]*SnapshotMetadataInfo, []string, int, error,
) { ) {
snapshotMetadata := make(map[string]*SnapshotMetadataInfo) snapshotMetadata := make(map[string]*SnapshotMetadataInfo)
skippedManifestCount := 0 skippedManifests := 0
metadataCh := v.Storage.ListStream(v.ctx, "metadata/") metadataCh := v.Storage.ListStream(v.ctx, "metadata/")
for obj := range metadataCh { for obj := range metadataCh {
@@ -332,7 +324,7 @@ func (v *Vaultik) collectSnapshotMetadata() (
"key", obj.Key) "key", obj.Key)
if isManifest { if isManifest {
skippedManifestCount++ skippedManifests++
} }
continue continue
@@ -364,7 +356,7 @@ func (v *Vaultik) collectSnapshotMetadata() (
sort.Strings(snapshotIDs) sort.Strings(snapshotIDs)
return snapshotMetadata, snapshotIDs, skippedManifestCount, nil return snapshotMetadata, snapshotIDs, skippedManifests, nil
} }
// collectReferencedBlobsFromManifests downloads the listed manifests // collectReferencedBlobsFromManifests downloads the listed manifests
@@ -468,7 +460,7 @@ func (v *Vaultik) scanRemoteBlobStorage(
// A blob named only by a manifest that could not be read, or by one // A blob named only by a manifest that could not be read, or by one
// under a skipped name, would be counted as orphaned, so the orphan // under a skipped name, would be counted as orphaned, so the orphan
// figures stay unknown. // figures stay unknown.
if len(result.UnreadableManifests) > 0 || result.SkippedManifestCount > 0 { if len(result.UnreadableManifests) > 0 || result.SkippedManifests > 0 {
return nil return nil
} }
@@ -544,7 +536,7 @@ func (v *Vaultik) printRemoteInfoTable(result *RemoteInfoResult) {
v.stdoutf("Orphaned (unreferenced): unknown "+ v.stdoutf("Orphaned (unreferenced): unknown "+
"(%d manifest(s) could not be read, "+ "(%d manifest(s) could not be read, "+
"%d manifest(s) under a non-conforming name skipped)\n", "%d manifest(s) under a non-conforming name skipped)\n",
len(result.UnreadableManifests), result.SkippedManifestCount) len(result.UnreadableManifests), result.SkippedManifests)
return return
} }
+2 -3
View File
@@ -112,7 +112,7 @@ func TestRemoteInfo_SkipsNonConformingMetadataName(t *testing.T) {
assert.Nil(t, doc["orphaned_blob_count"]) assert.Nil(t, doc["orphaned_blob_count"])
assert.Contains(t, doc, "orphaned_blob_size") assert.Contains(t, doc, "orphaned_blob_size")
assert.Nil(t, doc["orphaned_blob_size"]) assert.Nil(t, doc["orphaned_blob_size"])
assert.InDelta(t, 1, doc["skipped_manifest_count"], 0) assert.InDelta(t, 1, doc["skipped_manifests"], 0)
assert.NotContains(t, doc, "unreadable_manifests") assert.NotContains(t, doc, "unreadable_manifests")
} }
@@ -143,7 +143,6 @@ func TestRemoteInfo_DirectoryWithoutManifestLeavesOrphansKnown(t *testing.T) {
text := env.stdout.String() text := env.stdout.String()
assert.NotContains(t, text, "\x1b") assert.NotContains(t, text, "\x1b")
assert.Contains(t, text, "Downloading 1 manifest(s)...")
assert.Contains(t, text, "Orphaned (unreferenced): 1 (") assert.Contains(t, text, "Orphaned (unreferenced): 1 (")
assert.Contains(t, text, "Run 'vaultik prune' to remove orphaned blobs.") assert.Contains(t, text, "Run 'vaultik prune' to remove orphaned blobs.")
@@ -155,5 +154,5 @@ func TestRemoteInfo_DirectoryWithoutManifestLeavesOrphansKnown(t *testing.T) {
require.NoError(t, json.Unmarshal(env.stdout.Bytes(), &doc)) require.NoError(t, json.Unmarshal(env.stdout.Bytes(), &doc))
assert.InDelta(t, 1, doc["orphaned_blob_count"], 0) assert.InDelta(t, 1, doc["orphaned_blob_count"], 0)
assert.NotContains(t, doc, "unreadable_manifests") assert.NotContains(t, doc, "unreadable_manifests")
assert.NotContains(t, doc, "skipped_manifest_count") assert.NotContains(t, doc, "skipped_manifests")
} }