Make config set create a list for an index under a missing key (closes #308)

`config set age_recipients.0 age1...` on a file without `age_recipients`
wrote a map keyed `0`, after which no command could load the config. A
missing key is now created as a list when the next path element is a
number.

`config set` checks its result with the config loader before writing and
refuses a failure the change brings in: a config that loaded must still
load, and one still being filled in (failing validation until complete)
must still decode into the config type. The new `config.Parse`, split out
of `config.Load`, decodes config data, filling in `${...}` references,
without validating.

Rule suppressed: gosec G304 where `config.Load` and the new tests read a
file.

Model: opus-5-5

README.md

@@ -271,8 +271,16 @@ existing file. Created with mode `0600` since it will contain credentials.
 **`config set`**: Set a scalar config value by dotted YAML path
 (e.g. `vaultik config set compression_level 9`,
 `vaultik config set storage_url "file:///mnt/backups"`). Comments and
-formatting in the file are preserved; intermediate maps are created as
-needed.
+formatting in the file are preserved. Missing keys along the path are
+created: as a list when the next path element is a number (so
+`vaultik config set age_recipients.0 age1...` works on a file without
+`age_recipients`), otherwise as a map. If the config loads, a change
+after which it would not, such as `vaultik config set compression_level 99`,
+is refused and the file is left unchanged. A config that does not load
+yet, such as the one `config init` writes before `storage_url` is set,
+can be filled in one setting at a time: only a change that gives a
+setting the wrong type, such as a single value where a list belongs or a
+word where a number belongs, is refused.
 
 **`snapshot create`**: Perform incremental backup of configured snapshots.
 * Optional snapshot names argument to create specific snapshots (default: all)
@@ -293,8 +301,8 @@ needed.
 **`snapshot list`**: Show every snapshot known to this host — the union
 of the local index and the backup destination store — with timestamps
 and three sizes per snapshot (compressed remote size; total
-uncompressed chunk size; size of chunks newly referenced by that
-snapshot).
+uncompressed chunk size; size of the chunks that snapshot stored that
+were not stored before, recorded when its backup ran).
 
 Listing the destination store does **not** require the age secret key,
 so it works in vaultik's intended configuration, where the backed-up
@@ -307,8 +315,8 @@ only the hash, so hostname and snapshot name exist solely in the local
 index and in the encrypted per-snapshot database. Snapshots found only
 on the destination store are therefore listed as
 `<remote only:<abbreviated remote key>>` and show `<remote only>` in
-the uncompressed and "new chunk" columns, which can only be computed
-from the local index. Their timestamp and compressed size are real,
+the uncompressed and "new chunk" columns, which come only from the
+local index. Their timestamp and compressed size are real,
 read from the manifest.
 
 Snapshots in the local index with no counterpart on the destination

docs/DATAMODEL.md

@@ -119,6 +119,7 @@ Tracks backup snapshots.
 - `completed_at` (INTEGER) - Completion timestamp (NULL if in progress)
 - `file_count` (INTEGER) - Number of files in snapshot
 - `chunk_count` (INTEGER) - Number of chunks this snapshot stored that were not stored before
+- `new_chunk_size` (INTEGER) - Total size of those chunks before compression; `snapshot list` shows it as NEW CHUNK SIZE
 - `blob_count` (INTEGER) - Number of blobs this snapshot created
 - `total_size` (INTEGER) - Total size of all files
 - `blob_size` (INTEGER) - Total compressed size of all referenced blobs

internal/cli/config.go

@@ -44,6 +44,9 @@ var (
 	errNeedNumericIndex = errors.New("key is a list; use a numeric index")
 	errIndexOutOfRange  = errors.New("index out of range")
 	errNotMapOrList     = errors.New("key is not a map or list")
+	errWouldNotLoad     = errors.New(
+		"vaultik would not load the config after this change, " +
+			"so the file was not changed")
 )
 
 const defaultConfigTemplate = `# vaultik configuration
@@ -391,8 +394,15 @@ func newConfigSetCommand() *cobra.Command {
 		Use:   "set <key> <value>",
 		Short: "Set a config value by dotted path (e.g. compression_level 5)",
 		Long: `Sets a scalar config value addressed by dotted YAML path and writes
-the file back, preserving comments and formatting. Intermediate maps
-are created as needed.
+the file back, preserving comments and formatting. Missing keys along
+the path are created: as a list when the next path element is a
+number, otherwise as a map.
+
+If the config loads, a change after which it would not, such as
+compression_level 99, is refused and the file is left unchanged. A
+config that does not load yet, such as the one config init writes
+before storage_url is set, can be filled in one setting at a time:
+only a change that gives a setting the wrong type is refused.
 
 Examples:
   vaultik config set storage_url "file:///mnt/backups"
@@ -422,6 +432,11 @@ func writeConfigSet(out *ui.Writer, path, key, value string) error {
 		return err
 	}
 
+	oldData, err := marshalConfigYAML(root)
+	if err != nil {
+		return fmt.Errorf("marshaling config: %w", err)
+	}
+
 	err = yamlPathSet(root, strings.Split(key, "."), value)
 	if err != nil {
 		return err
@@ -432,6 +447,11 @@ func writeConfigSet(out *ui.Writer, path, key, value string) error {
 		return fmt.Errorf("marshaling config: %w", err)
 	}
 
+	err = checkConfigChange(oldData, data)
+	if err != nil {
+		return fmt.Errorf("%w: %w", errWouldNotLoad, err)
+	}
+
 	err = os.WriteFile(path, data, configFileMode)
 	if err != nil {
 		return fmt.Errorf("writing config file: %w", err)
@@ -453,6 +473,28 @@ func writeConfigSet(out *ui.Writer, path, key, value string) error {
 	return nil
 }
 
+// checkConfigChange returns the reason vaultik would not load newData, the
+// config after a change, if the change brings that failure in. If oldData,
+// the config before it, decodes into the config type, the new one must
+// too; if the old one also passes validation, the new one must pass it. A
+// config still being filled in fails validation until it is complete, so
+// only decoding is checked for it.
+func checkConfigChange(oldData, newData []byte) error {
+	oldCfg, oldErr := config.Parse(oldData)
+	if oldErr == nil {
+		newCfg, err := config.Parse(newData)
+		if err != nil {
+			return err
+		}
+
+		if oldCfg.Validate() == nil {
+			return newCfg.Validate()
+		}
+	}
+
+	return nil
+}
+
 // marshalConfigYAML renders a config document tree with 2-space indentation,
 // matching defaultConfigTemplate. yaml.Marshal defaults to 4 spaces, which
 // would reindent the whole file on the first `config set` despite the promise
@@ -559,8 +601,8 @@ func yamlPathGet(root *yaml.Node, keys []string) (*yaml.Node, error) {
 	return node, nil
 }
 
-// yamlPathSet navigates a dotted key path, creating intermediate maps as
-// needed, and sets the final key to the given scalar value. Numeric path
+// yamlPathSet navigates a dotted key path, creating missing keys as needed,
+// and sets the final key to the given scalar value. Numeric path
 // components index into sequences; an index equal to the sequence length
 // appends a new element (e.g. "age_recipients.1" on a 1-element list).
 func yamlPathSet(root *yaml.Node, keys []string, value string) error {
@@ -573,12 +615,12 @@ func yamlPathSet(root *yaml.Node, keys []string, value string) error {
 		node = node.Content[0]
 	}
 
-	for i, key := range keys {
+	for i := range keys {
 		last := i == len(keys)-1
 
 		switch node.Kind {
 		case yaml.MappingNode:
-			node = yamlSetInMapping(node, key, value, last)
+			node = yamlSetInMapping(node, keys, i, value, last)
 		case yaml.SequenceNode:
 			next, err := yamlSetInSequence(node, keys, i, value, last)
 			if err != nil {
@@ -647,14 +689,16 @@ func yamlField(typ reflect.Type, key string) (reflect.StructField, bool) {
 	return reflect.StructField{}, false
 }
 
-// yamlSetInMapping resolves (creating if needed) the value node for key
-// within a mapping node, setting it to value when it is the final path
-// element, and returns the node to descend into.
-func yamlSetInMapping(node *yaml.Node, key, value string, last bool) *yaml.Node {
+// yamlSetInMapping resolves (creating if needed) the value node for the
+// path element keys[i] within a mapping node, setting it to value when it
+// is the final path element, and returns the node to descend into.
+func yamlSetInMapping(
+	node *yaml.Node, keys []string, i int, value string, last bool,
+) *yaml.Node {
 	var valueNode *yaml.Node
 
 	for j := 0; j+1 < len(node.Content); j += 2 {
-		if node.Content[j].Value == key {
+		if node.Content[j].Value == keys[i] {
 			valueNode = node.Content[j+1]
 
 			break
@@ -662,13 +706,8 @@ func yamlSetInMapping(node *yaml.Node, key, value string, last bool) *yaml.Node
 	}
 
 	if valueNode == nil {
-		keyNode := &yaml.Node{Kind: yaml.ScalarNode, Value: key}
-
-		valueNode = &yaml.Node{Kind: yaml.MappingNode}
-		if last {
-			valueNode = &yaml.Node{Kind: yaml.ScalarNode, Value: value}
-		}
-
+		keyNode := &yaml.Node{Kind: yaml.ScalarNode, Value: keys[i]}
+		valueNode = newPathNode(keys, i, value)
 		node.Content = append(node.Content, keyNode, valueNode)
 	} else if last {
 		setScalar(valueNode, value)
@@ -696,12 +735,7 @@ func yamlSetInSequence(
 	}
 
 	if idx == len(node.Content) {
-		newNode := &yaml.Node{Kind: yaml.MappingNode}
-		if last {
-			newNode = &yaml.Node{Kind: yaml.ScalarNode, Value: value}
-		}
-
-		node.Content = append(node.Content, newNode)
+		node.Content = append(node.Content, newPathNode(keys, i, value))
 	} else if last {
 		setScalar(node.Content[idx], value)
 	}
@@ -709,6 +743,22 @@ func yamlSetInSequence(
 	return node.Content[idx], nil
 }
 
+// newPathNode returns the node to create for the missing path element
+// keys[i]: the value when it is the final path element, otherwise an empty
+// list when the next path element is a number and an empty map when not.
+func newPathNode(keys []string, i int, value string) *yaml.Node {
+	if i == len(keys)-1 {
+		return &yaml.Node{Kind: yaml.ScalarNode, Value: value}
+	}
+
+	_, err := strconv.Atoi(keys[i+1])
+	if err == nil {
+		return &yaml.Node{Kind: yaml.SequenceNode}
+	}
+
+	return &yaml.Node{Kind: yaml.MappingNode}
+}
+
 // setScalar overwrites a node in place with a plain scalar value.
 func setScalar(n *yaml.Node, value string) {
 	n.Kind = yaml.ScalarNode

internal/cli/config_test.go

@@ -58,14 +58,17 @@ func TestDefaultConfigTemplateParses(t *testing.T) {
 	}
 }
 
+// testRecipient is a valid age public key.
+const testRecipient = "age1278m9q7dp3chsh2dcy82qk27v047zywyvtxwnj4cvt0z65jw6a7q5dqhfj"
+
 // TestConfigSetRecipientOnFreshConfig follows the README quickstart: on the
 // file `config init` writes, `config set age_recipients.0` and
 // `config set storage_url` give a config that loads with that recipient.
+// Neither is refused, although the config does not load until both are set;
+// once it loads, a recipient that is not a valid key is.
 func TestConfigSetRecipientOnFreshConfig(t *testing.T) {
 	t.Parallel()
 
-	const recipient = "age1278m9q7dp3chsh2dcy82qk27v047zywyvtxwnj4cvt0z65jw6a7q5dqhfj"
-
 	path := filepath.Join(t.TempDir(), "config.yml")
 
 	err := os.WriteFile(path, []byte(defaultConfigTemplate), configFileMode)
@@ -75,7 +78,7 @@ func TestConfigSetRecipientOnFreshConfig(t *testing.T) {
 
 	out := ui.NewWithColor(&bytes.Buffer{}, false)
 
-	err = writeConfigSet(out, path, "age_recipients.0", recipient)
+	err = writeConfigSet(out, path, "age_recipients.0", testRecipient)
 	if err != nil {
 		t.Fatalf("config set age_recipients.0: %v", err)
 	}
@@ -90,8 +93,186 @@ func TestConfigSetRecipientOnFreshConfig(t *testing.T) {
 		t.Fatalf("config.Load: %v", err)
 	}
 
-	if len(cfg.AgeRecipients) != 1 || cfg.AgeRecipients[0] != recipient {
-		t.Errorf("age_recipients = %v, want [%s]", cfg.AgeRecipients, recipient)
+	if len(cfg.AgeRecipients) != 1 || cfg.AgeRecipients[0] != testRecipient {
+		t.Errorf("age_recipients = %v, want [%s]", cfg.AgeRecipients, testRecipient)
+	}
+
+	err = writeConfigSet(out, path, "age_recipients.0", "notakey")
+	if err == nil || !strings.Contains(err.Error(), "not a valid recipient") {
+		t.Errorf("config set age_recipients.0 notakey gives error %v, "+
+			"want one saying it is not a valid recipient", err)
+	}
+
+	_, err = config.Load(path)
+	if err != nil {
+		t.Errorf("config.Load after the refused change: %v", err)
+	}
+}
+
+// configWithoutRecipients loads, but has no age_recipients key and no
+// snapshot named work.
+const configWithoutRecipients = `storage_url: file:///mnt/backups
+snapshots:
+  home:
+    paths:
+      - "~"
+`
+
+// TestConfigSetIndexUnderMissingKey checks that a list index under a key
+// the file does not have creates a list, and that the result loads.
+func TestConfigSetIndexUnderMissingKey(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		key   string
+		value string
+		field func(cfg *config.Config) []string
+	}{
+		{"age_recipients.0", testRecipient,
+			func(cfg *config.Config) []string { return cfg.AgeRecipients }},
+		{"snapshots.work.paths.0", "/srv/work",
+			func(cfg *config.Config) []string { return cfg.Snapshots["work"].Paths }},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.key, func(t *testing.T) {
+			t.Parallel()
+
+			path := filepath.Join(t.TempDir(), "config.yml")
+
+			err := os.WriteFile(path, []byte(configWithoutRecipients), configFileMode)
+			if err != nil {
+				t.Fatalf("write config: %v", err)
+			}
+
+			err = writeConfigSet(ui.NewWithColor(&bytes.Buffer{}, false),
+				path, tt.key, tt.value)
+			if err != nil {
+				t.Fatalf("config set %s: %v", tt.key, err)
+			}
+
+			cfg, err := config.Load(path)
+			if err != nil {
+				t.Fatalf("config.Load: %v", err)
+			}
+
+			got := tt.field(cfg)
+			if len(got) != 1 || got[0] != tt.value {
+				t.Errorf("%s gives %v, want [%s]", tt.key, got, tt.value)
+			}
+		})
+	}
+}
+
+// TestConfigSetKeepsFileThatWouldNotLoad checks that config set refuses a
+// change after which the config would not load, with an error giving the
+// reason, and leaves the file as it was. On the file `config init` writes,
+// which does not load until storage_url is set, only a value of the wrong
+// type is refused.
+func TestConfigSetKeepsFileThatWouldNotLoad(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		config string
+		key    string
+		value  string
+		reason string
+	}{
+		// Index 1 of the empty list the missing key gives.
+		{configWithoutRecipients, "age_recipients.1", testRecipient,
+			"index out of range"},
+		// A list set to a single value.
+		{configWithoutRecipients, "snapshots.home.paths", "/srv",
+			"into []string"},
+		{configWithoutRecipients, "chunk_size", "fast",
+			"invalid size format"},
+		// Values of the right type that fail validation.
+		{configWithoutRecipients, "compression_level", "99",
+			"compression_level must be between 1 and 19"},
+		{configWithoutRecipients, "chunk_size", "1KB",
+			"chunk_size must be at least 1MB"},
+		// The type check on a config that does not load yet.
+		{defaultConfigTemplate, "snapshots.apps.paths", "/srv",
+			"into []string"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.key+" "+tt.value, func(t *testing.T) {
+			t.Parallel()
+
+			path := filepath.Join(t.TempDir(), "config.yml")
+
+			err := os.WriteFile(path, []byte(tt.config), configFileMode)
+			if err != nil {
+				t.Fatalf("write config: %v", err)
+			}
+
+			err = writeConfigSet(ui.NewWithColor(&bytes.Buffer{}, false),
+				path, tt.key, tt.value)
+			if err == nil || !strings.Contains(err.Error(), tt.reason) {
+				t.Errorf("config set %s %s gives error %v, want one saying %q",
+					tt.key, tt.value, err, tt.reason)
+			}
+
+			data, err := os.ReadFile(path) //nolint:gosec // G304: test-controlled path
+			if err != nil {
+				t.Fatalf("read config: %v", err)
+			}
+
+			if string(data) != tt.config {
+				t.Errorf("config set %s changed the file to:\n%s",
+					tt.key, data)
+			}
+		})
+	}
+}
+
+// TestConfigSetRefusesOnlyItsOwnFailure checks that setting another key
+// works on a config with a ${...} reference on compression_level, which
+// loads because loading fills the reference in, and on one whose
+// compression_level already fails to load.
+func TestConfigSetRefusesOnlyItsOwnFailure(t *testing.T) {
+	t.Parallel()
+
+	levelPath := filepath.Join(t.TempDir(), "level")
+
+	err := os.WriteFile(levelPath, []byte("9"), configFileMode)
+	if err != nil {
+		t.Fatalf("write level: %v", err)
+	}
+
+	configs := map[string]string{
+		"reference": configWithoutRecipients +
+			"compression_level: ${FILE:" + levelPath + "}\n",
+		"failing": configWithoutRecipients + "compression_level: fast\n",
+	}
+
+	for name, configYAML := range configs {
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+
+			path := filepath.Join(t.TempDir(), "config.yml")
+
+			err := os.WriteFile(path, []byte(configYAML), configFileMode)
+			if err != nil {
+				t.Fatalf("write config: %v", err)
+			}
+
+			err = writeConfigSet(ui.NewWithColor(&bytes.Buffer{}, false),
+				path, "hostname", "myhost")
+			if err != nil {
+				t.Fatalf("config set hostname: %v", err)
+			}
+
+			data, err := os.ReadFile(path) //nolint:gosec // G304: test-controlled path
+			if err != nil {
+				t.Fatalf("read config: %v", err)
+			}
+
+			if !strings.Contains(string(data), "hostname: myhost") {
+				t.Errorf("config set hostname wrote:\n%s", data)
+			}
+		})
 	}
 }
 

internal/config/config.go

@@ -4,6 +4,7 @@
 package config
 
 import (
+	"bytes"
 	"errors"
 	"fmt"
 	"os"
@@ -233,14 +234,48 @@ func New(path Path) (*Config, error) {
 	return cfg, nil
 }
 
-// Load reads and parses the configuration file from the specified path.
-// It applies default values for optional fields, performs environment variable
-// substitution using smartconfig, and validates the configuration.
-// The configuration file should be in YAML format. Returns an error if the file
-// cannot be read, parsed, or if validation fails.
+// Load reads the configuration file at path, parses it with Parse, warns
+// when others can read the file, and validates the configuration.
+// Returns an error if the file cannot be read, parsed, or if validation
+// fails.
 func Load(path string) (*Config, error) {
+	data, err := os.ReadFile(path) //nolint:gosec // G304: config path is operator-supplied
+	if err != nil {
+		return nil, fmt.Errorf("failed to load config file: %w", err)
+	}
+
+	cfg, err := Parse(data)
+	if err != nil {
+		return nil, err
+	}
+
+	// Check config file permissions (warn if world or group readable)
+	//nolint:gosec // G703: config path is operator-supplied by design
+	info, statErr := os.Stat(path)
+	if statErr == nil {
+		mode := info.Mode().Perm()
+		if mode&0044 != 0 { // group or world readable
+			log.Warn(cfg.readableByOthersWarning(),
+				"path", path,
+				"mode", fmt.Sprintf("%04o", mode),
+				"recommendation", "chmod 600 "+path)
+		}
+	}
+
+	err = cfg.Validate()
+	if err != nil {
+		return nil, fmt.Errorf("invalid config: %w", err)
+	}
+
+	return cfg, nil
+}
+
+// Parse reads YAML config data as Load does, without validating it. It
+// applies default values for optional fields, fills in ${...} references
+// using smartconfig, and expands ~ in paths.
+func Parse(data []byte) (*Config, error) {
 	// Load config using smartconfig for interpolation
-	sc, err := smartconfig.NewFromConfigPath(path)
+	sc, err := smartconfig.NewFromReader(bytes.NewReader(data))
 	if err != nil {
 		return nil, fmt.Errorf("failed to load config file: %w", err)
 	}
@@ -302,24 +337,6 @@ func Load(path string) (*Config, error) {
 		cfg.S3.Region = "us-east-1"
 	}
 
-	// Check config file permissions (warn if world or group readable)
-	//nolint:gosec // G703: config path is operator-supplied by design
-	info, statErr := os.Stat(path)
-	if statErr == nil {
-		mode := info.Mode().Perm()
-		if mode&0044 != 0 { // group or world readable
-			log.Warn(cfg.readableByOthersWarning(),
-				"path", path,
-				"mode", fmt.Sprintf("%04o", mode),
-				"recommendation", "chmod 600 "+path)
-		}
-	}
-
-	err = cfg.Validate()
-	if err != nil {
-		return nil, fmt.Errorf("invalid config: %w", err)
-	}
-
 	return cfg, nil
 }
 

internal/database/schema/001.sql

@@ -86,6 +86,7 @@ CREATE TABLE IF NOT EXISTS snapshots (
     completed_at INTEGER,
     file_count INTEGER NOT NULL DEFAULT 0,
     chunk_count INTEGER NOT NULL DEFAULT 0,
+    new_chunk_size INTEGER NOT NULL DEFAULT 0,
     blob_count INTEGER NOT NULL DEFAULT 0,
     total_size INTEGER NOT NULL DEFAULT 0,
     blob_size INTEGER NOT NULL DEFAULT 0,

internal/database/snapshots.go

@@ -129,6 +129,7 @@ func (r *SnapshotRepository) UpdateExtendedStats(
 	compressionLevel int,
 	uploadBytes int64,
 	uploadDurationMs int64,
+	newChunkSize int64,
 ) error {
 	compressionRatio, err := r.extendedCompressionRatio(
 		ctx, tx, snapshotID, blobUncompressedSize,
@@ -143,18 +144,19 @@ func (r *SnapshotRepository) UpdateExtendedStats(
 		    compression_ratio = ?,
 		    compression_level = ?,
 		    upload_bytes = ?,
-		    upload_duration_ms = ?
+		    upload_duration_ms = ?,
+		    new_chunk_size = ?
 		WHERE id = ?
 	`
 
 	if tx != nil {
 		_, err = tx.ExecContext(ctx, query,
 			blobUncompressedSize, compressionRatio, compressionLevel,
-			uploadBytes, uploadDurationMs, snapshotID)
+			uploadBytes, uploadDurationMs, newChunkSize, snapshotID)
 	} else {
 		_, err = r.db.ExecWithLog(ctx, query,
 			blobUncompressedSize, compressionRatio, compressionLevel,
-			uploadBytes, uploadDurationMs, snapshotID)
+			uploadBytes, uploadDurationMs, newChunkSize, snapshotID)
 	}
 
 	if err != nil {
@@ -594,45 +596,24 @@ func (r *SnapshotRepository) GetSnapshotUncompressedChunkSize(
 	return totalSize, nil
 }
 
-// GetSnapshotNewChunkSize returns the sum of plaintext sizes of chunks that are
-// referenced by this snapshot but not by any earlier completed snapshot known to
-// the local database. The result is the marginal uncompressed data this snapshot
-// added to the dedup pool — i.e., the delta from prior snapshots.
+// GetSnapshotNewChunkSize returns the total plaintext size of the chunks the
+// snapshot's backup stored that were not stored before, as the backup
+// recorded it. It is not derived from the snapshot's files: a file's row is
+// shared by every snapshot that lists it and is rewritten when the file
+// changes, so it shows only the file's latest chunks.
 func (r *SnapshotRepository) GetSnapshotNewChunkSize(
 	ctx context.Context, snapshotID string,
 ) (int64, error) {
-	query := `
-		WITH this_snap_chunks AS (
-			SELECT DISTINCT fc.chunk_hash
-			FROM snapshot_files sf
-			JOIN file_chunks fc ON sf.file_id = fc.file_id
-			WHERE sf.snapshot_id = ?
-		),
-		prior_chunks AS (
-			SELECT DISTINCT fc.chunk_hash
-			FROM snapshots s
-			JOIN snapshot_files sf ON sf.snapshot_id = s.id
-			JOIN file_chunks fc ON fc.file_id = sf.file_id
-			WHERE s.completed_at IS NOT NULL
-			  AND s.id != ?
-			  AND s.started_at < (SELECT started_at FROM snapshots WHERE id = ?)
-		)
-		SELECT COALESCE(SUM(c.size), 0)
-		FROM chunks c
-		JOIN this_snap_chunks t ON c.chunk_hash = t.chunk_hash
-		WHERE c.chunk_hash NOT IN (SELECT chunk_hash FROM prior_chunks)
-	`
+	query := `SELECT new_chunk_size FROM snapshots WHERE id = ?`
 
-	var totalSize int64
+	var newChunkSize int64
 
-	err := r.db.conn.QueryRowContext(
-		ctx, query, snapshotID, snapshotID, snapshotID,
-	).Scan(&totalSize)
+	err := r.db.conn.QueryRowContext(ctx, query, snapshotID).Scan(&newChunkSize)
 	if err != nil {
 		return 0, fmt.Errorf("querying new chunk size: %w", err)
 	}
 
-	return totalSize, nil
+	return newChunkSize, nil
 }
 
 // GetIncompleteSnapshots returns all snapshots that haven't been completed

internal/snapshot/scanner.go

@@ -135,7 +135,9 @@ type ScannerConfig struct {
 // are counted per file: BytesScanned is the size of the new and changed
 // files, BytesSkipped that of the unchanged ones. FilesFailed counts the
 // new and changed files that phase 2 could not store; FilesScanned
-// includes them and BytesScanned does not.
+// includes them and BytesScanned does not. ChunksCreated counts the
+// chunks that were not already stored, and NewChunkSize is their total
+// size.
 type ScanResult struct {
 	FilesScanned   int
 	FilesSkipped   int
@@ -145,6 +147,7 @@ type ScanResult struct {
 	BytesSkipped   int64
 	BytesDeleted   int64
 	ChunksCreated  int
+	NewChunkSize   int64
 	BlobsCreated   int
 	BlobsUploaded  int
 	BytesUploaded  int64
@@ -1866,11 +1869,12 @@ func (s *Scanner) processFileStreaming(
 	return nil
 }
 
-// updateChunkStats counts a chunk that was not already stored. The scan
-// result's file counts, BytesScanned and BytesSkipped are not touched
-// here: the scan phase counts each file once.
+// updateChunkStats counts a chunk that was not already stored, and its
+// size. The scan result's file counts, BytesScanned and BytesSkipped are
+// not touched here: the scan phase counts each file once.
 func (s *Scanner) updateChunkStats(chunkSize int64, result *ScanResult) {
 	result.ChunksCreated++
+	result.NewChunkSize += chunkSize
 
 	if s.progress != nil {
 		s.progress.GetStats().ChunksCreated.Add(1)

internal/snapshot/snapshot.go

@@ -199,6 +199,7 @@ func (sm *SnapshotManager) UpdateSnapshotStatsExtended(
 			stats.CompressionLevel,
 			stats.BytesUploaded,
 			stats.UploadDurationMs,
+			stats.NewChunkSize,
 		)
 	})
 }
@@ -906,6 +907,7 @@ type ExtendedBackupStats struct {
 	BlobUncompressedSize int64 // Total uncompressed size of all referenced blobs
 	CompressionLevel     int   // Compression level used for this snapshot
 	UploadDurationMs     int64 // Total milliseconds spent uploading to S3
+	NewChunkSize         int64 // Total size of the chunks counted in ChunksCreated
 }
 
 // deleteOtherSnapshots deletes all snapshots except the current one

internal/vaultik/new_chunk_size_test.go

@@ -0,0 +1,133 @@
+package vaultik_test
+
+import (
+	"bytes"
+	"context"
+	"maps"
+	"path/filepath"
+	"testing"
+
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"sneak.berlin/go/vaultik/internal/chunker"
+	"sneak.berlin/go/vaultik/internal/database"
+	"sneak.berlin/go/vaultik/internal/log"
+	"sneak.berlin/go/vaultik/internal/storage"
+	"sneak.berlin/go/vaultik/internal/vaultik"
+)
+
+// TestNewChunkSizeOfChangedFileStaysFixed covers
+// https://git.eeqj.de/sneak/vaultik/issues/311. A backup whose only change
+// is a modified file lists the size of the chunks that change added as its
+// NEW CHUNK SIZE, and a later backup that changes the file again leaves
+// every earlier snapshot's figure as it was. The file's row in the local
+// index is shared by all three snapshots and rewritten by each backup.
+func TestNewChunkSizeOfChangedFileStaysFixed(t *testing.T) {
+	log.Initialize(log.Config{})
+	t.Parallel()
+
+	fs := afero.NewOsFs()
+	tempDir := t.TempDir()
+	dataDir := filepath.Join(tempDir, "src")
+	dbPath := filepath.Join(tempDir, "index.sqlite")
+	changedPath := filepath.Join(dataDir, "changed.bin")
+
+	ctx := context.Background()
+	files := writeFaultSourceTree(t, fs, dataDir)
+	cfg := changedFileConfig(dataDir, dbPath)
+
+	appendRandomBytes(t, fs, files, changedPath, appendedFileSize)
+
+	store, err := storage.NewFileStorer(filepath.Join(tempDir, "remote"))
+	require.NoError(t, err)
+
+	db, err := database.New(ctx, dbPath)
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = db.Close() })
+
+	repos := database.NewRepositories(db)
+	v := newBackupVaultik(ctx, cfg, store, repos, db, fs)
+
+	require.NoError(t, backUp(v, "first"))
+
+	firstFiles := maps.Clone(files)
+	firstID := localSnapshotID(ctx, t, repos, "first")
+
+	appendRandomBytes(t, fs, files, changedPath, faultChunkSize)
+	require.NoError(t, backUp(v, "second"))
+
+	secondID := localSnapshotID(ctx, t, repos, "second")
+	secondAdded := sizeOfNewChunks(t, files, firstFiles)
+	require.Positive(t, secondAdded)
+	assert.Equal(t, secondAdded, listedNewChunkSizes(t, v)[secondID],
+		"the second snapshot must count the chunks the change added")
+
+	appendRandomBytes(t, fs, files, changedPath, faultChunkSize)
+	require.NoError(t, backUp(v, "third"))
+
+	sizes := listedNewChunkSizes(t, v)
+	assert.Equal(t, sizeOfNewChunks(t, firstFiles, nil), sizes[firstID],
+		"a later backup must not change the first snapshot's figure")
+	assert.Equal(t, secondAdded, sizes[secondID],
+		"a later backup must not change the second snapshot's figure")
+}
+
+// sizeOfNewChunks returns the total size of the distinct chunks of the
+// files in after that no file in before has, cut the way the backup cuts
+// them.
+func sizeOfNewChunks(t *testing.T, after, before map[string][]byte) int64 {
+	t.Helper()
+
+	known := make(map[string]bool)
+	for _, chunk := range chunksOf(t, before) {
+		known[chunk.Hash] = true
+	}
+
+	var total int64
+
+	for _, chunk := range chunksOf(t, after) {
+		if !known[chunk.Hash] {
+			known[chunk.Hash] = true
+			total += chunk.Size
+		}
+	}
+
+	return total
+}
+
+// chunksOf returns the chunks of every file in files.
+func chunksOf(t *testing.T, files map[string][]byte) []chunker.Chunk {
+	t.Helper()
+
+	c := chunker.NewChunker(faultChunkSize)
+
+	var all []chunker.Chunk
+
+	for _, content := range files {
+		chunks, err := c.ChunkReader(bytes.NewReader(content))
+		require.NoError(t, err)
+
+		all = append(all, chunks...)
+	}
+
+	return all
+}
+
+// listedNewChunkSizes runs `snapshot list --json` and returns each listed
+// snapshot's new_chunk_size by snapshot ID.
+func listedNewChunkSizes(t *testing.T, v *vaultik.Vaultik) map[string]int64 {
+	t.Helper()
+
+	var out bytes.Buffer
+
+	v.Stdout = &out
+	require.NoError(t, v.ListSnapshots(true))
+
+	sizes := make(map[string]int64)
+	for _, row := range decodeListJSON(t, out.String()) {
+		sizes[row.ID] = row.NewChunkSize
+	}
+
+	return sizes
+}

internal/vaultik/restore.go

@@ -309,11 +309,22 @@ func (v *Vaultik) restoreAllFiles(
 		return nil, err
 	}
 
-	plan, err := newRestorePlan(v.ctx, repos, files, chunkToBlobMap, blobIDToHash)
+	plan, unplanned, err := newRestorePlan(
+		v.ctx, repos, files, chunkToBlobMap, blobIDToHash)
 	if err != nil {
 		return nil, fmt.Errorf("building restore plan: %w", err)
 	}
 
+	// Ranging over files rather than the map reports them in a stable order.
+	for _, file := range files {
+		if planErr, ok := unplanned[file.ID]; ok {
+			err = v.handleRestoreFileError(plan, opts, result, file, file.ID, planErr)
+			if err != nil {
+				return nil, err
+			}
+		}
+	}
+
 	filesByID, totalBytesExpected := indexRestoreFiles(files)
 
 	v.UI.Beginf("Restoring %s files (%s) to %s.",

internal/vaultik/restore_malformed_db_test.go

@@ -61,8 +61,8 @@ func craftedFile(
 }
 
 // TestRestoreShortChunkHashInNoBlob proves a file whose short chunk hash
-// has no blob_chunks row fails restore planning and the chunk write with
-// an error.
+// has no blob_chunks row is left out of the restore plan with an error,
+// and fails the chunk write with an error.
 func TestRestoreShortChunkHashInNoBlob(t *testing.T) {
 	t.Parallel()
 
@@ -77,9 +77,11 @@ func TestRestoreShortChunkHashInNoBlob(t *testing.T) {
 	chunkToBlobMap, err := v.buildChunkToBlobMap(ctx, repos)
 	require.NoError(t, err)
 
-	_, err = newRestorePlan(ctx, repos, []*database.File{file},
+	plan, unplanned, err := newRestorePlan(ctx, repos, []*database.File{file},
 		chunkToBlobMap, map[string]string{})
-	require.ErrorIs(t, err, errPlanChunkMissing)
+	require.NoError(t, err)
+	require.ErrorIs(t, unplanned[file.ID], errPlanChunkMissing)
+	require.False(t, plan.hasPending())
 
 	fileChunks, err := repos.FileChunks.GetByFileID(ctx, file.ID)
 	require.NoError(t, err)

internal/vaultik/restore_plan.go

@@ -41,20 +41,25 @@ type restorePlan struct {
 
 // newRestorePlan builds the file→blob index for the given files. Files
 // whose chunks reference no blobs (symlinks, directories) start in the
-// ready queue immediately.
+// ready queue immediately. A file with a chunk that no blob holds cannot
+// be restored: it is left out of the plan and returned, by ID, with the
+// reason.
 func newRestorePlan(
 	ctx context.Context,
 	repos *database.Repositories,
 	files []*database.File,
 	chunkToBlobMap map[string]*database.BlobChunk,
 	blobIDToHash map[string]string,
-) (*restorePlan, error) {
+) (*restorePlan, map[types.FileID]error, error) {
 	p := &restorePlan{
 		fileBlobs: make(map[types.FileID]map[string]struct{}, len(files)),
 		blobFiles: make(map[string]map[types.FileID]struct{}),
 		ready:     make([]types.FileID, 0, len(files)),
 		cached:    make(map[string]struct{}),
 	}
+	unplanned := make(map[types.FileID]error)
+
+nextFile:
 	for _, f := range files {
 		if f.IsSymlink() || f.Mode&uint32(os.ModeDir) != 0 {
 			// No chunks to fetch — restore can run immediately.
@@ -66,7 +71,7 @@ func newRestorePlan(
 
 		fileChunks, err := repos.FileChunks.GetByFileID(ctx, f.ID)
 		if err != nil {
-			return nil, fmt.Errorf("planning %s: %w", f.Path, err)
+			return nil, nil, fmt.Errorf("planning %s: %w", f.Path, err)
 		}
 
 		blobs := make(map[string]struct{})
@@ -74,14 +79,18 @@ func newRestorePlan(
 		for _, fc := range fileChunks {
 			bc, ok := chunkToBlobMap[fc.ChunkHash.String()]
 			if !ok {
-				return nil, fmt.Errorf("planning %s: %w: %s",
-					f.Path, errPlanChunkMissing, shortHash(fc.ChunkHash.String()))
+				unplanned[f.ID] = fmt.Errorf("%w: %s",
+					errPlanChunkMissing, shortHash(fc.ChunkHash.String()))
+
+				continue nextFile
 			}
 
 			hash, ok := blobIDToHash[bc.BlobID.String()]
 			if !ok {
-				return nil, fmt.Errorf("planning %s: %w: %s",
-					f.Path, errPlanBlobIDMissing, bc.BlobID)
+				unplanned[f.ID] = fmt.Errorf("%w: %s",
+					errPlanBlobIDMissing, bc.BlobID)
+
+				continue nextFile
 			}
 
 			blobs[hash] = struct{}{}
@@ -103,7 +112,7 @@ func newRestorePlan(
 		}
 	}
 
-	return p, nil
+	return p, unplanned, nil
 }
 
 // markBlobCached records that the named blob is now resident in the

internal/vaultik/restore_skip_errors_test.go

@@ -28,7 +28,8 @@ const (
 )
 
 // missingBlobBackup is a snapshot of single-chunk files spread over two
-// blobs, with one of those blobs deleted from the store.
+// blobs, with one of those blobs lost: deleted from the store, or missing
+// from the snapshot metadata.
 type missingBlobBackup struct {
 	fs         afero.Fs
 	cfg        *config.Config
@@ -37,20 +38,39 @@ type missingBlobBackup struct {
 	restoreDir string
 	// files holds the original content by source path.
 	files map[string][]byte
-	// lost holds the source paths whose chunk is in the deleted blob.
+	// lost holds the source paths whose chunk is in the lost blob.
 	lost map[string]bool
 }
 
 // TestRestoreSkipErrorsSkipsFilesOfMissingBlob restores with SkipErrors
-// after one blob of a two-blob snapshot was deleted. Every file stored in
-// that blob must be reported as failed and left absent, every other file
-// must be restored intact, and Restore must still return an error.
+// after one blob of a two-blob snapshot was deleted from the store.
 func TestRestoreSkipErrorsSkipsFilesOfMissingBlob(t *testing.T) {
 	log.Initialize(log.Config{})
 	t.Parallel()
 
 	ctx := context.Background()
-	backup := backupThenDeleteOneBlob(ctx, t)
+	assertRestoreSkipsLostFiles(ctx, t, backupThenDeleteOneBlob(ctx, t))
+}
+
+// TestRestoreSkipErrorsSkipsFilesWithChunkInNoBlob restores with
+// SkipErrors a snapshot whose metadata lists chunks that no blob holds, as
+// earlier backup defects left behind.
+func TestRestoreSkipErrorsSkipsFilesWithChunkInNoBlob(t *testing.T) {
+	log.Initialize(log.Config{})
+	t.Parallel()
+
+	ctx := context.Background()
+	assertRestoreSkipsLostFiles(ctx, t, backupThenDropOneBlobFromMetadata(ctx, t))
+}
+
+// assertRestoreSkipsLostFiles restores backup with SkipErrors. Every file
+// stored in the lost blob must be reported as failed and left absent,
+// every other file must be restored intact, and Restore must still return
+// an error.
+func assertRestoreSkipsLostFiles(
+	ctx context.Context, t *testing.T, backup *missingBlobBackup,
+) {
+	t.Helper()
 
 	var out bytes.Buffer
 
@@ -72,14 +92,14 @@ func TestRestoreSkipErrorsSkipsFilesOfMissingBlob(t *testing.T) {
 
 		if backup.lost[path] {
 			assert.Containsf(t, out.String(), path,
-				"%s needs the deleted blob and must be reported", path)
+				"%s needs the lost blob and must be reported", path)
 			assert.NoFileExists(t, restored)
 
 			continue
 		}
 
 		got, err := afero.ReadFile(backup.fs, restored)
-		require.NoErrorf(t, err, "%s does not need the deleted blob", path)
+		require.NoErrorf(t, err, "%s does not need the lost blob", path)
 		assert.Equalf(t, content, got, "%s restored with wrong content", path)
 	}
 }
@@ -103,6 +123,30 @@ func TestRestoreMissingBlobAbortsWithoutSkipErrors(t *testing.T) {
 	require.ErrorIs(t, err, storage.ErrNotFound)
 }
 
+// TestRestoreChunkInNoBlobAbortsWithoutSkipErrors checks that a chunk held
+// by no blob still ends the restore, before any file is written, when
+// SkipErrors is not set.
+func TestRestoreChunkInNoBlobAbortsWithoutSkipErrors(t *testing.T) {
+	log.Initialize(log.Config{})
+	t.Parallel()
+
+	ctx := context.Background()
+	backup := backupThenDropOneBlobFromMetadata(ctx, t)
+
+	v := newReaderVaultik(ctx, backup.cfg, backup.storer, nil, backup.fs)
+
+	err := v.Restore(&vaultik.RestoreOptions{
+		SnapshotID: backup.snapshotID,
+		TargetDir:  backup.restoreDir,
+	})
+
+	require.ErrorContains(t, err, "chunk missing from blob map")
+
+	for path := range backup.files {
+		assert.NoFileExists(t, filepath.Join(backup.restoreDir, path))
+	}
+}
+
 // backupThenDeleteOneBlob backs up missingBlobFileCount single-chunk
 // files, then deletes from the store the blob holding the first of them.
 func backupThenDeleteOneBlob(
@@ -110,6 +154,54 @@ func backupThenDeleteOneBlob(
 ) *missingBlobBackup {
 	t.Helper()
 
+	backup, db, lostBlob := backupSingleChunkFiles(ctx, t)
+	require.NoError(t, db.Close())
+
+	require.NoError(t, backup.storer.Delete(ctx, fmt.Sprintf(
+		"blobs/%s/%s/%s", lostBlob[:2], lostBlob[2:4], lostBlob)))
+
+	return backup
+}
+
+// backupThenDropOneBlobFromMetadata backs up missingBlobFileCount
+// single-chunk files, then removes the blob holding the first of them from
+// the snapshot and exports the snapshot metadata again. The exported
+// metadata then lists chunks that no blob holds.
+func backupThenDropOneBlobFromMetadata(
+	ctx context.Context, t *testing.T,
+) *missingBlobBackup {
+	t.Helper()
+
+	backup, db, lostBlob := backupSingleChunkFiles(ctx, t)
+
+	// The uploads row refers to the blob row, so it would keep the export
+	// from deleting it.
+	for _, query := range []string{
+		"DELETE FROM snapshot_blobs WHERE blob_hash = ?",
+		"DELETE FROM uploads WHERE blob_hash = ?",
+	} {
+		_, err := db.Conn().ExecContext(ctx, query, lostBlob)
+		require.NoError(t, err)
+	}
+
+	sm := newFaultSnapshotManager(
+		backup.fs, backup.storer, backup.cfg, database.NewRepositories(db))
+	require.NoError(t,
+		sm.ExportSnapshotMetadata(ctx, db.Path(), backup.snapshotID))
+	require.NoError(t, db.Close())
+
+	return backup
+}
+
+// backupSingleChunkFiles backs up missingBlobFileCount single-chunk files.
+// It returns the backup, with lost naming the files stored in the same
+// blob as the first of them, the still open local index, and the hash of
+// that blob.
+func backupSingleChunkFiles(
+	ctx context.Context, t *testing.T,
+) (*missingBlobBackup, *database.DB, string) {
+	t.Helper()
+
 	fs := afero.NewOsFs()
 	tempDir := t.TempDir()
 	dataDir := filepath.Join(tempDir, "src")
@@ -143,16 +235,11 @@ func backupThenDeleteOneBlob(
 		blobOfFile[path] = blobHashOfSingleChunkFile(ctx, t, repos, path)
 	}
 
-	require.NoError(t, db.Close())
-
-	deleted := blobOfFile[filepath.Join(dataDir, "file-00.bin")]
-	require.NoError(t, storer.Delete(ctx, fmt.Sprintf(
-		"blobs/%s/%s/%s", deleted[:2], deleted[2:4], deleted)))
-
+	lostBlob := blobOfFile[filepath.Join(dataDir, "file-00.bin")]
 	lost := make(map[string]bool)
 
 	for path, hash := range blobOfFile {
-		if hash == deleted {
+		if hash == lostBlob {
 			lost[path] = true
 		}
 	}
@@ -168,7 +255,7 @@ func backupThenDeleteOneBlob(
 		restoreDir: filepath.Join(tempDir, "restored"),
 		files:      files,
 		lost:       lost,
-	}
+	}, db, lostBlob
 }
 
 // blobHashOfSingleChunkFile returns the hash of the blob holding the one

internal/vaultik/snapshot.go

@@ -209,6 +209,7 @@ type snapshotStats struct {
 	totalFiles         int
 	totalBytes         int64
 	totalChunks        int
+	newChunkSize       int64
 	totalBlobs         int
 	totalBytesSkipped  int64
 	totalFilesSkipped  int
@@ -341,6 +342,7 @@ func (v *Vaultik) scanAllDirectories(
 		stats.totalFiles += result.FilesScanned
 		stats.totalBytes += result.BytesScanned
 		stats.totalChunks += result.ChunksCreated
+		stats.newChunkSize += result.NewChunkSize
 		stats.totalBlobs += result.BlobsCreated
 		stats.totalFilesSkipped += result.FilesSkipped
 		stats.totalFilesFailed += result.FilesFailed
@@ -403,6 +405,7 @@ func (v *Vaultik) finalizeSnapshotMetadata(
 		BlobUncompressedSize: stats.blobUncompressedSize,
 		CompressionLevel:     v.Config.CompressionLevel,
 		UploadDurationMs:     stats.uploadDuration.Milliseconds(),
+		NewChunkSize:         stats.newChunkSize,
 	}
 
 	err = v.SnapshotManager.UpdateSnapshotStatsExtended(v.ctx, snapshotID, extStats)

internal/vaultik/snapshot_list_test.go

@@ -487,6 +487,7 @@ type listJSONRow struct {
 	RemoteKey      string `json:"remote_key"`
 	Timestamp      string `json:"timestamp"`
 	CompressedSize int64  `json:"compressed_size"`
+	NewChunkSize   int64  `json:"new_chunk_size"`
 	LocallyTracked bool   `json:"locally_tracked"`
 	RemotePresent  *bool  `json:"remote_present"`
 }