clawbot
bffd4578fc
check / check (push) Waiting to run
snapshot list worked out NEW CHUNK SIZE from the snapshot's file rows. A file's row is shared by every snapshot that lists it and rewritten when the file changes, so a snapshot whose change was a modified file showed 0 B, and older snapshots' figures moved after later backups. The scanner now totals the size of the chunks it stores that were not stored before (the chunks chunk_count counts), the backup writes that total to a new snapshots column, new_chunk_size, and snapshot list reads it. Judgement call: the figure is fixed when the backup runs, so removing an earlier snapshot no longer raises a later snapshot's figure. Model: opus-5-5
134 lines
3.7 KiB
Go
134 lines
3.7 KiB
Go
package vaultik_test
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"maps"
|
|
"path/filepath"
|
|
"testing"
|
|
|
|
"github.com/spf13/afero"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
"sneak.berlin/go/vaultik/internal/chunker"
|
|
"sneak.berlin/go/vaultik/internal/database"
|
|
"sneak.berlin/go/vaultik/internal/log"
|
|
"sneak.berlin/go/vaultik/internal/storage"
|
|
"sneak.berlin/go/vaultik/internal/vaultik"
|
|
)
|
|
|
|
// TestNewChunkSizeOfChangedFileStaysFixed covers
|
|
// https://git.eeqj.de/sneak/vaultik/issues/311. A backup whose only change
|
|
// is a modified file lists the size of the chunks that change added as its
|
|
// NEW CHUNK SIZE, and a later backup that changes the file again leaves
|
|
// every earlier snapshot's figure as it was. The file's row in the local
|
|
// index is shared by all three snapshots and rewritten by each backup.
|
|
func TestNewChunkSizeOfChangedFileStaysFixed(t *testing.T) {
|
|
log.Initialize(log.Config{})
|
|
t.Parallel()
|
|
|
|
fs := afero.NewOsFs()
|
|
tempDir := t.TempDir()
|
|
dataDir := filepath.Join(tempDir, "src")
|
|
dbPath := filepath.Join(tempDir, "index.sqlite")
|
|
changedPath := filepath.Join(dataDir, "changed.bin")
|
|
|
|
ctx := context.Background()
|
|
files := writeFaultSourceTree(t, fs, dataDir)
|
|
cfg := changedFileConfig(dataDir, dbPath)
|
|
|
|
appendRandomBytes(t, fs, files, changedPath, appendedFileSize)
|
|
|
|
store, err := storage.NewFileStorer(filepath.Join(tempDir, "remote"))
|
|
require.NoError(t, err)
|
|
|
|
db, err := database.New(ctx, dbPath)
|
|
require.NoError(t, err)
|
|
t.Cleanup(func() { _ = db.Close() })
|
|
|
|
repos := database.NewRepositories(db)
|
|
v := newBackupVaultik(ctx, cfg, store, repos, db, fs)
|
|
|
|
require.NoError(t, backUp(v, "first"))
|
|
|
|
firstFiles := maps.Clone(files)
|
|
firstID := localSnapshotID(ctx, t, repos, "first")
|
|
|
|
appendRandomBytes(t, fs, files, changedPath, faultChunkSize)
|
|
require.NoError(t, backUp(v, "second"))
|
|
|
|
secondID := localSnapshotID(ctx, t, repos, "second")
|
|
secondAdded := sizeOfNewChunks(t, files, firstFiles)
|
|
require.Positive(t, secondAdded)
|
|
assert.Equal(t, secondAdded, listedNewChunkSizes(t, v)[secondID],
|
|
"the second snapshot must count the chunks the change added")
|
|
|
|
appendRandomBytes(t, fs, files, changedPath, faultChunkSize)
|
|
require.NoError(t, backUp(v, "third"))
|
|
|
|
sizes := listedNewChunkSizes(t, v)
|
|
assert.Equal(t, sizeOfNewChunks(t, firstFiles, nil), sizes[firstID],
|
|
"a later backup must not change the first snapshot's figure")
|
|
assert.Equal(t, secondAdded, sizes[secondID],
|
|
"a later backup must not change the second snapshot's figure")
|
|
}
|
|
|
|
// sizeOfNewChunks returns the total size of the distinct chunks of the
|
|
// files in after that no file in before has, cut the way the backup cuts
|
|
// them.
|
|
func sizeOfNewChunks(t *testing.T, after, before map[string][]byte) int64 {
|
|
t.Helper()
|
|
|
|
known := make(map[string]bool)
|
|
for _, chunk := range chunksOf(t, before) {
|
|
known[chunk.Hash] = true
|
|
}
|
|
|
|
var total int64
|
|
|
|
for _, chunk := range chunksOf(t, after) {
|
|
if !known[chunk.Hash] {
|
|
known[chunk.Hash] = true
|
|
total += chunk.Size
|
|
}
|
|
}
|
|
|
|
return total
|
|
}
|
|
|
|
// chunksOf returns the chunks of every file in files.
|
|
func chunksOf(t *testing.T, files map[string][]byte) []chunker.Chunk {
|
|
t.Helper()
|
|
|
|
c := chunker.NewChunker(faultChunkSize)
|
|
|
|
var all []chunker.Chunk
|
|
|
|
for _, content := range files {
|
|
chunks, err := c.ChunkReader(bytes.NewReader(content))
|
|
require.NoError(t, err)
|
|
|
|
all = append(all, chunks...)
|
|
}
|
|
|
|
return all
|
|
}
|
|
|
|
// listedNewChunkSizes runs `snapshot list --json` and returns each listed
|
|
// snapshot's new_chunk_size by snapshot ID.
|
|
func listedNewChunkSizes(t *testing.T, v *vaultik.Vaultik) map[string]int64 {
|
|
t.Helper()
|
|
|
|
var out bytes.Buffer
|
|
|
|
v.Stdout = &out
|
|
require.NoError(t, v.ListSnapshots(true))
|
|
|
|
sizes := make(map[string]int64)
|
|
for _, row := range decodeListJSON(t, out.String()) {
|
|
sizes[row.ID] = row.NewChunkSize
|
|
}
|
|
|
|
return sizes
|
|
}
|