clawbot
49eed7a3e5
check / check (push) Waiting to run
The scanner added a changed file's bytes again for each new chunk and counted a file as unchanged for each chunk already stored. It now counts files and bytes once, in the scan phase, and counts its own uploads, so a --cron run, which has no progress reporter, records them. The blob count no longer adds earlier paths' blobs again. The snapshots row now stores the size of all files in total_size and the referenced blobs' sizes in blob_size, blob_uncompressed_size and compression_ratio, as docs/DATAMODEL.md says. Those sizes come from one query, and a failed query fails the snapshot. DATAMODEL.md now says chunk_count and blob_count count what the run added. Removed UpdateSnapshotStats and GetCountBySnapshot, which nothing calls any more. Model: opus-5-5
425 lines
12 KiB
Go
425 lines
12 KiB
Go
package database_test
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"math"
|
|
"testing"
|
|
"time"
|
|
|
|
"sneak.berlin/go/vaultik/internal/database"
|
|
"sneak.berlin/go/vaultik/internal/types"
|
|
)
|
|
|
|
const (
|
|
Mebibyte = 1024 * 1024
|
|
oneHundredMebibytes = 100 * Mebibyte
|
|
fortyMebibytes = 40 * Mebibyte
|
|
sixtyMebibytes = 60 * Mebibyte
|
|
twoHundredMebibytes = 200 * Mebibyte
|
|
compressionRatioPoint4 = 0.4
|
|
compressionRatioPoint3 = 0.3
|
|
)
|
|
|
|
func TestSnapshotRepository(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
db, cleanup := setupTestDB(t)
|
|
defer cleanup()
|
|
|
|
ctx := context.Background()
|
|
repo := database.NewSnapshotRepository(db)
|
|
|
|
// Test Create
|
|
snapshot := &database.Snapshot{
|
|
ID: "2024-01-01T12:00:00Z",
|
|
Hostname: testHostname,
|
|
VaultikVersion: testVersion,
|
|
StartedAt: time.Now().Truncate(time.Second),
|
|
CompletedAt: nil,
|
|
FileCount: 100,
|
|
ChunkCount: 500,
|
|
BlobCount: 10,
|
|
TotalSize: oneHundredMebibytes,
|
|
BlobSize: fortyMebibytes,
|
|
CompressionRatio: compressionRatioPoint4, // 40MB / 100MB
|
|
}
|
|
|
|
err := repo.Create(ctx, nil, snapshot)
|
|
if err != nil {
|
|
t.Fatalf("failed to create snapshot: %v", err)
|
|
}
|
|
|
|
// Test GetByID
|
|
retrieved, err := repo.GetByID(ctx, snapshot.ID.String())
|
|
if err != nil {
|
|
t.Fatalf("failed to get snapshot: %v", err)
|
|
}
|
|
|
|
if retrieved == nil {
|
|
t.Fatal("expected snapshot, got nil")
|
|
}
|
|
|
|
if retrieved.ID != snapshot.ID {
|
|
t.Errorf("ID mismatch: got %s, want %s", retrieved.ID, snapshot.ID)
|
|
}
|
|
|
|
if retrieved.Hostname != snapshot.Hostname {
|
|
t.Errorf("hostname mismatch: got %s, want %s",
|
|
retrieved.Hostname, snapshot.Hostname)
|
|
}
|
|
|
|
if retrieved.FileCount != snapshot.FileCount {
|
|
t.Errorf("file count mismatch: got %d, want %d",
|
|
retrieved.FileCount, snapshot.FileCount)
|
|
}
|
|
}
|
|
|
|
func TestSnapshotRepositoryUpdateCounts(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
db, cleanup := setupTestDB(t)
|
|
defer cleanup()
|
|
|
|
ctx := context.Background()
|
|
repo := database.NewSnapshotRepository(db)
|
|
|
|
snapshot := &database.Snapshot{
|
|
ID: "2024-01-02T12:00:00Z",
|
|
Hostname: testHostname,
|
|
VaultikVersion: testVersion,
|
|
StartedAt: time.Now().Truncate(time.Second),
|
|
CompletedAt: nil,
|
|
FileCount: 100,
|
|
ChunkCount: 500,
|
|
BlobCount: 10,
|
|
TotalSize: oneHundredMebibytes,
|
|
BlobSize: fortyMebibytes,
|
|
CompressionRatio: compressionRatioPoint4,
|
|
}
|
|
|
|
err := repo.Create(ctx, nil, snapshot)
|
|
if err != nil {
|
|
t.Fatalf("failed to create snapshot: %v", err)
|
|
}
|
|
|
|
// Test UpdateCounts
|
|
err = repo.UpdateCounts(ctx, nil, snapshot.ID.String(),
|
|
200, 1000, 20, twoHundredMebibytes, sixtyMebibytes)
|
|
if err != nil {
|
|
t.Fatalf("failed to update counts: %v", err)
|
|
}
|
|
|
|
retrieved, err := repo.GetByID(ctx, snapshot.ID.String())
|
|
if err != nil {
|
|
t.Fatalf("failed to get updated snapshot: %v", err)
|
|
}
|
|
|
|
if retrieved.FileCount != 200 {
|
|
t.Errorf("file count not updated: got %d, want %d", retrieved.FileCount, 200)
|
|
}
|
|
|
|
if retrieved.ChunkCount != 1000 {
|
|
t.Errorf("chunk count not updated: got %d, want %d",
|
|
retrieved.ChunkCount, 1000)
|
|
}
|
|
|
|
if retrieved.BlobCount != 20 {
|
|
t.Errorf("blob count not updated: got %d, want %d", retrieved.BlobCount, 20)
|
|
}
|
|
|
|
if retrieved.TotalSize != twoHundredMebibytes {
|
|
t.Errorf("total size not updated: got %d, want %d",
|
|
retrieved.TotalSize, twoHundredMebibytes)
|
|
}
|
|
|
|
if retrieved.BlobSize != sixtyMebibytes {
|
|
t.Errorf("blob size not updated: got %d, want %d",
|
|
retrieved.BlobSize, sixtyMebibytes)
|
|
}
|
|
|
|
expectedRatio := compressionRatioPoint3 // 0.3
|
|
if math.Abs(retrieved.CompressionRatio-expectedRatio) > 0.001 {
|
|
t.Errorf("compression ratio not updated: got %f, want %f",
|
|
retrieved.CompressionRatio, expectedRatio)
|
|
}
|
|
}
|
|
|
|
// GetSnapshotBlobSizes totals the blobs the snapshot references, and only
|
|
// those.
|
|
func TestSnapshotRepositoryGetSnapshotBlobSizes(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
db, cleanup := setupTestDB(t)
|
|
defer cleanup()
|
|
|
|
ctx := context.Background()
|
|
repos := database.NewRepositories(db)
|
|
|
|
snapshot := &database.Snapshot{
|
|
ID: "2024-01-03T12:00:00Z",
|
|
Hostname: testHostname,
|
|
VaultikVersion: testVersion,
|
|
StartedAt: time.Now().Truncate(time.Second),
|
|
}
|
|
|
|
err := repos.Snapshots.Create(ctx, nil, snapshot)
|
|
if err != nil {
|
|
t.Fatalf("failed to create snapshot: %v", err)
|
|
}
|
|
|
|
blobs := []*database.Blob{
|
|
{Hash: "referenced-1", CompressedSize: 10, UncompressedSize: 100},
|
|
{Hash: "referenced-2", CompressedSize: 20, UncompressedSize: 200},
|
|
{Hash: "unreferenced", CompressedSize: 40, UncompressedSize: 400},
|
|
}
|
|
|
|
for _, blob := range blobs {
|
|
blob.ID = types.NewBlobID()
|
|
blob.CreatedTS = time.Now().Truncate(time.Second)
|
|
|
|
err = repos.Blobs.Create(ctx, nil, blob)
|
|
if err != nil {
|
|
t.Fatalf("failed to create blob %s: %v", blob.Hash, err)
|
|
}
|
|
}
|
|
|
|
for _, blob := range blobs[:2] {
|
|
err = repos.Snapshots.AddBlob(ctx, nil, snapshot.ID.String(),
|
|
blob.ID, blob.Hash)
|
|
if err != nil {
|
|
t.Fatalf("failed to add blob %s to snapshot: %v", blob.Hash, err)
|
|
}
|
|
}
|
|
|
|
compressed, uncompressed, err := repos.Snapshots.GetSnapshotBlobSizes(
|
|
ctx, snapshot.ID.String())
|
|
if err != nil {
|
|
t.Fatalf("failed to get snapshot blob sizes: %v", err)
|
|
}
|
|
|
|
if compressed != 30 || uncompressed != 300 {
|
|
t.Errorf("blob sizes: got %d and %d, want 30 and 300",
|
|
compressed, uncompressed)
|
|
}
|
|
}
|
|
|
|
func TestSnapshotRepositoryListRecent(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
db, cleanup := setupTestDB(t)
|
|
defer cleanup()
|
|
|
|
ctx := context.Background()
|
|
repo := database.NewSnapshotRepository(db)
|
|
|
|
// Add snapshots
|
|
for i := 1; i <= 5; i++ {
|
|
s := &database.Snapshot{
|
|
ID: types.SnapshotID(fmt.Sprintf("2024-01-0%dT12:00:00Z", i)),
|
|
Hostname: testHostname,
|
|
VaultikVersion: testVersion,
|
|
StartedAt: time.Now().Add(time.Duration(i) * time.Hour).Truncate(time.Second),
|
|
CompletedAt: nil,
|
|
FileCount: int64(100 * i),
|
|
ChunkCount: int64(500 * i),
|
|
BlobCount: int64(10 * i),
|
|
}
|
|
|
|
err := repo.Create(ctx, nil, s)
|
|
if err != nil {
|
|
t.Fatalf("failed to create snapshot %d: %v", i, err)
|
|
}
|
|
}
|
|
|
|
// Test listing with limit
|
|
recent, err := repo.ListRecent(ctx, 3)
|
|
if err != nil {
|
|
t.Fatalf("failed to list recent snapshots: %v", err)
|
|
}
|
|
|
|
if len(recent) != 3 {
|
|
t.Errorf("expected 3 recent snapshots, got %d", len(recent))
|
|
}
|
|
|
|
// Verify order (most recent first)
|
|
for i := range len(recent) - 1 {
|
|
if recent[i].StartedAt.Before(recent[i+1].StartedAt) {
|
|
t.Error("snapshots not in descending order")
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestSnapshotTimestampsDecodeAsUTC pins the zone every snapshot reader
|
|
// returns. started_at and completed_at are stored as bare Unix seconds,
|
|
// so the zone is a decode choice, and callers (notably `snapshot list`)
|
|
// render these timestamps through zone-less format strings in the same
|
|
// column as timestamps read from remote manifests, which are always
|
|
// UTC. If one reader decodes in the host's local zone, that column
|
|
// silently shows two different wall clocks for the same instant.
|
|
//
|
|
// The assertions compare *time.Location pointers, so this fails on a
|
|
// UTC host too: time.Unix returns time.Local, which is never the same
|
|
// Location value as time.UTC no matter what the host's offset is.
|
|
func TestSnapshotTimestampsDecodeAsUTC(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
db, cleanup := setupTestDB(t)
|
|
defer cleanup()
|
|
|
|
ctx := context.Background()
|
|
repo := database.NewSnapshotRepository(db)
|
|
|
|
startedAt := time.Date(2026, 3, 1, 10, 0, 0, 0, time.UTC)
|
|
completedAt := startedAt.Add(time.Minute)
|
|
|
|
completed := &database.Snapshot{
|
|
ID: types.SnapshotID("testhost_home_2026-03-01T10:00:00Z"),
|
|
Hostname: testHostname,
|
|
VaultikVersion: testVersion,
|
|
StartedAt: startedAt,
|
|
CompletedAt: &completedAt,
|
|
}
|
|
|
|
err := repo.Create(ctx, nil, completed)
|
|
if err != nil {
|
|
t.Fatalf("failed to create completed snapshot: %v", err)
|
|
}
|
|
|
|
// An incomplete row as well, so the scanner shared by the two
|
|
// GetIncomplete* readers is covered with a nil completed_at too.
|
|
incomplete := &database.Snapshot{
|
|
ID: types.SnapshotID("testhost_home_2026-03-02T10:00:00Z"),
|
|
Hostname: testHostname,
|
|
VaultikVersion: testVersion,
|
|
StartedAt: startedAt.Add(time.Hour),
|
|
CompletedAt: nil,
|
|
}
|
|
|
|
err = repo.Create(ctx, nil, incomplete)
|
|
if err != nil {
|
|
t.Fatalf("failed to create incomplete snapshot: %v", err)
|
|
}
|
|
|
|
byID, err := repo.GetByID(ctx, completed.ID.String())
|
|
if err != nil {
|
|
t.Fatalf("failed to get snapshot by id: %v", err)
|
|
}
|
|
|
|
recent, err := repo.ListRecent(ctx, 10)
|
|
if err != nil {
|
|
t.Fatalf("failed to list recent snapshots: %v", err)
|
|
}
|
|
|
|
incompletes, err := repo.GetIncompleteSnapshots(ctx)
|
|
if err != nil {
|
|
t.Fatalf("failed to list incomplete snapshots: %v", err)
|
|
}
|
|
|
|
byHost, err := repo.GetIncompleteByHostname(ctx, testHostname)
|
|
if err != nil {
|
|
t.Fatalf("failed to list incomplete snapshots by hostname: %v", err)
|
|
}
|
|
|
|
read := make([]*database.Snapshot, 0,
|
|
1+len(recent)+len(incompletes)+len(byHost))
|
|
read = append(read, byID)
|
|
read = append(read, recent...)
|
|
read = append(read, incompletes...)
|
|
read = append(read, byHost...)
|
|
|
|
if len(read) < 5 {
|
|
t.Fatalf("expected every reader to return rows, got %d", len(read))
|
|
}
|
|
|
|
assertTimestampsAreUTC(t, read)
|
|
|
|
// And the wall clock is the UTC one, not the host's rendering of it.
|
|
rendered := byID.StartedAt.Format("2006-01-02 15:04:05")
|
|
if rendered != "2026-03-01 10:00:00" {
|
|
t.Errorf("started_at rendered as %q, want the UTC wall clock", rendered)
|
|
}
|
|
}
|
|
|
|
// assertTimestampsAreUTC fails for any snapshot whose timestamps did not
|
|
// decode in UTC. It compares *time.Location pointers rather than
|
|
// offsets, so it is equally strict on a host whose local zone happens to
|
|
// be UTC: time.Unix returns time.Local, which is never the same Location
|
|
// value as time.UTC.
|
|
func assertTimestampsAreUTC(t *testing.T, snapshots []*database.Snapshot) {
|
|
t.Helper()
|
|
|
|
for _, snapshot := range snapshots {
|
|
if snapshot.StartedAt.Location() != time.UTC {
|
|
t.Errorf("snapshot %s: started_at decoded in %s, want UTC",
|
|
snapshot.ID, snapshot.StartedAt.Location())
|
|
}
|
|
|
|
if snapshot.CompletedAt != nil &&
|
|
snapshot.CompletedAt.Location() != time.UTC {
|
|
t.Errorf("snapshot %s: completed_at decoded in %s, want UTC",
|
|
snapshot.ID, snapshot.CompletedAt.Location())
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestSnapshotRepositoryNotFound(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
db, cleanup := setupTestDB(t)
|
|
defer cleanup()
|
|
|
|
ctx := context.Background()
|
|
repo := database.NewSnapshotRepository(db)
|
|
|
|
// Test GetByID with non-existent ID
|
|
snapshot, err := repo.GetByID(ctx, "nonexistent")
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
|
|
if snapshot != nil {
|
|
t.Error("expected nil for non-existent snapshot")
|
|
}
|
|
|
|
// Test UpdateCounts on non-existent snapshot
|
|
err = repo.UpdateCounts(ctx, nil, "nonexistent",
|
|
100, 200, 10, oneHundredMebibytes, fortyMebibytes)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
// No error expected, but no rows should be affected
|
|
}
|
|
|
|
func TestSnapshotRepositoryDuplicate(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
db, cleanup := setupTestDB(t)
|
|
defer cleanup()
|
|
|
|
ctx := context.Background()
|
|
repo := database.NewSnapshotRepository(db)
|
|
|
|
snapshot := &database.Snapshot{
|
|
ID: "2024-01-01T12:00:00Z",
|
|
Hostname: testHostname,
|
|
VaultikVersion: testVersion,
|
|
StartedAt: time.Now().Truncate(time.Second),
|
|
CompletedAt: nil,
|
|
FileCount: 100,
|
|
ChunkCount: 500,
|
|
BlobCount: 10,
|
|
}
|
|
|
|
err := repo.Create(ctx, nil, snapshot)
|
|
if err != nil {
|
|
t.Fatalf("failed to create snapshot: %v", err)
|
|
}
|
|
|
|
// Try to create duplicate - should fail due to primary key constraint
|
|
err = repo.Create(ctx, nil, snapshot)
|
|
if err == nil {
|
|
t.Error("expected error for duplicate snapshot")
|
|
}
|
|
}
|