sneak/vaultik
1
0
Fork 0
Code Issues 19 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Update golangci-lint to v2.12.2 with canonical config (#62)
All checks were successful
check / check (push) Successful in 5s
Details

Browse Source 
Updates golangci-lint to v2.12.2 everywhere it is pinned and installs the canonical `.golangci.yml`, then remediates every finding the new linter/config surfaces so `make check` is green.

## Version bump

- `Dockerfile` lint stage: `golangci/golangci-lint:v2.11.3-alpine` -> `v2.12.2-alpine` (digest-pinned, date comment updated)
- `Makefile` `deps` target: `go install` moved from the old v1 module path at `@latest` to the pinned `github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.12.2`
- `.golangci.yml` replaced with the canonical config (v2 schema; settings under `linters.settings` so the thresholds actually apply; `default: all` with the standard six disables)
- `script/bootstrap` installs golangci-lint via the system package manager and carries no version pin, so it is unchanged
- CI (`.gitea/workflows/check.yml`) only runs `script/cibuild`, so it needed no change

## Lint remediation

The canonical config surfaced ~3,300 findings across 56k lines. All are fixed, behavior-preserving; incorporates and supersedes the per-package mechanical passes already merged to `main` (refs #61). Highlights:

- `err113`: dynamic errors replaced with package sentinels + `%w` wrapping; comparisons via `errors.Is`
- `goprintffuncname`: printf-style helpers renamed with an `f` suffix (`ui.Writer` message methods, `cli.ReportErrorf`, `database.Fatalf`) and all call sites updated
- `revive` stutter renames: `blob.Handler`, `blob.WithReader`, `blob.ChunkPosition`, `storage.URL`, `storage.Info`; missing doc comments added
- `contextcheck`/`noctx`: `context.Context` threaded through `blob.Packer` and the scanner call sites; context-aware `exec`/`sql` variants
- `funlen`/`cyclop`/`gocognit`/`dupl`: oversized and duplicated functions split into focused helpers (production and test code)
- tests: `t.Parallel()` added where safe (global logger init kept in the serial phase for `-race`), `t.TempDir()`/`t.Helper()` adopted, several suites converted to external test packages
- `gosec`: bounded integer conversions, `ReadHeaderTimeout` on the test HTTP server; remaining warnings suppressed per-site with justifications
- remaining `nolint` directives are rare, targeted, and each carries a reason (e.g. `nilnil` not-found contract in the repository layer, fx module globals, on-disk snake_case struct tags)
- removed the deprecated `log.LogOptions` alias (callers migrated to `log.Options`)

`make check` (tests with `-race`, lint, fmt-check) passes.

Co-authored-by: sneak <sneak@sneak.berlin>
Reviewed-on: #62
Co-authored-by: clawbot <clawbot@noreply.example.org>
Co-committed-by: clawbot <clawbot@noreply.example.org>
This commit was merged in pull request #62.
This commit is contained in:
clawbot
2026-08-07 23:22:48 +02:00
committed by Jeffrey Paul
parent b87b72d4b9
commit cc58583130
126 changed files with 8184 additions and 5470 deletions
Download Patch File Download Diff File Expand all files Collapse all files

521
internal/snapshot/backup_test.go
View File

@@ -1,4 +1,4 @@
package snapshot
package snapshot_test
import (
"context"
@@ -19,6 +19,12 @@ import (
"sneak.berlin/go/vaultik/internal/types"
)
// errBlobNotFound is returned by the mock S3 client for unknown hashes.
var errBlobNotFound = errors.New("blob not found")
// testFile1Name is the shared fixture filename used across backup tests.
const testFile1Name = "file1.txt"
// MockS3Client is a mock implementation of S3 operations for testing
type MockS3Client struct {
storage map[string][]byte
@@ -30,39 +36,149 @@ func NewMockS3Client() *MockS3Client {
}
}
func (m *MockS3Client) PutBlob(ctx context.Context, hash string, data []byte) error {
func (m *MockS3Client) PutBlob(_ context.Context, hash string, data []byte) error {
m.storage[hash] = data
return nil
}
func (m *MockS3Client) GetBlob(ctx context.Context, hash string) ([]byte, error) {
func (m *MockS3Client) GetBlob(_ context.Context, hash string) ([]byte, error) {
data, ok := m.storage[hash]
if !ok {
return nil, fmt.Errorf("blob not found: %s", hash)
return nil, fmt.Errorf("%w: %s", errBlobNotFound, hash)
}
return data, nil
}
func (m *MockS3Client) BlobExists(ctx context.Context, hash string) (bool, error) {
func (m *MockS3Client) BlobExists(_ context.Context, hash string) (bool, error) {
_, ok := m.storage[hash]
return ok, nil
}
func (m *MockS3Client) CreateBucket(ctx context.Context, bucket string) error {
func (m *MockS3Client) CreateBucket(_ context.Context, _ string) error {
return nil
}
// verifyBackupFiles checks the file records created by a backup against
// the fixture filesystem.
func verifyBackupFiles(
ctx context.Context,
t *testing.T,
repos *database.Repositories,
testFS fstest.MapFS,
) {
t.Helper()
files, err := repos.Files.ListByPrefix(ctx, "")
if err != nil {
t.Fatalf("Failed to list files: %v", err)
}
expectedFiles := map[string]bool{
testFile1Name: true,
"dir1/file2.txt": true,
"dir1/subdir/file3.txt": true,
"largefile.bin": true,
}
if len(files) != len(expectedFiles) {
t.Errorf("Expected %d files, got %d", len(expectedFiles), len(files))
}
for _, file := range files {
if !expectedFiles[file.Path.String()] {
t.Errorf("Unexpected file in database: %s", file.Path)
}
delete(expectedFiles, file.Path.String())
// Verify file metadata
fsFile := testFS[file.Path.String()]
if fsFile == nil {
t.Errorf("File %s not found in test filesystem", file.Path)
continue
}
if file.Size != int64(len(fsFile.Data)) {
t.Errorf("File %s: expected size %d, got %d",
file.Path, len(fsFile.Data), file.Size)
}
if file.Mode != uint32(fsFile.Mode) {
t.Errorf("File %s: expected mode %o, got %o",
file.Path, fsFile.Mode, file.Mode)
}
}
if len(expectedFiles) > 0 {
t.Errorf("Files not found in database: %v", expectedFiles)
}
}
// verifyBackupChunksAndBlobs checks that chunking produced the expected
// records and every referenced blob exists in the mock S3 store.
func verifyBackupChunksAndBlobs(
ctx context.Context,
t *testing.T,
repos *database.Repositories,
s3Client *MockS3Client,
snapshotID string,
) {
t.Helper()
chunks, err := repos.Chunks.List(ctx)
if err != nil {
t.Fatalf("Failed to list chunks: %v", err)
}
if len(chunks) == 0 {
t.Error("No chunks found in database")
}
// The large file should create 10 chunks (10MB / 1MB chunk size)
// Plus the small files
minExpectedChunks := 10 + 3
if len(chunks) < minExpectedChunks {
t.Errorf("Expected at least %d chunks, got %d", minExpectedChunks, len(chunks))
}
// Verify at least one blob was created and uploaded
// We can't list blobs directly, but we can check via snapshot blobs
blobHashes, err := repos.Snapshots.GetBlobHashes(ctx, snapshotID)
if err != nil {
t.Fatalf("Failed to get blob hashes: %v", err)
}
if len(blobHashes) == 0 {
t.Error("Expected at least one blob to be created")
}
for _, blobHash := range blobHashes {
// Check blob exists in mock S3
exists, err := s3Client.BlobExists(ctx, blobHash)
if err != nil {
t.Errorf("Failed to check blob %s: %v", blobHash, err)
}
if !exists {
t.Errorf("Blob %s not found in S3", blobHash)
}
}
}
func TestBackupWithInMemoryFS(t *testing.T) {
t.Parallel()
// Create a temporary directory for the database
tempDir := t.TempDir()
dbPath := filepath.Join(tempDir, "test.db")
// Create test filesystem
testFS := fstest.MapFS{
"file1.txt": &fstest.MapFile{
testFile1Name: &fstest.MapFile{
Data: []byte("Hello, World!"),
Mode: 0644,
ModTime: time.Now(),
@@ -129,100 +245,21 @@ func TestBackupWithInMemoryFS(t *testing.T) {
t.Error("Expected snapshot to have files")
}
// Verify files in database
files, err := repos.Files.ListByPrefix(ctx, "")
if err != nil {
t.Fatalf("Failed to list files: %v", err)
}
expectedFiles := map[string]bool{
"file1.txt": true,
"dir1/file2.txt": true,
"dir1/subdir/file3.txt": true,
"largefile.bin": true,
}
if len(files) != len(expectedFiles) {
t.Errorf("Expected %d files, got %d", len(expectedFiles), len(files))
}
for _, file := range files {
if !expectedFiles[file.Path.String()] {
t.Errorf("Unexpected file in database: %s", file.Path)
}
delete(expectedFiles, file.Path.String())
// Verify file metadata
fsFile := testFS[file.Path.String()]
if fsFile == nil {
t.Errorf("File %s not found in test filesystem", file.Path)
continue
}
if file.Size != int64(len(fsFile.Data)) {
t.Errorf("File %s: expected size %d, got %d", file.Path, len(fsFile.Data), file.Size)
}
if file.Mode != uint32(fsFile.Mode) {
t.Errorf("File %s: expected mode %o, got %o", file.Path, fsFile.Mode, file.Mode)
}
}
if len(expectedFiles) > 0 {
t.Errorf("Files not found in database: %v", expectedFiles)
}
// Verify chunks
chunks, err := repos.Chunks.List(ctx)
if err != nil {
t.Fatalf("Failed to list chunks: %v", err)
}
if len(chunks) == 0 {
t.Error("No chunks found in database")
}
// The large file should create 10 chunks (10MB / 1MB chunk size)
// Plus the small files
minExpectedChunks := 10 + 3
if len(chunks) < minExpectedChunks {
t.Errorf("Expected at least %d chunks, got %d", minExpectedChunks, len(chunks))
}
// Verify at least one blob was created and uploaded
// We can't list blobs directly, but we can check via snapshot blobs
blobHashes, err := repos.Snapshots.GetBlobHashes(ctx, snapshotID)
if err != nil {
t.Fatalf("Failed to get blob hashes: %v", err)
}
if len(blobHashes) == 0 {
t.Error("Expected at least one blob to be created")
}
for _, blobHash := range blobHashes {
// Check blob exists in mock S3
exists, err := s3Client.BlobExists(ctx, blobHash)
if err != nil {
t.Errorf("Failed to check blob %s: %v", blobHash, err)
}
if !exists {
t.Errorf("Blob %s not found in S3", blobHash)
}
}
// Verify files, chunks, and blob records
verifyBackupFiles(ctx, t, repos, testFS)
verifyBackupChunksAndBlobs(ctx, t, repos, s3Client, snapshotID)
}
func TestBackupDeduplication(t *testing.T) {
t.Parallel()
// Create a temporary directory for the database
tempDir := t.TempDir()
dbPath := filepath.Join(tempDir, "test.db")
// Create test filesystem with duplicate content
testFS := fstest.MapFS{
"file1.txt": &fstest.MapFile{
testFile1Name: &fstest.MapFile{
Data: []byte("Duplicate content"),
Mode: 0644,
ModTime: time.Now(),
@@ -290,7 +327,8 @@ func TestBackupDeduplication(t *testing.T) {
// The duplicate content chunk should be referenced by 2 files
if chunk.Size == int64(len("Duplicate content")) && len(files) != 2 {
t.Errorf("Expected duplicate chunk to be referenced by 2 files, got %d", len(files))
t.Errorf("Expected duplicate chunk to be referenced by 2 files, got %d",
len(files))
}
}
}
@@ -304,8 +342,19 @@ type BackupEngine struct {
}
}
// backupCounters accumulates statistics across a test backup run.
type backupCounters struct {
fileCount int64
chunkCount int64
blobCount int64
totalSize int64
blobSize int64
}
// Backup performs a backup of the given filesystem
func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (string, error) {
func (b *BackupEngine) Backup(
ctx context.Context, fsys fs.FS, root string,
) (string, error) {
// Create a new snapshot
hostname, _ := os.Hostname()
snapshotID := time.Now().Format(time.RFC3339)
@@ -325,8 +374,7 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
return "", err
}
// Track counters
var fileCount, chunkCount, blobCount, totalSize, blobSize int64
counters := &backupCounters{}
// Track which chunks we've seen to handle deduplication
processedChunks := make(map[string]bool)
@@ -354,122 +402,170 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
return nil
}
// Create file record in a short transaction
file := &database.File{
Path: types.FilePath(path),
Size: info.Size(),
Mode: uint32(info.Mode()),
MTime: info.ModTime(),
UID: 1000, // Default UID for test
GID: 1000, // Default GID for test
}
err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
return b.repos.Files.Create(ctx, tx, file)
})
if err != nil {
return err
}
fileCount++
totalSize += info.Size()
// Read and process file in chunks
f, err := fsys.Open(path)
if err != nil {
return err
}
defer func() {
err := f.Close()
if err != nil {
// Log but don't fail since we're already in an error path potentially
fmt.Fprintf(os.Stderr, "Failed to close file: %v\n", err)
}
}()
// Process file in chunks
chunkIndex := 0
buffer := make([]byte, defaultChunkSize)
for {
n, err := f.Read(buffer)
if err != nil && !errors.Is(err, io.EOF) {
return err
}
if n == 0 {
break
}
chunkData := buffer[:n]
chunkHash := calculateHash(chunkData)
// Check if chunk already exists (outside of transaction)
existingChunk, _ := b.repos.Chunks.GetByHash(ctx, chunkHash)
if existingChunk == nil {
// Create new chunk in a short transaction
err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
chunk := &database.Chunk{
ChunkHash: types.ChunkHash(chunkHash),
Size: int64(n),
}
return b.repos.Chunks.Create(ctx, tx, chunk)
})
if err != nil {
return err
}
processedChunks[chunkHash] = true
}
// Create file-chunk mapping in a short transaction
err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
fileChunk := &database.FileChunk{
FileID: file.ID,
Idx: chunkIndex,
ChunkHash: types.ChunkHash(chunkHash),
}
return b.repos.FileChunks.Create(ctx, tx, fileChunk)
})
if err != nil {
return err
}
// Create chunk-file mapping in a short transaction
err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
chunkFile := &database.ChunkFile{
ChunkHash: types.ChunkHash(chunkHash),
FileID: file.ID,
FileOffset: int64(chunkIndex * defaultChunkSize),
Length: int64(n),
}
return b.repos.ChunkFiles.Create(ctx, tx, chunkFile)
})
if err != nil {
return err
}
chunkIndex++
}
return nil
return b.backupOneFile(ctx, fsys, path, info, processedChunks, counters)
})
if err != nil {
return "", err
}
// After all files are processed, create blobs for new chunks
err = b.createBlobsForChunks(ctx, snapshotID, processedChunks, counters)
if err != nil {
return "", err
}
// Update snapshot with final counts
err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
return b.repos.Snapshots.UpdateCounts(ctx, tx, snapshotID,
counters.fileCount, counters.chunkCount, counters.blobCount,
counters.totalSize, counters.blobSize)
})
if err != nil {
return "", err
}
return snapshotID, nil
}
// backupOneFile records a single regular file and its chunks.
func (b *BackupEngine) backupOneFile(
ctx context.Context,
fsys fs.FS,
path string,
info fs.FileInfo,
processedChunks map[string]bool,
counters *backupCounters,
) error {
// Create file record in a short transaction
file := &database.File{
Path: types.FilePath(path),
Size: info.Size(),
Mode: uint32(info.Mode()),
MTime: info.ModTime(),
UID: 1000, // Default UID for test
GID: 1000, // Default GID for test
}
err := b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
return b.repos.Files.Create(ctx, tx, file)
})
if err != nil {
return err
}
counters.fileCount++
counters.totalSize += info.Size()
// Read and process file in chunks
f, err := fsys.Open(path)
if err != nil {
return err
}
defer func() {
err := f.Close()
if err != nil {
// Log but don't fail since we're already in an error path potentially
fmt.Fprintf(os.Stderr, "Failed to close file: %v\n", err)
}
}()
// Process file in chunks
chunkIndex := 0
buffer := make([]byte, defaultChunkSize)
for {
n, err := f.Read(buffer)
if err != nil && !errors.Is(err, io.EOF) {
return err
}
if n == 0 {
break
}
err = b.recordChunk(ctx, file, chunkIndex, buffer[:n], processedChunks)
if err != nil {
return err
}
chunkIndex++
}
return nil
}
// recordChunk creates the chunk record (if new) and its file associations.
func (b *BackupEngine) recordChunk(
ctx context.Context,
file *database.File,
chunkIndex int,
chunkData []byte,
processedChunks map[string]bool,
) error {
chunkHash := calculateHash(chunkData)
// Check if chunk already exists (outside of transaction)
existingChunk, _ := b.repos.Chunks.GetByHash(ctx, chunkHash)
if existingChunk == nil {
// Create new chunk in a short transaction
err := b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
chunk := &database.Chunk{
ChunkHash: types.ChunkHash(chunkHash),
Size: int64(len(chunkData)),
}
return b.repos.Chunks.Create(ctx, tx, chunk)
})
if err != nil {
return err
}
processedChunks[chunkHash] = true
}
// Create file-chunk mapping in a short transaction
err := b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
fileChunk := &database.FileChunk{
FileID: file.ID,
Idx: chunkIndex,
ChunkHash: types.ChunkHash(chunkHash),
}
return b.repos.FileChunks.Create(ctx, tx, fileChunk)
})
if err != nil {
return err
}
// Create chunk-file mapping in a short transaction
return b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
chunkFile := &database.ChunkFile{
ChunkHash: types.ChunkHash(chunkHash),
FileID: file.ID,
FileOffset: int64(chunkIndex * defaultChunkSize),
Length: int64(len(chunkData)),
}
return b.repos.ChunkFiles.Create(ctx, tx, chunkFile)
})
}
// createBlobsForChunks uploads one blob per new chunk and records the blob
// metadata and snapshot association.
func (b *BackupEngine) createBlobsForChunks(
ctx context.Context,
snapshotID string,
processedChunks map[string]bool,
counters *backupCounters,
) error {
for chunkHash := range processedChunks {
// Get chunk data (outside of transaction)
chunk, err := b.repos.Chunks.GetByHash(ctx, chunkHash)
if err != nil {
return "", err
return err
}
chunkCount++
counters.chunkCount++
// In a real system, blobs would contain multiple chunks and be encrypted
// For testing, we'll create a blob with a "blob-" prefix to differentiate
@@ -481,7 +577,7 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
// Upload to S3 as a blob
err = b.s3Client.PutBlob(ctx, blobHash, dummyData)
if err != nil {
return "", err
return err
}
// Create blob entry in a short transaction
@@ -497,11 +593,11 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
return b.repos.Blobs.Create(ctx, tx, blob)
})
if err != nil {
return "", err
return err
}
blobCount++
blobSize += chunk.Size
counters.blobCount++
counters.blobSize += chunk.Size
// Create blob-chunk mapping in a short transaction
err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
@@ -515,27 +611,20 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
return b.repos.BlobChunks.Create(ctx, tx, blobChunk)
})
if err != nil {
return "", err
return err
}
// Add blob to snapshot in a short transaction
err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
return b.repos.Snapshots.AddBlob(ctx, tx, snapshotID, blobID, types.BlobHash(blobHash))
return b.repos.Snapshots.AddBlob(ctx, tx, snapshotID, blobID,
types.BlobHash(blobHash))
})
if err != nil {
return "", err
return err
}
}
// Update snapshot with final counts
err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
return b.repos.Snapshots.UpdateCounts(ctx, tx, snapshotID, fileCount, chunkCount, blobCount, totalSize, blobSize)
})
if err != nil {
return "", err
}
return snapshotID, nil
return nil
}
func calculateHash(data []byte) string {

129
internal/snapshot/exclude_test.go
View File

@@ -10,16 +10,15 @@ import (
"github.com/spf13/afero"
"github.com/stretchr/testify/require"
"sneak.berlin/go/vaultik/internal/database"
"sneak.berlin/go/vaultik/internal/log"
"sneak.berlin/go/vaultik/internal/snapshot"
"sneak.berlin/go/vaultik/internal/types"
)
func setupExcludeTestFS(t *testing.T) afero.Fs {
func setupExcludeTestFS(t *testing.T) *afero.MemMapFs {
t.Helper()
// Create in-memory filesystem
fs := afero.NewMemMapFs()
fs := &afero.MemMapFs{}
// Create test directory structure:
// /backup/
@@ -77,12 +76,11 @@ func setupExcludeTestFS(t *testing.T) afero.Fs {
return fs
}
func createTestScanner(t *testing.T, fs afero.Fs, excludePatterns []string) (*snapshot.Scanner, *database.Repositories, func()) {
func createTestScanner(
t *testing.T, fs afero.Fs, excludePatterns []string,
) (*snapshot.Scanner, *database.Repositories, func()) {
t.Helper()
// Initialize logger
log.Initialize(log.Config{})
// Create test database
db, err := database.NewTestDB()
require.NoError(t, err)
@@ -95,8 +93,9 @@ func createTestScanner(t *testing.T, fs afero.Fs, excludePatterns []string) (*sn
Repositories: repos,
MaxBlobSize: 1024 * 1024,
CompressionLevel: 3,
AgeRecipients: []string{"age1ql3z7hjy54pw3hyww5ayyfg7zqgvc7w3j2elw8zmrj2kg5sfn9aqmcac8p"},
Exclude: excludePatterns,
AgeRecipients: []string{
"age1ql3z7hjy54pw3hyww5ayyfg7zqgvc7w3j2elw8zmrj2kg5sfn9aqmcac8p"},
Exclude: excludePatterns,
})
cleanup := func() {
@@ -106,14 +105,16 @@ func createTestScanner(t *testing.T, fs afero.Fs, excludePatterns []string) (*sn
return scanner, repos, cleanup
}
func createSnapshotRecord(t *testing.T, ctx context.Context, repos *database.Repositories, snapshotID string) {
func createSnapshotRecord(
ctx context.Context, t *testing.T, repos *database.Repositories, snapshotID string,
) {
t.Helper()
err := repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
snap := &database.Snapshot{
ID: types.SnapshotID(snapshotID),
Hostname: "test-host",
VaultikVersion: "test",
Hostname: testHost,
VaultikVersion: testVersion,
StartedAt: time.Now(),
CompletedAt: nil,
FileCount: 0,
@@ -130,6 +131,8 @@ func createSnapshotRecord(t *testing.T, ctx context.Context, repos *database.Rep
}
func TestExcludePatterns_ExcludeGitDirectory(t *testing.T) {
t.Parallel()
fs := setupExcludeTestFS(t)
scanner, repos, cleanup := createTestScanner(t, fs, []string{".git"})
@@ -138,13 +141,14 @@ func TestExcludePatterns_ExcludeGitDirectory(t *testing.T) {
require.NotNil(t, scanner)
ctx := context.Background()
createSnapshotRecord(t, ctx, repos, "test-snapshot")
createSnapshotRecord(ctx, t, repos, "test-snapshot")
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
require.NoError(t, err)
// Should have scanned files but NOT .git directory contents
// Expected: file1.txt, file2.log, src/main.go, src/test.go, node_modules/package/index.js,
// Expected: file1.txt, file2.log, src/main.go, src/test.go,
// node_modules/package/index.js,
// cache/temp.dat, build/output.bin, docs/readme.md, .DS_Store, thumbs.db,
// src/.hidden, important.log.bak
// Excluded: .git/config, .git/objects/pack/data.pack
@@ -152,6 +156,8 @@ func TestExcludePatterns_ExcludeGitDirectory(t *testing.T) {
}
func TestExcludePatterns_ExcludeByExtension(t *testing.T) {
t.Parallel()
fs := setupExcludeTestFS(t)
scanner, repos, cleanup := createTestScanner(t, fs, []string{"*.log"})
@@ -160,7 +166,7 @@ func TestExcludePatterns_ExcludeByExtension(t *testing.T) {
require.NotNil(t, scanner)
ctx := context.Background()
createSnapshotRecord(t, ctx, repos, "test-snapshot")
createSnapshotRecord(ctx, t, repos, "test-snapshot")
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
require.NoError(t, err)
@@ -171,6 +177,8 @@ func TestExcludePatterns_ExcludeByExtension(t *testing.T) {
}
func TestExcludePatterns_ExcludeNodeModules(t *testing.T) {
t.Parallel()
fs := setupExcludeTestFS(t)
scanner, repos, cleanup := createTestScanner(t, fs, []string{"node_modules"})
@@ -179,7 +187,7 @@ func TestExcludePatterns_ExcludeNodeModules(t *testing.T) {
require.NotNil(t, scanner)
ctx := context.Background()
createSnapshotRecord(t, ctx, repos, "test-snapshot")
createSnapshotRecord(ctx, t, repos, "test-snapshot")
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
require.NoError(t, err)
@@ -190,25 +198,32 @@ func TestExcludePatterns_ExcludeNodeModules(t *testing.T) {
}
func TestExcludePatterns_MultiplePatterns(t *testing.T) {
t.Parallel()
fs := setupExcludeTestFS(t)
scanner, repos, cleanup := createTestScanner(t, fs, []string{".git", "node_modules", "*.log", ".DS_Store", "thumbs.db", "cache", "build"})
scanner, repos, cleanup := createTestScanner(t, fs, []string{
".git", "node_modules", "*.log", ".DS_Store", "thumbs.db", "cache", "build"})
defer cleanup()
require.NotNil(t, scanner)
ctx := context.Background()
createSnapshotRecord(t, ctx, repos, "test-snapshot")
createSnapshotRecord(ctx, t, repos, "test-snapshot")
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
require.NoError(t, err)
// Should only have: file1.txt, src/main.go, src/test.go, docs/readme.md, src/.hidden, important.log.bak
// Excluded: .git/*, node_modules/*, *.log (file2.log), .DS_Store, thumbs.db, cache/*, build/*
// Should only have: file1.txt, src/main.go, src/test.go, docs/readme.md,
// src/.hidden, important.log.bak
// Excluded: .git/*, node_modules/*, *.log (file2.log), .DS_Store,
// thumbs.db, cache/*, build/*
require.Equal(t, 6, result.FilesScanned, "Should exclude multiple patterns")
}
func TestExcludePatterns_NoExclusions(t *testing.T) {
t.Parallel()
fs := setupExcludeTestFS(t)
scanner, repos, cleanup := createTestScanner(t, fs, []string{})
@@ -217,7 +232,7 @@ func TestExcludePatterns_NoExclusions(t *testing.T) {
require.NotNil(t, scanner)
ctx := context.Background()
createSnapshotRecord(t, ctx, repos, "test-snapshot")
createSnapshotRecord(ctx, t, repos, "test-snapshot")
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
require.NoError(t, err)
@@ -227,6 +242,8 @@ func TestExcludePatterns_NoExclusions(t *testing.T) {
}
func TestExcludePatterns_ExcludeHiddenFiles(t *testing.T) {
t.Parallel()
fs := setupExcludeTestFS(t)
scanner, repos, cleanup := createTestScanner(t, fs, []string{".*"})
@@ -235,17 +252,21 @@ func TestExcludePatterns_ExcludeHiddenFiles(t *testing.T) {
require.NotNil(t, scanner)
ctx := context.Background()
createSnapshotRecord(t, ctx, repos, "test-snapshot")
createSnapshotRecord(ctx, t, repos, "test-snapshot")
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
require.NoError(t, err)
// Should exclude: .git/*, .DS_Store, src/.hidden
// Total files: 14, excluded: 4 (.git/config, .git/objects/pack/data.pack, .DS_Store, src/.hidden)
require.Equal(t, 10, result.FilesScanned, "Should exclude hidden files and directories")
// Total files: 14, excluded: 4 (.git/config,
// .git/objects/pack/data.pack, .DS_Store, src/.hidden)
require.Equal(t, 10, result.FilesScanned,
"Should exclude hidden files and directories")
}
func TestExcludePatterns_DoubleStarGlob(t *testing.T) {
t.Parallel()
fs := setupExcludeTestFS(t)
scanner, repos, cleanup := createTestScanner(t, fs, []string{"**/*.pack"})
@@ -254,7 +275,7 @@ func TestExcludePatterns_DoubleStarGlob(t *testing.T) {
require.NotNil(t, scanner)
ctx := context.Background()
createSnapshotRecord(t, ctx, repos, "test-snapshot")
createSnapshotRecord(ctx, t, repos, "test-snapshot")
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
require.NoError(t, err)
@@ -265,6 +286,8 @@ func TestExcludePatterns_DoubleStarGlob(t *testing.T) {
}
func TestExcludePatterns_ExactFileName(t *testing.T) {
t.Parallel()
fs := setupExcludeTestFS(t)
scanner, repos, cleanup := createTestScanner(t, fs, []string{"thumbs.db", ".DS_Store"})
@@ -273,7 +296,7 @@ func TestExcludePatterns_ExactFileName(t *testing.T) {
require.NotNil(t, scanner)
ctx := context.Background()
createSnapshotRecord(t, ctx, repos, "test-snapshot")
createSnapshotRecord(ctx, t, repos, "test-snapshot")
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
require.NoError(t, err)
@@ -284,6 +307,8 @@ func TestExcludePatterns_ExactFileName(t *testing.T) {
}
func TestExcludePatterns_CaseSensitive(t *testing.T) {
t.Parallel()
// Pattern matching should be case-sensitive
fs := setupExcludeTestFS(t)
@@ -293,7 +318,7 @@ func TestExcludePatterns_CaseSensitive(t *testing.T) {
require.NotNil(t, scanner)
ctx := context.Background()
createSnapshotRecord(t, ctx, repos, "test-snapshot")
createSnapshotRecord(ctx, t, repos, "test-snapshot")
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
require.NoError(t, err)
@@ -304,6 +329,8 @@ func TestExcludePatterns_CaseSensitive(t *testing.T) {
}
func TestExcludePatterns_DirectoryWithTrailingSlash(t *testing.T) {
t.Parallel()
fs := setupExcludeTestFS(t)
// Some users might add trailing slashes to directory patterns
scanner, repos, cleanup := createTestScanner(t, fs, []string{"cache/", "build/"})
@@ -312,17 +339,20 @@ func TestExcludePatterns_DirectoryWithTrailingSlash(t *testing.T) {
require.NotNil(t, scanner)
ctx := context.Background()
createSnapshotRecord(t, ctx, repos, "test-snapshot")
createSnapshotRecord(ctx, t, repos, "test-snapshot")
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
require.NoError(t, err)
// Should exclude cache/temp.dat and build/output.bin
// Total files: 14, excluded: 2
require.Equal(t, 12, result.FilesScanned, "Should handle directory patterns with trailing slashes")
require.Equal(t, 12, result.FilesScanned,
"Should handle directory patterns with trailing slashes")
}
func TestExcludePatterns_PatternInSubdirectory(t *testing.T) {
t.Parallel()
fs := setupExcludeTestFS(t)
// Exclude .hidden file specifically in src directory
scanner, repos, cleanup := createTestScanner(t, fs, []string{"src/.hidden"})
@@ -331,7 +361,7 @@ func TestExcludePatterns_PatternInSubdirectory(t *testing.T) {
require.NotNil(t, scanner)
ctx := context.Background()
createSnapshotRecord(t, ctx, repos, "test-snapshot")
createSnapshotRecord(ctx, t, repos, "test-snapshot")
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
require.NoError(t, err)
@@ -350,13 +380,14 @@ func TestExcludePatterns_PatternInSubdirectory(t *testing.T) {
// file.txt (should be excluded with /projectname)
// otherproject/
// projectname/
// file.txt (should NOT be excluded with /projectname, only with projectname)
// file.txt (should NOT be excluded with /projectname,
// only with projectname)
// src/
// file.go
func setupAnchoredTestFS(t *testing.T) afero.Fs {
func setupAnchoredTestFS(t *testing.T) *afero.MemMapFs {
t.Helper()
fs := afero.NewMemMapFs()
fs := &afero.MemMapFs{}
files := map[string]string{
"/backup/projectname/file.txt": "root project file",
@@ -381,6 +412,8 @@ func setupAnchoredTestFS(t *testing.T) afero.Fs {
}
func TestExcludePatterns_AnchoredPattern(t *testing.T) {
t.Parallel()
// Pattern starting with / should only match from root of source dir
fs := setupAnchoredTestFS(t)
@@ -390,7 +423,7 @@ func TestExcludePatterns_AnchoredPattern(t *testing.T) {
require.NotNil(t, scanner)
ctx := context.Background()
createSnapshotRecord(t, ctx, repos, "test-snapshot")
createSnapshotRecord(ctx, t, repos, "test-snapshot")
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
require.NoError(t, err)
@@ -398,10 +431,13 @@ func TestExcludePatterns_AnchoredPattern(t *testing.T) {
// /projectname should ONLY exclude /backup/projectname/file.txt (1 file)
// /backup/otherproject/projectname/file.txt should NOT be excluded
// Total files: 4, excluded: 1
require.Equal(t, 3, result.FilesScanned, "Anchored pattern /projectname should only match at root of source dir")
require.Equal(t, 3, result.FilesScanned,
"Anchored pattern /projectname should only match at root of source dir")
}
func TestExcludePatterns_UnanchoredPattern(t *testing.T) {
t.Parallel()
// Pattern without leading / should match anywhere in path
fs := setupAnchoredTestFS(t)
@@ -411,7 +447,7 @@ func TestExcludePatterns_UnanchoredPattern(t *testing.T) {
require.NotNil(t, scanner)
ctx := context.Background()
createSnapshotRecord(t, ctx, repos, "test-snapshot")
createSnapshotRecord(ctx, t, repos, "test-snapshot")
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
require.NoError(t, err)
@@ -420,10 +456,13 @@ func TestExcludePatterns_UnanchoredPattern(t *testing.T) {
// - /backup/projectname/file.txt
// - /backup/otherproject/projectname/file.txt
// Total files: 4, excluded: 2
require.Equal(t, 2, result.FilesScanned, "Unanchored pattern should match anywhere in path")
require.Equal(t, 2, result.FilesScanned,
"Unanchored pattern should match anywhere in path")
}
func TestExcludePatterns_AnchoredPatternWithGlob(t *testing.T) {
t.Parallel()
// Anchored pattern with glob
fs := setupAnchoredTestFS(t)
@@ -433,7 +472,7 @@ func TestExcludePatterns_AnchoredPatternWithGlob(t *testing.T) {
require.NotNil(t, scanner)
ctx := context.Background()
createSnapshotRecord(t, ctx, repos, "test-snapshot")
createSnapshotRecord(ctx, t, repos, "test-snapshot")
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
require.NoError(t, err)
@@ -444,6 +483,8 @@ func TestExcludePatterns_AnchoredPatternWithGlob(t *testing.T) {
}
func TestExcludePatterns_AnchoredPatternFile(t *testing.T) {
t.Parallel()
// Anchored pattern for exact file at root
fs := setupAnchoredTestFS(t)
@@ -453,7 +494,7 @@ func TestExcludePatterns_AnchoredPatternFile(t *testing.T) {
require.NotNil(t, scanner)
ctx := context.Background()
createSnapshotRecord(t, ctx, repos, "test-snapshot")
createSnapshotRecord(ctx, t, repos, "test-snapshot")
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
require.NoError(t, err)
@@ -461,10 +502,13 @@ func TestExcludePatterns_AnchoredPatternFile(t *testing.T) {
// /file.txt should ONLY exclude /backup/file.txt
// NOT /backup/projectname/file.txt or /backup/otherproject/projectname/file.txt
// Total files: 4, excluded: 1
require.Equal(t, 3, result.FilesScanned, "Anchored pattern for file should only match at root")
require.Equal(t, 3, result.FilesScanned,
"Anchored pattern for file should only match at root")
}
func TestExcludePatterns_UnanchoredPatternFile(t *testing.T) {
t.Parallel()
// Unanchored pattern for file should match anywhere
fs := setupAnchoredTestFS(t)
@@ -474,7 +518,7 @@ func TestExcludePatterns_UnanchoredPatternFile(t *testing.T) {
require.NotNil(t, scanner)
ctx := context.Background()
createSnapshotRecord(t, ctx, repos, "test-snapshot")
createSnapshotRecord(ctx, t, repos, "test-snapshot")
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
require.NoError(t, err)
@@ -484,5 +528,6 @@ func TestExcludePatterns_UnanchoredPatternFile(t *testing.T) {
// - /backup/projectname/file.txt
// - /backup/otherproject/projectname/file.txt
// Total files: 4, excluded: 3
require.Equal(t, 1, result.FilesScanned, "Unanchored pattern for file should match anywhere")
require.Equal(t, 1, result.FilesScanned,
"Unanchored pattern for file should match anywhere")
}

136
internal/snapshot/file_change_test.go
View File

@@ -2,7 +2,6 @@ package snapshot_test
import (
"context"
"database/sql"
"testing"
"time"
@@ -15,11 +14,55 @@ import (
"sneak.berlin/go/vaultik/internal/types"
)
// verifyChunkChange checks that after a content change the file references
// the new chunk, the old chunk still exists, and the old chunk no longer
// maps to the modified file.
func verifyChunkChange(
ctx context.Context,
t *testing.T,
repos *database.Repositories,
oldChunkHash, newChunkHash types.ChunkHash,
) {
t.Helper()
// Verify the chunk hashes are different
assert.NotEqual(t, oldChunkHash, newChunkHash,
"Chunk hash should change when content changes")
// Get chunk files from second scan
chunkFiles2, err := repos.ChunkFiles.GetByFilePath(ctx, "/test.txt")
require.NoError(t, err)
assert.Len(t, chunkFiles2, 1)
assert.Equal(t, newChunkHash, chunkFiles2[0].ChunkHash)
// Verify old chunk still exists (it's still valid data)
oldChunk, err := repos.Chunks.GetByHash(ctx, oldChunkHash.String())
require.NoError(t, err)
assert.NotNil(t, oldChunk)
// Verify new chunk exists
newChunk, err := repos.Chunks.GetByHash(ctx, newChunkHash.String())
require.NoError(t, err)
assert.NotNil(t, newChunk)
// Verify that chunk_files for old chunk no longer references this file
oldChunkFiles, err := repos.ChunkFiles.GetByChunkHash(ctx, oldChunkHash)
require.NoError(t, err)
for _, cf := range oldChunkFiles {
file, err := repos.Files.GetByID(ctx, cf.FileID)
require.NoError(t, err)
assert.NotEqual(t, "/data/test.txt", file.Path,
"Old chunk should not be associated with the modified file")
}
}
// TestFileContentChange verifies that when a file's content changes,
// the old chunks are properly disassociated
func TestFileContentChange(t *testing.T) {
// Initialize logger for tests
log.Initialize(log.Config{})
t.Parallel()
// Create in-memory filesystem
fs := afero.NewMemMapFs()
@@ -48,23 +91,13 @@ func TestFileContentChange(t *testing.T) {
Repositories: repos,
MaxBlobSize: int64(1024 * 1024), // 1MB blobs
CompressionLevel: 3,
AgeRecipients: []string{"age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05gl0sjq9q9wjg"}, // Test public key
AgeRecipients: []string{testAgePublicKey},
})
// Create first snapshot
ctx := context.Background()
snapshotID1 := "snapshot1"
err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
snapshot := &database.Snapshot{
ID: types.SnapshotID(snapshotID1),
Hostname: "test-host",
VaultikVersion: "test",
StartedAt: time.Now(),
}
return repos.Snapshots.Create(ctx, tx, snapshot)
})
require.NoError(t, err)
createSnapshotRecord(ctx, t, repos, snapshotID1)
// First scan - should create chunks for initial content
result1, err := scanner.Scan(ctx, "/", snapshotID1)
@@ -85,22 +118,13 @@ func TestFileContentChange(t *testing.T) {
// Modify the file
time.Sleep(10 * time.Millisecond) // Ensure mtime changes
err = afero.WriteFile(fs, "/test.txt", []byte("Modified content with different data"), 0644)
err = afero.WriteFile(fs, "/test.txt",
[]byte("Modified content with different data"), 0644)
require.NoError(t, err)
// Create second snapshot
snapshotID2 := "snapshot2"
err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
snapshot := &database.Snapshot{
ID: types.SnapshotID(snapshotID2),
Hostname: "test-host",
VaultikVersion: "test",
StartedAt: time.Now(),
}
return repos.Snapshots.Create(ctx, tx, snapshot)
})
require.NoError(t, err)
createSnapshotRecord(ctx, t, repos, snapshotID2)
// Second scan - should create new chunks and remove old associations
result2, err := scanner.Scan(ctx, "/", snapshotID2)
@@ -113,40 +137,14 @@ func TestFileContentChange(t *testing.T) {
assert.Len(t, fileChunks2, 1) // Still 1 chunk but different hash
newChunkHash := fileChunks2[0].ChunkHash
// Verify the chunk hashes are different
assert.NotEqual(t, oldChunkHash, newChunkHash, "Chunk hash should change when content changes")
// Get chunk files from second scan
chunkFiles2, err := repos.ChunkFiles.GetByFilePath(ctx, "/test.txt")
require.NoError(t, err)
assert.Len(t, chunkFiles2, 1)
assert.Equal(t, newChunkHash, chunkFiles2[0].ChunkHash)
// Verify old chunk still exists (it's still valid data)
oldChunk, err := repos.Chunks.GetByHash(ctx, oldChunkHash.String())
require.NoError(t, err)
assert.NotNil(t, oldChunk)
// Verify new chunk exists
newChunk, err := repos.Chunks.GetByHash(ctx, newChunkHash.String())
require.NoError(t, err)
assert.NotNil(t, newChunk)
// Verify that chunk_files for old chunk no longer references this file
oldChunkFiles, err := repos.ChunkFiles.GetByChunkHash(ctx, oldChunkHash)
require.NoError(t, err)
for _, cf := range oldChunkFiles {
file, err := repos.Files.GetByID(ctx, cf.FileID)
require.NoError(t, err)
assert.NotEqual(t, "/data/test.txt", file.Path, "Old chunk should not be associated with the modified file")
}
verifyChunkChange(ctx, t, repos, oldChunkHash, newChunkHash)
}
// TestMultipleFileChanges verifies handling of multiple file changes in one scan
func TestMultipleFileChanges(t *testing.T) {
// Initialize logger for tests
log.Initialize(log.Config{})
t.Parallel()
// Create in-memory filesystem
fs := afero.NewMemMapFs()
@@ -183,23 +181,13 @@ func TestMultipleFileChanges(t *testing.T) {
Repositories: repos,
MaxBlobSize: int64(1024 * 1024), // 1MB blobs
CompressionLevel: 3,
AgeRecipients: []string{"age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05gl0sjq9q9wjg"}, // Test public key
AgeRecipients: []string{testAgePublicKey},
})
// Create first snapshot
ctx := context.Background()
snapshotID1 := "snapshot1"
err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
snapshot := &database.Snapshot{
ID: types.SnapshotID(snapshotID1),
Hostname: "test-host",
VaultikVersion: "test",
StartedAt: time.Now(),
}
return repos.Snapshots.Create(ctx, tx, snapshot)
})
require.NoError(t, err)
createSnapshotRecord(ctx, t, repos, snapshotID1)
// First scan
result1, err := scanner.Scan(ctx, "/", snapshotID1)
@@ -217,17 +205,7 @@ func TestMultipleFileChanges(t *testing.T) {
// Create second snapshot
snapshotID2 := "snapshot2"
err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
snapshot := &database.Snapshot{
ID: types.SnapshotID(snapshotID2),
Hostname: "test-host",
VaultikVersion: "test",
StartedAt: time.Now(),
}
return repos.Snapshots.Create(ctx, tx, snapshot)
})
require.NoError(t, err)
createSnapshotRecord(ctx, t, repos, snapshotID2)
// Second scan
result2, err := scanner.Scan(ctx, "/", snapshotID2)
@@ -240,10 +218,12 @@ func TestMultipleFileChanges(t *testing.T) {
for path := range files {
fileChunks, err := repos.FileChunks.GetByPath(ctx, path)
require.NoError(t, err)
assert.Len(t, fileChunks, 1, "File %s should have exactly 1 chunk association", path)
assert.Len(t, fileChunks, 1,
"File %s should have exactly 1 chunk association", path)
chunkFiles, err := repos.ChunkFiles.GetByFilePath(ctx, path)
require.NoError(t, err)
assert.Len(t, chunkFiles, 1, "File %s should have exactly 1 chunk-file association", path)
assert.Len(t, chunkFiles, 1,
"File %s should have exactly 1 chunk-file association", path)
}
}

7
internal/snapshot/manifest.go
View File

@@ -10,6 +10,8 @@ import (
)
// Manifest represents the structure of a snapshot's blob manifest
//
//nolint:tagliatelle // snake_case is the established on-disk manifest format
type Manifest struct {
SnapshotID string `json:"snapshot_id"`
Timestamp string `json:"timestamp"`
@@ -19,6 +21,8 @@ type Manifest struct {
}
// BlobInfo represents information about a single blob in the manifest
//
//nolint:tagliatelle // snake_case is the established on-disk manifest format
type BlobInfo struct {
Hash string `json:"hash"`
CompressedSize int64 `json:"compressed_size"`
@@ -55,7 +59,8 @@ func EncodeManifest(manifest *Manifest, compressionLevel int) ([]byte, error) {
// Compress using zstd
var compressedBuf bytes.Buffer
writer, err := zstd.NewWriter(&compressedBuf, zstd.WithEncoderLevel(zstd.EncoderLevelFromZstd(compressionLevel)))
writer, err := zstd.NewWriter(&compressedBuf,
zstd.WithEncoderLevel(zstd.EncoderLevelFromZstd(compressionLevel)))
if err != nil {
return nil, fmt.Errorf("creating zstd writer: %w", err)
}

6
internal/snapshot/module.go
View File

@@ -21,6 +21,8 @@ type ScannerParams struct {
// Module exports backup functionality as an fx module.
// It provides a ScannerFactory that can create Scanner instances
// with custom parameters while sharing common dependencies.
//
//nolint:gochecknoglobals // fx module definitions are conventionally globals
var Module = fx.Module("backup",
fx.Provide(
provideScannerFactory,
@@ -31,7 +33,9 @@ var Module = fx.Module("backup",
// ScannerFactory creates scanners with custom parameters
type ScannerFactory func(params ScannerParams) *Scanner
func provideScannerFactory(cfg *config.Config, repos *database.Repositories, storer storage.Storer) ScannerFactory {
func provideScannerFactory(
cfg *config.Config, repos *database.Repositories, storer storage.Storer,
) ScannerFactory {
return func(params ScannerParams) *Scanner {
// Use provided excludes, or fall back to global config excludes
excludes := params.Exclude

27
internal/snapshot/permission_error_test.go
View File

@@ -1,3 +1,4 @@
//nolint:testpackage // needs access to unexported wrapPermissionError
package snapshot
import (
@@ -9,12 +10,15 @@ import (
"testing"
)
func TestWrapPermissionError(t *testing.T) {
// Non-permission errors pass through unchanged.
plain := errors.New("disk on fire")
// errDiskOnFire is a non-permission sentinel used to verify pass-through.
var errDiskOnFire = errors.New("disk on fire")
got := wrapPermissionError("/some/path", plain)
if !errors.Is(got, plain) {
func TestWrapPermissionError(t *testing.T) {
t.Parallel()
// Non-permission errors pass through unchanged.
got := wrapPermissionError("/some/path", errDiskOnFire)
if !errors.Is(got, errDiskOnFire) {
t.Errorf("non-permission error should pass through, got %v", got)
}
@@ -32,15 +36,16 @@ func TestWrapPermissionError(t *testing.T) {
if runtime.GOOS == "darwin" {
if !strings.Contains(wrapped.Error(), "Full Disk Access") {
t.Errorf("macOS permission error should mention Full Disk Access:\n%s", wrapped.Error())
t.Errorf("macOS permission error should mention Full Disk Access:\n%s",
wrapped.Error())
}
if !strings.Contains(wrapped.Error(), "System Settings") {
t.Errorf("macOS permission error should point at System Settings:\n%s", wrapped.Error())
}
} else {
if !strings.Contains(wrapped.Error(), "--skip-errors") {
t.Errorf("non-macOS permission error should mention --skip-errors:\n%s", wrapped.Error())
t.Errorf("macOS permission error should point at System Settings:\n%s",
wrapped.Error())
}
} else if !strings.Contains(wrapped.Error(), "--skip-errors") {
t.Errorf("non-macOS permission error should mention --skip-errors:\n%s",
wrapped.Error())
}
}

357
internal/snapshot/progress.go
View File

@@ -19,14 +19,41 @@ const (
// These updates show current progress, ETA, and the file being processed.
SummaryInterval = 10 * time.Second
// DetailInterval defines how often multi-line detailed status reports are printed.
// These reports include comprehensive statistics about files, chunks, blobs, and uploads.
// DetailInterval defines how often multi-line detailed status reports are
// printed. These reports include comprehensive statistics about files,
// chunks, blobs, and uploads.
DetailInterval = 60 * time.Second
// UploadProgressInterval defines how often upload progress messages are logged.
UploadProgressInterval = 15 * time.Second
)
const (
// bitsPerByte converts byte counts to bit counts for speed display.
bitsPerByte = 8
// percentScale converts a ratio to a percentage.
percentScale = 100
// currentFileMaxLen is the display width used for current-file paths.
currentFileMaxLen = 40
// secondsPerMinute and minutesPerHour are used for duration formatting.
secondsPerMinute = 60
minutesPerHour = 60
// Bit-rate thresholds for human-readable upload speed formatting.
bitsPerGbit = 1e9
bitsPerMbit = 1e6
bitsPerKbit = 1e3
// ellipsis prefixes truncated paths and suffixes shortened hashes.
ellipsis = "..."
// hashPrefixLen is how many hex characters of a blob hash to show in logs.
hashPrefixLen = 8
)
// ProgressStats holds atomic counters for progress tracking
type ProgressStats struct {
FilesScanned atomic.Int64 // Total files seen during scan (includes skipped)
@@ -64,7 +91,7 @@ type UploadInfo struct {
// ProgressReporter handles periodic progress reporting
type ProgressReporter struct {
stats *ProgressStats
ctx context.Context
ctx context.Context //nolint:containedctx // bound at construction
cancel context.CancelFunc
wg sync.WaitGroup
detailTicker *time.Ticker
@@ -127,6 +154,161 @@ func (pr *ProgressReporter) SetTotalSize(size int64) {
pr.stats.ProcessStartTime.Store(time.Now().UTC())
}
// Helper functions
func formatDuration(d time.Duration) string {
if d < 0 {
return "unknown"
}
if d < time.Minute {
return fmt.Sprintf("%ds", int(d.Seconds()))
}
if d < time.Hour {
return fmt.Sprintf("%dm%ds", int(d.Minutes()), int(d.Seconds())%secondsPerMinute)
}
return fmt.Sprintf("%dh%dm", int(d.Hours()), int(d.Minutes())%minutesPerHour)
}
func formatPercent(numerator, denominator int64) string {
if denominator == 0 {
return "0.0%"
}
return fmt.Sprintf("%.1f%%", float64(numerator)/float64(denominator)*percentScale)
}
func formatRatio(compressed, uncompressed int64) string {
if uncompressed == 0 {
return "1.00"
}
ratio := float64(compressed) / float64(uncompressed)
return fmt.Sprintf("%.2f", ratio)
}
func truncatePath(path string, maxLen int) string {
if len(path) <= maxLen {
return path
}
// Keep the last maxLen-len(ellipsis) characters and prepend the ellipsis.
return ellipsis + path[len(path)-(maxLen-len(ellipsis)):]
}
// safeUint64 converts a non-negative int64 counter to uint64 for display,
// clamping negative values to zero.
func safeUint64(n int64) uint64 {
if n < 0 {
return 0
}
return uint64(n)
}
// ReportUploadStart marks the beginning of a blob upload
func (pr *ProgressReporter) ReportUploadStart(blobHash string, size int64) {
info := &UploadInfo{
BlobHash: blobHash,
Size: size,
StartTime: time.Now().UTC(),
}
pr.stats.CurrentUpload.Store(info)
// Log the start of upload
log.Info("Starting blob upload",
"hash", blobHash[:hashPrefixLen]+ellipsis,
"size", humanize.Bytes(safeUint64(size)))
}
// ReportUploadComplete marks the completion of a blob upload
func (pr *ProgressReporter) ReportUploadComplete(
blobHash string, size int64, duration time.Duration,
) {
// Clear current upload
pr.stats.CurrentUpload.Store((*UploadInfo)(nil))
// Add to total upload duration
pr.stats.UploadDurationMs.Add(duration.Milliseconds())
// Calculate speed
if duration < time.Millisecond {
duration = time.Millisecond
}
bytesPerSec := float64(size) / duration.Seconds()
bitsPerSec := bytesPerSec * bitsPerByte
// Format speed
var speedStr string
switch {
case bitsPerSec >= bitsPerGbit:
speedStr = fmt.Sprintf("%.1fGbit/sec", bitsPerSec/bitsPerGbit)
case bitsPerSec >= bitsPerMbit:
speedStr = fmt.Sprintf("%.0fMbit/sec", bitsPerSec/bitsPerMbit)
case bitsPerSec >= bitsPerKbit:
speedStr = fmt.Sprintf("%.0fKbit/sec", bitsPerSec/bitsPerKbit)
default:
speedStr = fmt.Sprintf("%.0fbit/sec", bitsPerSec)
}
log.Info("Blob upload completed",
"hash", blobHash[:hashPrefixLen]+ellipsis,
"size", humanize.Bytes(safeUint64(size)),
"duration", formatDuration(duration),
"speed", speedStr)
}
// UpdateChunkingActivity updates the last chunking time
func (pr *ProgressReporter) UpdateChunkingActivity() {
pr.stats.mu.Lock()
pr.stats.lastChunkingTime = time.Now().UTC()
pr.stats.mu.Unlock()
}
// ReportUploadProgress reports current upload progress with instantaneous speed
func (pr *ProgressReporter) ReportUploadProgress(
blobHash string, bytesUploaded, totalSize int64, instantSpeed float64,
) {
// Update the current upload info with progress
uploadInfo, ok := pr.stats.CurrentUpload.Load().(*UploadInfo)
if ok && uploadInfo != nil {
now := time.Now()
// Only log at the configured interval
if now.Sub(uploadInfo.LastLogTime) >= UploadProgressInterval {
// Format speed in bits/second using humanize
bitsPerSec := instantSpeed * bitsPerByte
speedStr := humanize.SI(bitsPerSec, "bit/sec")
percent := float64(bytesUploaded) / float64(totalSize) * percentScale
// Calculate ETA based on current speed
etaStr := "unknown"
if instantSpeed > 0 && bytesUploaded < totalSize {
remainingBytes := totalSize - bytesUploaded
remainingSeconds := float64(remainingBytes) / instantSpeed
eta := time.Duration(remainingSeconds * float64(time.Second))
etaStr = formatDuration(eta)
}
log.Info("Blob upload progress",
"hash", blobHash[:hashPrefixLen]+ellipsis,
"progress", fmt.Sprintf("%.1f%%", percent),
"uploaded", humanize.Bytes(safeUint64(bytesUploaded)),
"total", humanize.Bytes(safeUint64(totalSize)),
"speed", speedStr,
"eta", etaStr)
uploadInfo.LastLogTime = now
}
}
}
// run is the main progress reporting loop
func (pr *ProgressReporter) run() {
defer pr.wg.Done()
@@ -150,7 +332,8 @@ func (pr *ProgressReporter) run() {
// printSummaryStatus prints a one-line status update
func (pr *ProgressReporter) printSummaryStatus() {
// Check if we're currently uploading
if uploadInfo, ok := pr.stats.CurrentUpload.Load().(*UploadInfo); ok && uploadInfo != nil {
uploadInfo, ok := pr.stats.CurrentUpload.Load().(*UploadInfo)
if ok && uploadInfo != nil {
// Show upload progress instead
pr.printUploadProgress(uploadInfo)
@@ -172,7 +355,7 @@ func (pr *ProgressReporter) printSummaryStatus() {
bytesSkipped := pr.stats.BytesSkipped.Load()
bytesProcessed := pr.stats.BytesProcessed.Load()
totalSize := pr.stats.TotalSize.Load()
currentFile := pr.stats.CurrentFile.Load().(string)
currentFile, _ := pr.stats.CurrentFile.Load().(string)
// Calculate ETA if we have total size and are processing
etaStr := ""
@@ -201,15 +384,15 @@ func (pr *ProgressReporter) printSummaryStatus() {
status := fmt.Sprintf("Snapshot progress: %d/%d files, %s/%s (%.1f%%), %s/s%s",
filesProcessed,
totalFiles,
humanize.Bytes(uint64(bytesProcessed)),
humanize.Bytes(uint64(totalSize)),
float64(bytesProcessed)/float64(totalSize)*100,
humanize.Bytes(safeUint64(bytesProcessed)),
humanize.Bytes(safeUint64(totalSize)),
float64(bytesProcessed)/float64(totalSize)*percentScale,
humanize.Bytes(uint64(rate)),
etaStr,
)
if currentFile != "" {
status += " | Current: " + truncatePath(currentFile, 40)
status += " | Current: " + truncatePath(currentFile, currentFileMaxLen)
}
log.Info(status)
@@ -232,7 +415,7 @@ func (pr *ProgressReporter) printDetailedStatus() {
blobsCreated := pr.stats.BlobsCreated.Load()
blobsUploaded := pr.stats.BlobsUploaded.Load()
bytesUploaded := pr.stats.BytesUploaded.Load()
currentFile := pr.stats.CurrentFile.Load().(string)
currentFile, _ := pr.stats.CurrentFile.Load().(string)
totalBytes := bytesScanned + bytesSkipped
rate := float64(totalBytes) / elapsed.Seconds()
@@ -251,11 +434,11 @@ func (pr *ProgressReporter) printDetailedStatus() {
remainingBytes := totalSize - bytesProcessed
remainingSeconds := float64(remainingBytes) / processRate
eta := time.Duration(remainingSeconds * float64(time.Second))
percentComplete := float64(bytesProcessed) / float64(totalSize) * 100
percentComplete := float64(bytesProcessed) / float64(totalSize) * percentScale
log.Info("Overall progress",
"percent", fmt.Sprintf("%.1f%%", percentComplete),
"processed", humanize.Bytes(uint64(bytesProcessed)),
"total", humanize.Bytes(uint64(totalSize)),
"processed", humanize.Bytes(safeUint64(bytesProcessed)),
"total", humanize.Bytes(safeUint64(totalSize)),
"rate", humanize.Bytes(uint64(processRate))+"/s",
"eta", formatDuration(eta))
}
@@ -268,9 +451,9 @@ func (pr *ProgressReporter) printDetailedStatus() {
"total", filesScanned,
"skip_rate", formatPercent(filesSkipped, filesScanned))
log.Info("Data scanned",
"new", humanize.Bytes(uint64(bytesScanned)),
"skipped", humanize.Bytes(uint64(bytesSkipped)),
"total", humanize.Bytes(uint64(totalBytes)),
"new", humanize.Bytes(safeUint64(bytesScanned)),
"skipped", humanize.Bytes(safeUint64(bytesSkipped)),
"total", humanize.Bytes(safeUint64(totalBytes)),
"scan_rate", humanize.Bytes(uint64(rate))+"/s")
log.Info("Chunks created", "count", chunksCreated)
log.Info("Blobs status",
@@ -278,7 +461,7 @@ func (pr *ProgressReporter) printDetailedStatus() {
"uploaded", blobsUploaded,
"pending", blobsCreated-blobsUploaded)
log.Info("Total uploaded to remote",
"uploaded", humanize.Bytes(uint64(bytesUploaded)),
"uploaded", humanize.Bytes(safeUint64(bytesUploaded)),
"compression_ratio", formatRatio(bytesUploaded, bytesScanned))
if currentFile != "" {
@@ -288,146 +471,8 @@ func (pr *ProgressReporter) printDetailedStatus() {
log.Notice("=============================")
}
// Helper functions
func formatDuration(d time.Duration) string {
if d < 0 {
return "unknown"
}
if d < time.Minute {
return fmt.Sprintf("%ds", int(d.Seconds()))
}
if d < time.Hour {
return fmt.Sprintf("%dm%ds", int(d.Minutes()), int(d.Seconds())%60)
}
return fmt.Sprintf("%dh%dm", int(d.Hours()), int(d.Minutes())%60)
}
func formatPercent(numerator, denominator int64) string {
if denominator == 0 {
return "0.0%"
}
return fmt.Sprintf("%.1f%%", float64(numerator)/float64(denominator)*100)
}
func formatRatio(compressed, uncompressed int64) string {
if uncompressed == 0 {
return "1.00"
}
ratio := float64(compressed) / float64(uncompressed)
return fmt.Sprintf("%.2f", ratio)
}
func truncatePath(path string, maxLen int) string {
if len(path) <= maxLen {
return path
}
// Keep the last maxLen-3 characters and prepend "..."
return "..." + path[len(path)-(maxLen-3):]
}
// printUploadProgress prints upload progress
func (pr *ProgressReporter) printUploadProgress(info *UploadInfo) {
func (pr *ProgressReporter) printUploadProgress(_ *UploadInfo) {
// This function is called repeatedly during upload, not just at start
// Don't print anything here - the actual progress is shown by ReportUploadProgress
}
// ReportUploadStart marks the beginning of a blob upload
func (pr *ProgressReporter) ReportUploadStart(blobHash string, size int64) {
info := &UploadInfo{
BlobHash: blobHash,
Size: size,
StartTime: time.Now().UTC(),
}
pr.stats.CurrentUpload.Store(info)
// Log the start of upload
log.Info("Starting blob upload",
"hash", blobHash[:8]+"...",
"size", humanize.Bytes(uint64(size)))
}
// ReportUploadComplete marks the completion of a blob upload
func (pr *ProgressReporter) ReportUploadComplete(blobHash string, size int64, duration time.Duration) {
// Clear current upload
pr.stats.CurrentUpload.Store((*UploadInfo)(nil))
// Add to total upload duration
pr.stats.UploadDurationMs.Add(duration.Milliseconds())
// Calculate speed
if duration < time.Millisecond {
duration = time.Millisecond
}
bytesPerSec := float64(size) / duration.Seconds()
bitsPerSec := bytesPerSec * 8
// Format speed
var speedStr string
if bitsPerSec >= 1e9 {
speedStr = fmt.Sprintf("%.1fGbit/sec", bitsPerSec/1e9)
} else if bitsPerSec >= 1e6 {
speedStr = fmt.Sprintf("%.0fMbit/sec", bitsPerSec/1e6)
} else if bitsPerSec >= 1e3 {
speedStr = fmt.Sprintf("%.0fKbit/sec", bitsPerSec/1e3)
} else {
speedStr = fmt.Sprintf("%.0fbit/sec", bitsPerSec)
}
log.Info("Blob upload completed",
"hash", blobHash[:8]+"...",
"size", humanize.Bytes(uint64(size)),
"duration", formatDuration(duration),
"speed", speedStr)
}
// UpdateChunkingActivity updates the last chunking time
func (pr *ProgressReporter) UpdateChunkingActivity() {
pr.stats.mu.Lock()
pr.stats.lastChunkingTime = time.Now().UTC()
pr.stats.mu.Unlock()
}
// ReportUploadProgress reports current upload progress with instantaneous speed
func (pr *ProgressReporter) ReportUploadProgress(blobHash string, bytesUploaded, totalSize int64, instantSpeed float64) {
// Update the current upload info with progress
if uploadInfo, ok := pr.stats.CurrentUpload.Load().(*UploadInfo); ok && uploadInfo != nil {
now := time.Now()
// Only log at the configured interval
if now.Sub(uploadInfo.LastLogTime) >= UploadProgressInterval {
// Format speed in bits/second using humanize
bitsPerSec := instantSpeed * 8
speedStr := humanize.SI(bitsPerSec, "bit/sec")
percent := float64(bytesUploaded) / float64(totalSize) * 100
// Calculate ETA based on current speed
etaStr := "unknown"
if instantSpeed > 0 && bytesUploaded < totalSize {
remainingBytes := totalSize - bytesUploaded
remainingSeconds := float64(remainingBytes) / instantSpeed
eta := time.Duration(remainingSeconds * float64(time.Second))
etaStr = formatDuration(eta)
}
log.Info("Blob upload progress",
"hash", blobHash[:8]+"...",
"progress", fmt.Sprintf("%.1f%%", percent),
"uploaded", humanize.Bytes(uint64(bytesUploaded)),
"total", humanize.Bytes(uint64(totalSize)),
"speed", speedStr,
"eta", etaStr)
uploadInfo.LastLogTime = now
}
}
}

895
internal/snapshot/scanner.go
View File

File diff suppressed because it is too large Load Diff

213
internal/snapshot/scanner_test.go
View File

@@ -3,6 +3,7 @@ package snapshot_test
import (
"context"
"database/sql"
"os"
"path/filepath"
"testing"
"time"
@@ -14,9 +15,114 @@ import (
"sneak.berlin/go/vaultik/internal/types"
)
// Shared test fixture values for the snapshot_test package.
const (
// testHost is the hostname recorded on test snapshot rows.
testHost = "test-host"
// testVersion is the vaultik version recorded on test snapshot rows.
testVersion = "test"
// testAgePublicKey is the fixed age public key used for test encryption.
testAgePublicKey = "age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05gl0sjq9q9wjg"
)
// TestMain initializes the shared logger once, before any tests run, so
// parallel tests never race on the logger's global state.
func TestMain(m *testing.M) {
log.Initialize(log.Config{})
os.Exit(m.Run())
}
// createTestSnapshotRecord inserts an empty snapshot row used as the
// association target for scan tests.
func createTestSnapshotRecord(
ctx context.Context, t *testing.T, repos *database.Repositories, snapshotID string,
) {
t.Helper()
err := repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
snapshot := &database.Snapshot{
ID: types.SnapshotID(snapshotID),
Hostname: testHost,
VaultikVersion: testVersion,
StartedAt: time.Now(),
CompletedAt: nil,
FileCount: 0,
ChunkCount: 0,
BlobCount: 0,
TotalSize: 0,
BlobSize: 0,
CompressionRatio: 1.0,
}
return repos.Snapshots.Create(ctx, tx, snapshot)
})
if err != nil {
t.Fatalf("failed to create snapshot: %v", err)
}
}
// verifySimpleScanDatabase checks the database contents produced by
// TestScannerSimpleDirectory's scan.
func verifySimpleScanDatabase(
ctx context.Context, t *testing.T, repos *database.Repositories,
) {
t.Helper()
// Verify files in database - includes regular files and directories
files, err := repos.Files.ListByPrefix(ctx, "/source")
if err != nil {
t.Fatalf("failed to list files: %v", err)
}
// 6 regular files + 3 directories (/source, /source/subdir, /source/subdir2)
if len(files) != 9 {
t.Errorf("expected 9 entries in database (6 files + 3 dirs), got %d", len(files))
}
// Verify specific file
file1, err := repos.Files.GetByPath(ctx, "/source/file1.txt")
if err != nil {
t.Fatalf("failed to get file1.txt: %v", err)
}
if file1.Size != 13 {
t.Errorf("expected file1.txt size 13, got %d", file1.Size)
}
if file1.Mode != 0644 {
t.Errorf("expected file1.txt mode 0644, got %o", file1.Mode)
}
// Verify chunks were created
chunks, err := repos.FileChunks.GetByFile(ctx, "/source/file1.txt")
if err != nil {
t.Fatalf("failed to get chunks for file1.txt: %v", err)
}
if len(chunks) != 1 { // Small file should be one chunk
t.Errorf("expected 1 chunk for file1.txt, got %d", len(chunks))
}
// Verify deduplication - file3.txt and file4.txt have different content
// but we should still have the correct number of unique chunks
allChunks, err := repos.Chunks.List(ctx)
if err != nil {
t.Fatalf("failed to list all chunks: %v", err)
}
// We should have at most 6 chunks (one per unique file content)
// Empty file might not create a chunk
if len(allChunks) > 6 {
t.Errorf("expected at most 6 chunks, got %d", len(allChunks))
}
}
func TestScannerSimpleDirectory(t *testing.T) {
// Initialize logger for tests
log.Initialize(log.Config{})
t.Parallel()
// Create in-memory filesystem
fs := afero.NewMemMapFs()
@@ -74,38 +180,17 @@ func TestScannerSimpleDirectory(t *testing.T) {
Repositories: repos,
MaxBlobSize: int64(1024 * 1024), // 1MB blobs
CompressionLevel: 3,
AgeRecipients: []string{"age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05gl0sjq9q9wjg"}, // Test public key
AgeRecipients: []string{testAgePublicKey},
})
// Create a snapshot record for testing
ctx := context.Background()
snapshotID := "test-snapshot-001"
err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
snapshot := &database.Snapshot{
ID: types.SnapshotID(snapshotID),
Hostname: "test-host",
VaultikVersion: "test",
StartedAt: time.Now(),
CompletedAt: nil,
FileCount: 0,
ChunkCount: 0,
BlobCount: 0,
TotalSize: 0,
BlobSize: 0,
CompressionRatio: 1.0,
}
return repos.Snapshots.Create(ctx, tx, snapshot)
})
if err != nil {
t.Fatalf("failed to create snapshot: %v", err)
}
createTestSnapshotRecord(ctx, t, repos, snapshotID)
// Scan the directory
var result *snapshot.ScanResult
result, err = scanner.Scan(ctx, "/source", snapshotID)
result, err := scanner.Scan(ctx, "/source", snapshotID)
if err != nil {
t.Fatalf("scan failed: %v", err)
}
@@ -120,58 +205,13 @@ func TestScannerSimpleDirectory(t *testing.T) {
t.Errorf("expected at least 97 bytes scanned, got %d", result.BytesScanned)
}
// Verify files in database - includes regular files and directories
files, err := repos.Files.ListByPrefix(ctx, "/source")
if err != nil {
t.Fatalf("failed to list files: %v", err)
}
// 6 regular files + 3 directories (/source, /source/subdir, /source/subdir2)
if len(files) != 9 {
t.Errorf("expected 9 entries in database (6 files + 3 dirs), got %d", len(files))
}
// Verify specific file
file1, err := repos.Files.GetByPath(ctx, "/source/file1.txt")
if err != nil {
t.Fatalf("failed to get file1.txt: %v", err)
}
if file1.Size != 13 {
t.Errorf("expected file1.txt size 13, got %d", file1.Size)
}
if file1.Mode != 0644 {
t.Errorf("expected file1.txt mode 0644, got %o", file1.Mode)
}
// Verify chunks were created
chunks, err := repos.FileChunks.GetByFile(ctx, "/source/file1.txt")
if err != nil {
t.Fatalf("failed to get chunks for file1.txt: %v", err)
}
if len(chunks) != 1 { // Small file should be one chunk
t.Errorf("expected 1 chunk for file1.txt, got %d", len(chunks))
}
// Verify deduplication - file3.txt and file4.txt have different content
// but we should still have the correct number of unique chunks
allChunks, err := repos.Chunks.List(ctx)
if err != nil {
t.Fatalf("failed to list all chunks: %v", err)
}
// We should have at most 6 chunks (one per unique file content)
// Empty file might not create a chunk
if len(allChunks) > 6 {
t.Errorf("expected at most 6 chunks, got %d", len(allChunks))
}
verifySimpleScanDatabase(ctx, t, repos)
}
func TestScannerLargeFile(t *testing.T) {
// Initialize logger for tests
log.Initialize(log.Config{})
t.Parallel()
// Create in-memory filesystem
fs := afero.NewMemMapFs()
@@ -182,6 +222,7 @@ func TestScannerLargeFile(t *testing.T) {
// Fill with pseudo-random data to ensure chunk boundaries
for i := range largeContent {
// Simple pseudo-random generator for deterministic tests
//nolint:gosec // G115: intentional byte truncation of test data
largeContent[i] = byte((i * 7919) ^ (i >> 3))
}
@@ -216,38 +257,17 @@ func TestScannerLargeFile(t *testing.T) {
Repositories: repos,
MaxBlobSize: int64(1024 * 1024),
CompressionLevel: 3,
AgeRecipients: []string{"age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05gl0sjq9q9wjg"}, // Test public key
AgeRecipients: []string{testAgePublicKey},
})
// Create a snapshot record for testing
ctx := context.Background()
snapshotID := "test-snapshot-001"
err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
snapshot := &database.Snapshot{
ID: types.SnapshotID(snapshotID),
Hostname: "test-host",
VaultikVersion: "test",
StartedAt: time.Now(),
CompletedAt: nil,
FileCount: 0,
ChunkCount: 0,
BlobCount: 0,
TotalSize: 0,
BlobSize: 0,
CompressionRatio: 1.0,
}
return repos.Snapshots.Create(ctx, tx, snapshot)
})
if err != nil {
t.Fatalf("failed to create snapshot: %v", err)
}
createTestSnapshotRecord(ctx, t, repos, snapshotID)
// Scan the directory
var result *snapshot.ScanResult
result, err = scanner.Scan(ctx, "/source", snapshotID)
result, err := scanner.Scan(ctx, "/source", snapshotID)
if err != nil {
t.Fatalf("scan failed: %v", err)
}
@@ -259,7 +279,8 @@ func TestScannerLargeFile(t *testing.T) {
// The file size should be at least 1MB
if result.BytesScanned < 1024*1024 {
t.Errorf("expected at least %d bytes scanned, got %d", 1024*1024, result.BytesScanned)
t.Errorf("expected at least %d bytes scanned, got %d",
1024*1024, result.BytesScanned)
}
// Verify chunks

541
internal/snapshot/snapshot.go
View File

@@ -1,3 +1,6 @@
// Package snapshot implements snapshot creation: scanning source
// directories, chunking and deduplicating file data, packing chunks into
// encrypted blobs, and exporting per-snapshot metadata to remote storage.
package snapshot
// Snapshot Metadata Export Process
@@ -58,6 +61,8 @@ import (
)
// SnapshotManager handles snapshot creation and metadata export
//
//nolint:revive // renaming snapshot.SnapshotManager is a cross-package API change
type SnapshotManager struct {
repos *database.Repositories
storage storage.Storer
@@ -66,6 +71,8 @@ type SnapshotManager struct {
}
// SnapshotManagerParams holds dependencies for NewSnapshotManager
//
//nolint:revive // renaming this alongside SnapshotManager is a cross-package API change
type SnapshotManagerParams struct {
fx.In
@@ -88,15 +95,22 @@ func (sm *SnapshotManager) SetFilesystem(fs afero.Fs) {
sm.fs = fs
}
// CreateSnapshot creates a new snapshot record in the database at the start of a backup.
// CreateSnapshot creates a new snapshot record in the database at the
// start of a backup.
//
// Deprecated: Use CreateSnapshotWithName instead for multi-snapshot support.
func (sm *SnapshotManager) CreateSnapshot(ctx context.Context, hostname, version, gitRevision string) (string, error) {
func (sm *SnapshotManager) CreateSnapshot(
ctx context.Context, hostname, version, gitRevision string,
) (string, error) {
return sm.CreateSnapshotWithName(ctx, hostname, "", version, gitRevision)
}
// CreateSnapshotWithName creates a new snapshot record with an optional snapshot name.
// The snapshot ID format is: hostname_name_timestamp or hostname_timestamp if name is empty.
func (sm *SnapshotManager) CreateSnapshotWithName(ctx context.Context, hostname, name, version, gitRevision string) (string, error) {
// CreateSnapshotWithName creates a new snapshot record with an optional
// snapshot name. The snapshot ID format is: hostname_name_timestamp or
// hostname_timestamp if name is empty.
func (sm *SnapshotManager) CreateSnapshotWithName(
ctx context.Context, hostname, name, version, gitRevision string,
) (string, error) {
// Use short hostname (strip domain if present)
shortHostname := hostname
if before, _, ok := strings.Cut(hostname, "."); ok {
@@ -141,7 +155,9 @@ func (sm *SnapshotManager) CreateSnapshotWithName(ctx context.Context, hostname,
}
// UpdateSnapshotStats updates the statistics for a snapshot during backup
func (sm *SnapshotManager) UpdateSnapshotStats(ctx context.Context, snapshotID string, stats BackupStats) error {
func (sm *SnapshotManager) UpdateSnapshotStats(
ctx context.Context, snapshotID string, stats BackupStats,
) error {
err := sm.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
return sm.repos.Snapshots.UpdateCounts(ctx, tx, snapshotID,
int64(stats.FilesScanned),
@@ -160,7 +176,9 @@ func (sm *SnapshotManager) UpdateSnapshotStats(ctx context.Context, snapshotID s
// UpdateSnapshotStatsExtended updates snapshot statistics with extended metrics.
// This includes compression level, uncompressed blob size, and upload duration.
func (sm *SnapshotManager) UpdateSnapshotStatsExtended(ctx context.Context, snapshotID string, stats ExtendedBackupStats) error {
func (sm *SnapshotManager) UpdateSnapshotStatsExtended(
ctx context.Context, snapshotID string, stats ExtendedBackupStats,
) error {
return sm.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
// First update basic stats
err := sm.repos.Snapshots.UpdateCounts(ctx, tx, snapshotID,
@@ -187,7 +205,9 @@ func (sm *SnapshotManager) UpdateSnapshotStatsExtended(ctx context.Context, snap
// is populated with every blob holding any chunk referenced by the
// snapshot's files (including deduplicated blobs uploaded by prior
// snapshots). Without this, fully-deduplicated snapshots are unrestorable.
func (sm *SnapshotManager) CompleteSnapshot(ctx context.Context, snapshotID string) error {
func (sm *SnapshotManager) CompleteSnapshot(
ctx context.Context, snapshotID string,
) error {
err := sm.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
added, err := sm.repos.Snapshots.PopulateReferencedBlobs(ctx, tx, snapshotID)
if err != nil {
@@ -226,8 +246,11 @@ func (sm *SnapshotManager) CompleteSnapshot(ctx context.Context, snapshotID stri
// - Reopening the main database after this method returns
//
// This ensures database consistency during the copy operation.
func (sm *SnapshotManager) ExportSnapshotMetadata(ctx context.Context, dbPath string, snapshotID string) error {
log.Info("Phase 3/3: Exporting snapshot metadata", "snapshot_id", snapshotID, "source_db", dbPath)
func (sm *SnapshotManager) ExportSnapshotMetadata(
ctx context.Context, dbPath string, snapshotID string,
) error {
log.Info("Phase 3/3: Exporting snapshot metadata",
"snapshot_id", snapshotID, "source_db", dbPath)
// Create temp directory for all temporary files
tempDir, err := afero.TempDir(sm.fs, "", "vaultik-snapshot-*")
@@ -271,13 +294,127 @@ func (sm *SnapshotManager) ExportSnapshotMetadata(ctx context.Context, dbPath st
return nil
}
// prepareExportDB copies, cleans, vacuums, and compresses the snapshot database for export.
// Returns the compressed data and the path to the temporary database (needed for manifest generation).
func (sm *SnapshotManager) prepareExportDB(ctx context.Context, dbPath, snapshotID, tempDir string) ([]byte, string, error) {
// CleanupIncompleteSnapshots removes incomplete snapshots that don't have
// metadata in S3. This is critical for data safety: incomplete snapshots
// can cause deduplication to skip files that were never successfully
// backed up, resulting in data loss.
func (sm *SnapshotManager) CleanupIncompleteSnapshots(
ctx context.Context, hostname string,
) error {
log.Info("Checking for incomplete snapshots", "hostname", hostname)
// Get all incomplete snapshots for this hostname
incompleteSnapshots, err := sm.repos.Snapshots.GetIncompleteByHostname(ctx, hostname)
if err != nil {
return fmt.Errorf("getting incomplete snapshots: %w", err)
}
if len(incompleteSnapshots) == 0 {
log.Debug("No incomplete snapshots found")
return nil
}
log.Info("Found incomplete snapshots", "count", len(incompleteSnapshots))
// Check each incomplete snapshot for metadata in storage
for _, snapshot := range incompleteSnapshots {
// Check if metadata exists in storage (paths use the hashed
// remote key so we don't leak host info to the listing).
metadataKey := fmt.Sprintf("metadata/%s/db.zst",
RemoteSnapshotKey(snapshot.ID.String()))
_, err := sm.storage.Stat(ctx, metadataKey)
if err != nil {
// Metadata doesn't exist in S3 - this is an incomplete snapshot
log.Info("Cleaning up incomplete snapshot record",
"snapshot_id", snapshot.ID, "started_at", snapshot.StartedAt)
// Delete the snapshot and all its associations
err := sm.deleteSnapshot(ctx, snapshot.ID.String())
if err != nil {
return fmt.Errorf("deleting incomplete snapshot %s: %w",
snapshot.ID, err)
}
log.Info("Deleted incomplete snapshot record and associated data",
"snapshot_id", snapshot.ID)
} else {
// Metadata exists - this snapshot was completed but database wasn't updated
// This shouldn't happen in normal operation, but mark it complete
log.Warn("Found snapshot with remote metadata but incomplete in database",
"snapshot_id", snapshot.ID)
err := sm.repos.Snapshots.MarkComplete(ctx, nil, snapshot.ID.String())
if err != nil {
log.Error("Failed to mark snapshot as complete in database",
"snapshot_id", snapshot.ID, "error", err)
}
}
}
return nil
}
// CleanupOrphanedData removes files, chunks, and blobs that are no longer
// referenced by any snapshot. This should be called periodically to clean
// up data from deleted or incomplete snapshots.
func (sm *SnapshotManager) CleanupOrphanedData(ctx context.Context) error {
// Order is important to respect foreign key constraints:
// 1. Delete orphaned files (will cascade delete file_chunks)
// 2. Delete orphaned blobs (will cascade delete blob_chunks for deleted blobs)
// 3. Delete orphaned blob_chunks (where blob exists but chunk doesn't)
// 4. Delete orphaned chunks (now safe after all blob_chunks are gone)
// Delete orphaned files (files not in any snapshot)
log.Debug("Deleting orphaned file records from database")
err := sm.repos.Files.DeleteOrphaned(ctx)
if err != nil {
return fmt.Errorf("deleting orphaned files: %w", err)
}
// Delete orphaned blobs (blobs not in any snapshot)
// This will cascade delete blob_chunks for deleted blobs
log.Debug("Deleting orphaned blob records from database")
err = sm.repos.Blobs.DeleteOrphaned(ctx)
if err != nil {
return fmt.Errorf("deleting orphaned blobs: %w", err)
}
// Delete orphaned blob_chunks entries
// This handles cases where the blob still exists but chunks were deleted
log.Debug("Deleting orphaned blob_chunks associations from database")
err = sm.repos.BlobChunks.DeleteOrphaned(ctx)
if err != nil {
return fmt.Errorf("deleting orphaned blob_chunks: %w", err)
}
// Delete orphaned chunks (chunks not referenced by any file)
// This must come after cleaning up blob_chunks to avoid foreign key violations
log.Debug("Deleting orphaned chunk records from database")
err = sm.repos.Chunks.DeleteOrphaned(ctx)
if err != nil {
return fmt.Errorf("deleting orphaned chunks: %w", err)
}
return nil
}
// prepareExportDB copies, cleans, vacuums, and compresses the snapshot
// database for export. Returns the compressed data and the path to the
// temporary database (needed for manifest generation).
func (sm *SnapshotManager) prepareExportDB(
ctx context.Context, dbPath, snapshotID, tempDir string,
) ([]byte, string, error) {
// Step 1: Copy database to temp file
// The main database should be closed at this point
tempDBPath := filepath.Join(tempDir, "snapshot.db")
log.Debug("Copying database to temporary location", "source", dbPath, "destination", tempDBPath)
log.Debug("Copying database to temporary location",
"source", dbPath, "destination", tempDBPath)
err := sm.copyFile(dbPath, tempDBPath)
if err != nil {
@@ -296,22 +433,24 @@ func (sm *SnapshotManager) prepareExportDB(ctx context.Context, dbPath, snapshot
log.Info("Temporary database cleanup complete",
"db_path", tempDBPath,
"size_after_clean", humanize.Bytes(uint64(sm.getFileSize(tempDBPath))),
"size_after_clean", humanize.Bytes(safeUint64(sm.getFileSize(tempDBPath))),
"files", stats.FileCount,
"chunks", stats.ChunkCount,
"blobs", stats.BlobCount,
"total_compressed_size", humanize.Bytes(uint64(stats.CompressedSize)),
"total_uncompressed_size", humanize.Bytes(uint64(stats.UncompressedSize)),
"compression_ratio", fmt.Sprintf("%.2fx", float64(stats.UncompressedSize)/float64(stats.CompressedSize)))
"total_compressed_size", humanize.Bytes(safeUint64(stats.CompressedSize)),
"total_uncompressed_size", humanize.Bytes(safeUint64(stats.UncompressedSize)),
"compression_ratio", fmt.Sprintf("%.2fx",
float64(stats.UncompressedSize)/float64(stats.CompressedSize)))
// Step 3: VACUUM the database to remove deleted data and compact
// This is critical for security - ensures no stale/deleted data is uploaded
err = sm.vacuumDatabase(tempDBPath)
err = sm.vacuumDatabase(ctx, tempDBPath)
if err != nil {
return nil, "", fmt.Errorf("vacuuming database: %w", err)
}
log.Debug("Database vacuumed", "size", humanize.Bytes(uint64(sm.getFileSize(tempDBPath))))
log.Debug("Database vacuumed",
"size", humanize.Bytes(safeUint64(sm.getFileSize(tempDBPath))))
// Step 4: Compress and encrypt the binary database file
compressedPath := filepath.Join(tempDir, "db.zst.age")
@@ -322,8 +461,8 @@ func (sm *SnapshotManager) prepareExportDB(ctx context.Context, dbPath, snapshot
}
log.Debug("Compression complete",
"original_size", humanize.Bytes(uint64(sm.getFileSize(tempDBPath))),
"compressed_size", humanize.Bytes(uint64(sm.getFileSize(compressedPath))))
"original_size", humanize.Bytes(safeUint64(sm.getFileSize(tempDBPath))),
"compressed_size", humanize.Bytes(safeUint64(sm.getFileSize(compressedPath))))
// Step 5: Read compressed and encrypted data for upload
finalData, err := afero.ReadFile(sm.fs, compressedPath)
@@ -340,7 +479,9 @@ func (sm *SnapshotManager) prepareExportDB(ctx context.Context, dbPath, snapshot
// We never write the human-readable snapshot ID into any unencrypted
// part of remote storage so a listing of the destination bucket leaks
// no host, configuration, or scheduling information.
func (sm *SnapshotManager) uploadSnapshotArtifacts(ctx context.Context, snapshotID string, dbData, manifestData []byte) error {
func (sm *SnapshotManager) uploadSnapshotArtifacts(
ctx context.Context, snapshotID string, dbData, manifestData []byte,
) error {
remoteKey := RemoteSnapshotKey(snapshotID)
// Upload database backup (compressed and encrypted)
@@ -354,7 +495,8 @@ func (sm *SnapshotManager) uploadSnapshotArtifacts(ctx context.Context, snapshot
}
dbUploadDuration := time.Since(dbUploadStart)
dbUploadSpeed := float64(len(dbData)) * 8 / dbUploadDuration.Seconds() // bits per second
// bits per second
dbUploadSpeed := float64(len(dbData)) * bitsPerByte / dbUploadDuration.Seconds()
log.Info("Uploaded snapshot database",
"path", dbKey,
"size", humanize.Bytes(uint64(len(dbData))),
@@ -371,7 +513,9 @@ func (sm *SnapshotManager) uploadSnapshotArtifacts(ctx context.Context, snapshot
}
manifestUploadDuration := time.Since(manifestUploadStart)
manifestUploadSpeed := float64(len(manifestData)) * 8 / manifestUploadDuration.Seconds() // bits per second
// bits per second
manifestUploadSpeed := float64(len(manifestData)) * bitsPerByte /
manifestUploadDuration.Seconds()
log.Info("Uploaded blob manifest",
"path", manifestKey,
"size", humanize.Bytes(uint64(len(manifestData))),
@@ -393,16 +537,19 @@ type CleanupStats struct {
// cleanSnapshotDB removes all data except for the specified snapshot
//
// The cleanup is performed in a specific order to maintain referential integrity:
// 1. Delete other snapshots
// 2. Delete orphaned snapshot associations (snapshot_files, snapshot_blobs) for deleted snapshots
// 3. Delete orphaned files (not in the current snapshot)
// 4. Delete orphaned chunk-to-file mappings (references to deleted files)
// 5. Delete orphaned blobs (not in the current snapshot)
// 6. Delete orphaned blob-to-chunk mappings (references to deleted chunks)
// 7. Delete orphaned chunks (not referenced by any file)
// 1. Delete other snapshots
// 2. Delete orphaned snapshot associations (snapshot_files, snapshot_blobs)
// for deleted snapshots
// 3. Delete orphaned files (not in the current snapshot)
// 4. Delete orphaned chunk-to-file mappings (references to deleted files)
// 5. Delete orphaned blobs (not in the current snapshot)
// 6. Delete orphaned blob-to-chunk mappings (references to deleted chunks)
// 7. Delete orphaned chunks (not referenced by any file)
//
// Each step is implemented as a separate method for clarity and maintainability.
func (sm *SnapshotManager) cleanSnapshotDB(ctx context.Context, dbPath string, snapshotID string) (*CleanupStats, error) {
func (sm *SnapshotManager) cleanSnapshotDB(
ctx context.Context, dbPath string, snapshotID string,
) (*CleanupStats, error) {
// Open the temp database
db, err := database.New(ctx, dbPath)
if err != nil {
@@ -428,39 +575,31 @@ func (sm *SnapshotManager) cleanSnapshotDB(ctx context.Context, dbPath string, s
}()
// Execute cleanup steps in order
err = sm.deleteOtherSnapshots(ctx, tx, snapshotID)
if err != nil {
return nil, fmt.Errorf("step 1 - delete other snapshots: %w", err)
steps := []struct {
name string
fn func() error
}{
{"delete other snapshots",
func() error { return sm.deleteOtherSnapshots(ctx, tx, snapshotID) }},
{"delete orphaned snapshot associations",
func() error { return sm.deleteOrphanedSnapshotAssociations(ctx, tx, snapshotID) }},
{"delete orphaned files",
func() error { return sm.deleteOrphanedFiles(ctx, tx, snapshotID) }},
{"delete orphaned chunk-to-file mappings",
func() error { return sm.deleteOrphanedChunkToFileMappings(ctx, tx) }},
{"delete orphaned blobs",
func() error { return sm.deleteOrphanedBlobs(ctx, tx, snapshotID) }},
{"delete orphaned blob-to-chunk mappings",
func() error { return sm.deleteOrphanedBlobToChunkMappings(ctx, tx) }},
{"delete orphaned chunks",
func() error { return sm.deleteOrphanedChunks(ctx, tx) }},
}
err = sm.deleteOrphanedSnapshotAssociations(ctx, tx, snapshotID)
if err != nil {
return nil, fmt.Errorf("step 2 - delete orphaned snapshot associations: %w", err)
}
err = sm.deleteOrphanedFiles(ctx, tx, snapshotID)
if err != nil {
return nil, fmt.Errorf("step 3 - delete orphaned files: %w", err)
}
err = sm.deleteOrphanedChunkToFileMappings(ctx, tx)
if err != nil {
return nil, fmt.Errorf("step 4 - delete orphaned chunk-to-file mappings: %w", err)
}
err = sm.deleteOrphanedBlobs(ctx, tx, snapshotID)
if err != nil {
return nil, fmt.Errorf("step 5 - delete orphaned blobs: %w", err)
}
err = sm.deleteOrphanedBlobToChunkMappings(ctx, tx)
if err != nil {
return nil, fmt.Errorf("step 6 - delete orphaned blob-to-chunk mappings: %w", err)
}
err = sm.deleteOrphanedChunks(ctx, tx)
if err != nil {
return nil, fmt.Errorf("step 7 - delete orphaned chunks: %w", err)
for i, step := range steps {
err = step.fn()
if err != nil {
return nil, fmt.Errorf("step %d - %s: %w", i+1, step.name, err)
}
}
// Commit transaction
@@ -471,13 +610,19 @@ func (sm *SnapshotManager) cleanSnapshotDB(ctx context.Context, dbPath string, s
return nil, fmt.Errorf("committing transaction: %w", err)
}
// Collect statistics about the cleaned database
return sm.collectCleanupStats(ctx, db, snapshotID)
}
// collectCleanupStats gathers statistics about the cleaned database.
func (sm *SnapshotManager) collectCleanupStats(
ctx context.Context, db *database.DB, snapshotID string,
) (*CleanupStats, error) {
stats := &CleanupStats{}
// Count files
var fileCount int
err = db.QueryRowWithLog(ctx, "SELECT COUNT(*) FROM files").Scan(&fileCount)
err := db.QueryRowWithLog(ctx, "SELECT COUNT(*) FROM files").Scan(&fileCount)
if err != nil {
return nil, fmt.Errorf("counting files: %w", err)
}
@@ -501,9 +646,12 @@ func (sm *SnapshotManager) cleanSnapshotDB(ctx context.Context, dbPath string, s
)
err = db.QueryRowWithLog(ctx, `
SELECT COUNT(*), COALESCE(SUM(compressed_size), 0), COALESCE(SUM(uncompressed_size), 0)
FROM blobs
WHERE blob_hash IN (SELECT blob_hash FROM snapshot_blobs WHERE snapshot_id = ?)
SELECT COUNT(*),
COALESCE(SUM(compressed_size), 0),
COALESCE(SUM(uncompressed_size), 0)
FROM blobs
WHERE blob_hash IN
(SELECT blob_hash FROM snapshot_blobs WHERE snapshot_id = ?)
`, snapshotID).Scan(&blobCount, &compressedSize, &uncompressedSize)
if err != nil {
return nil, fmt.Errorf("counting blobs and sizes: %w", err)
@@ -518,9 +666,10 @@ func (sm *SnapshotManager) cleanSnapshotDB(ctx context.Context, dbPath string, s
// vacuumDatabase runs VACUUM on the database to remove deleted data and compact
// This is critical for security - ensures no stale/deleted data pages are uploaded
func (sm *SnapshotManager) vacuumDatabase(dbPath string) error {
func (sm *SnapshotManager) vacuumDatabase(ctx context.Context, dbPath string) error {
log.Debug("Running VACUUM on database", "path", dbPath)
cmd := exec.Command("sqlite3", dbPath, "VACUUM;")
//nolint:gosec // G204: fixed argv; dbPath is our own temp file path
cmd := exec.CommandContext(ctx, "sqlite3", dbPath, "VACUUM;")
output, err := cmd.CombinedOutput()
if err != nil {
@@ -557,7 +706,8 @@ func (sm *SnapshotManager) compressFile(inputPath, outputPath string) error {
// Use blobgen for compression and encryption
log.Debug("Compressing and encrypting data")
writer, err := blobgen.NewWriter(output, sm.config.CompressionLevel, sm.config.AgeRecipients)
writer, err := blobgen.NewWriter(output, sm.config.CompressionLevel,
sm.config.AgeRecipients)
if err != nil {
return fmt.Errorf("creating blobgen writer: %w", err)
}
@@ -636,7 +786,9 @@ func (sm *SnapshotManager) copyFile(src, dst string) error {
}
// generateBlobManifest creates a compressed JSON list of all blobs in the snapshot
func (sm *SnapshotManager) generateBlobManifest(ctx context.Context, dbPath string, snapshotID string) ([]byte, error) {
func (sm *SnapshotManager) generateBlobManifest(
ctx context.Context, dbPath string, snapshotID string,
) ([]byte, error) {
// Open the cleaned database using the database package
db, err := database.New(ctx, dbPath)
if err != nil {
@@ -734,61 +886,10 @@ type ExtendedBackupStats struct {
UploadDurationMs int64 // Total milliseconds spent uploading to S3
}
// CleanupIncompleteSnapshots removes incomplete snapshots that don't have metadata in S3.
// This is critical for data safety: incomplete snapshots can cause deduplication to skip
// files that were never successfully backed up, resulting in data loss.
func (sm *SnapshotManager) CleanupIncompleteSnapshots(ctx context.Context, hostname string) error {
log.Info("Checking for incomplete snapshots", "hostname", hostname)
// Get all incomplete snapshots for this hostname
incompleteSnapshots, err := sm.repos.Snapshots.GetIncompleteByHostname(ctx, hostname)
if err != nil {
return fmt.Errorf("getting incomplete snapshots: %w", err)
}
if len(incompleteSnapshots) == 0 {
log.Debug("No incomplete snapshots found")
return nil
}
log.Info("Found incomplete snapshots", "count", len(incompleteSnapshots))
// Check each incomplete snapshot for metadata in storage
for _, snapshot := range incompleteSnapshots {
// Check if metadata exists in storage (paths use the hashed
// remote key so we don't leak host info to the listing).
metadataKey := fmt.Sprintf("metadata/%s/db.zst", RemoteSnapshotKey(snapshot.ID.String()))
_, err := sm.storage.Stat(ctx, metadataKey)
if err != nil {
// Metadata doesn't exist in S3 - this is an incomplete snapshot
log.Info("Cleaning up incomplete snapshot record", "snapshot_id", snapshot.ID, "started_at", snapshot.StartedAt)
// Delete the snapshot and all its associations
err := sm.deleteSnapshot(ctx, snapshot.ID.String())
if err != nil {
return fmt.Errorf("deleting incomplete snapshot %s: %w", snapshot.ID, err)
}
log.Info("Deleted incomplete snapshot record and associated data", "snapshot_id", snapshot.ID)
} else {
// Metadata exists - this snapshot was completed but database wasn't updated
// This shouldn't happen in normal operation, but mark it complete
log.Warn("Found snapshot with remote metadata but incomplete in database", "snapshot_id", snapshot.ID)
err := sm.repos.Snapshots.MarkComplete(ctx, nil, snapshot.ID.String())
if err != nil {
log.Error("Failed to mark snapshot as complete in database", "snapshot_id", snapshot.ID, "error", err)
}
}
}
return nil
}
// deleteSnapshot removes a snapshot and all its associations from the database
func (sm *SnapshotManager) deleteSnapshot(ctx context.Context, snapshotID string) error {
func (sm *SnapshotManager) deleteSnapshot(
ctx context.Context, snapshotID string,
) error {
// Delete snapshot_files entries
err := sm.repos.Snapshots.DeleteSnapshotFiles(ctx, snapshotID)
if err != nil {
@@ -824,61 +925,20 @@ func (sm *SnapshotManager) deleteSnapshot(ctx context.Context, snapshotID string
return nil
}
// CleanupOrphanedData removes files, chunks, and blobs that are no longer referenced by any snapshot.
// This should be called periodically to clean up data from deleted or incomplete snapshots.
func (sm *SnapshotManager) CleanupOrphanedData(ctx context.Context) error {
// Order is important to respect foreign key constraints:
// 1. Delete orphaned files (will cascade delete file_chunks)
// 2. Delete orphaned blobs (will cascade delete blob_chunks for deleted blobs)
// 3. Delete orphaned blob_chunks (where blob exists but chunk doesn't)
// 4. Delete orphaned chunks (now safe after all blob_chunks are gone)
// Delete orphaned files (files not in any snapshot)
log.Debug("Deleting orphaned file records from database")
err := sm.repos.Files.DeleteOrphaned(ctx)
if err != nil {
return fmt.Errorf("deleting orphaned files: %w", err)
}
// Delete orphaned blobs (blobs not in any snapshot)
// This will cascade delete blob_chunks for deleted blobs
log.Debug("Deleting orphaned blob records from database")
err = sm.repos.Blobs.DeleteOrphaned(ctx)
if err != nil {
return fmt.Errorf("deleting orphaned blobs: %w", err)
}
// Delete orphaned blob_chunks entries
// This handles cases where the blob still exists but chunks were deleted
log.Debug("Deleting orphaned blob_chunks associations from database")
err = sm.repos.BlobChunks.DeleteOrphaned(ctx)
if err != nil {
return fmt.Errorf("deleting orphaned blob_chunks: %w", err)
}
// Delete orphaned chunks (chunks not referenced by any file)
// This must come after cleaning up blob_chunks to avoid foreign key violations
log.Debug("Deleting orphaned chunk records from database")
err = sm.repos.Chunks.DeleteOrphaned(ctx)
if err != nil {
return fmt.Errorf("deleting orphaned chunks: %w", err)
}
return nil
}
// deleteOtherSnapshots deletes all snapshots except the current one
func (sm *SnapshotManager) deleteOtherSnapshots(ctx context.Context, tx *sql.Tx, currentSnapshotID string) error {
log.Debug("[Temp DB Cleanup] Deleting all snapshot records except current", "keeping", currentSnapshotID)
func (sm *SnapshotManager) deleteOtherSnapshots(
ctx context.Context, tx *sql.Tx, currentSnapshotID string,
) error {
log.Debug("[Temp DB Cleanup] Deleting all snapshot records except current",
"keeping", currentSnapshotID)
// First delete uploads that reference other snapshots (no CASCADE DELETE on this FK)
database.LogSQL("Execute", "DELETE FROM uploads WHERE snapshot_id != ?", currentSnapshotID)
// First delete uploads that reference other snapshots (no CASCADE DELETE
// on this FK)
database.LogSQL("Execute", "DELETE FROM uploads WHERE snapshot_id != ?",
currentSnapshotID)
uploadResult, err := tx.ExecContext(ctx, "DELETE FROM uploads WHERE snapshot_id != ?", currentSnapshotID)
uploadResult, err := tx.ExecContext(ctx,
"DELETE FROM uploads WHERE snapshot_id != ?", currentSnapshotID)
if err != nil {
return fmt.Errorf("deleting uploads for other snapshots: %w", err)
}
@@ -887,66 +947,84 @@ func (sm *SnapshotManager) deleteOtherSnapshots(ctx context.Context, tx *sql.Tx,
log.Debug("[Temp DB Cleanup] Deleted upload records", "count", uploadsDeleted)
// Now we can safely delete the snapshots
database.LogSQL("Execute", "DELETE FROM snapshots WHERE id != ?", currentSnapshotID)
database.LogSQL("Execute", "DELETE FROM snapshots WHERE id != ?",
currentSnapshotID)
result, err := tx.ExecContext(ctx, "DELETE FROM snapshots WHERE id != ?", currentSnapshotID)
result, err := tx.ExecContext(ctx, "DELETE FROM snapshots WHERE id != ?",
currentSnapshotID)
if err != nil {
return fmt.Errorf("deleting other snapshots: %w", err)
}
rowsAffected, _ := result.RowsAffected()
log.Debug("[Temp DB Cleanup] Deleted snapshot records from database", "count", rowsAffected)
log.Debug("[Temp DB Cleanup] Deleted snapshot records from database",
"count", rowsAffected)
return nil
}
// deleteOrphanedSnapshotAssociations deletes snapshot_files and snapshot_blobs for deleted snapshots
func (sm *SnapshotManager) deleteOrphanedSnapshotAssociations(ctx context.Context, tx *sql.Tx, currentSnapshotID string) error {
// deleteOrphanedSnapshotAssociations deletes snapshot_files and
// snapshot_blobs for deleted snapshots
func (sm *SnapshotManager) deleteOrphanedSnapshotAssociations(
ctx context.Context, tx *sql.Tx, currentSnapshotID string,
) error {
// Delete orphaned snapshot_files
log.Debug("[Temp DB Cleanup] Deleting orphaned snapshot_files associations")
database.LogSQL("Execute", "DELETE FROM snapshot_files WHERE snapshot_id != ?", currentSnapshotID)
database.LogSQL("Execute", "DELETE FROM snapshot_files WHERE snapshot_id != ?",
currentSnapshotID)
result, err := tx.ExecContext(ctx, "DELETE FROM snapshot_files WHERE snapshot_id != ?", currentSnapshotID)
result, err := tx.ExecContext(ctx,
"DELETE FROM snapshot_files WHERE snapshot_id != ?", currentSnapshotID)
if err != nil {
return fmt.Errorf("deleting orphaned snapshot_files: %w", err)
}
rowsAffected, _ := result.RowsAffected()
log.Debug("[Temp DB Cleanup] Deleted snapshot_files associations", "count", rowsAffected)
log.Debug("[Temp DB Cleanup] Deleted snapshot_files associations",
"count", rowsAffected)
// Delete orphaned snapshot_blobs
log.Debug("[Temp DB Cleanup] Deleting orphaned snapshot_blobs associations")
database.LogSQL("Execute", "DELETE FROM snapshot_blobs WHERE snapshot_id != ?", currentSnapshotID)
database.LogSQL("Execute", "DELETE FROM snapshot_blobs WHERE snapshot_id != ?",
currentSnapshotID)
result, err = tx.ExecContext(ctx, "DELETE FROM snapshot_blobs WHERE snapshot_id != ?", currentSnapshotID)
result, err = tx.ExecContext(ctx,
"DELETE FROM snapshot_blobs WHERE snapshot_id != ?", currentSnapshotID)
if err != nil {
return fmt.Errorf("deleting orphaned snapshot_blobs: %w", err)
}
rowsAffected, _ = result.RowsAffected()
log.Debug("[Temp DB Cleanup] Deleted snapshot_blobs associations", "count", rowsAffected)
log.Debug("[Temp DB Cleanup] Deleted snapshot_blobs associations",
"count", rowsAffected)
return nil
}
// deleteOrphanedFiles deletes files not in the current snapshot
func (sm *SnapshotManager) deleteOrphanedFiles(ctx context.Context, tx *sql.Tx, currentSnapshotID string) error {
log.Debug("[Temp DB Cleanup] Deleting file records not referenced by current snapshot")
database.LogSQL("Execute", `DELETE FROM files WHERE NOT EXISTS (SELECT 1 FROM snapshot_files WHERE snapshot_files.file_id = files.id AND snapshot_files.snapshot_id = ?)`, currentSnapshotID)
func (sm *SnapshotManager) deleteOrphanedFiles(
ctx context.Context, tx *sql.Tx, currentSnapshotID string,
) error {
log.Debug(
"[Temp DB Cleanup] Deleting file records not referenced by current snapshot")
result, err := tx.ExecContext(ctx, `
DELETE FROM files
query := `
DELETE FROM files
WHERE NOT EXISTS (
SELECT 1 FROM snapshot_files
WHERE snapshot_files.file_id = files.id
SELECT 1 FROM snapshot_files
WHERE snapshot_files.file_id = files.id
AND snapshot_files.snapshot_id = ?
)`, currentSnapshotID)
)`
database.LogSQL("Execute", query, currentSnapshotID)
result, err := tx.ExecContext(ctx, query, currentSnapshotID)
if err != nil {
return fmt.Errorf("deleting orphaned files: %w", err)
}
rowsAffected, _ := result.RowsAffected()
log.Debug("[Temp DB Cleanup] Deleted file records from database", "count", rowsAffected)
log.Debug("[Temp DB Cleanup] Deleted file records from database",
"count", rowsAffected)
// Note: file_chunks will be deleted via CASCADE
log.Debug("[Temp DB Cleanup] file_chunks associations deleted via CASCADE")
@@ -955,65 +1033,81 @@ func (sm *SnapshotManager) deleteOrphanedFiles(ctx context.Context, tx *sql.Tx,
}
// deleteOrphanedChunkToFileMappings deletes chunk_files entries for deleted files
func (sm *SnapshotManager) deleteOrphanedChunkToFileMappings(ctx context.Context, tx *sql.Tx) error {
func (sm *SnapshotManager) deleteOrphanedChunkToFileMappings(
ctx context.Context, tx *sql.Tx,
) error {
log.Debug("[Temp DB Cleanup] Deleting orphaned chunk_files associations")
database.LogSQL("Execute", `DELETE FROM chunk_files WHERE NOT EXISTS (SELECT 1 FROM files WHERE files.id = chunk_files.file_id)`)
result, err := tx.ExecContext(ctx, `
DELETE FROM chunk_files
query := `
DELETE FROM chunk_files
WHERE NOT EXISTS (
SELECT 1 FROM files
SELECT 1 FROM files
WHERE files.id = chunk_files.file_id
)`)
)`
database.LogSQL("Execute", query)
result, err := tx.ExecContext(ctx, query)
if err != nil {
return fmt.Errorf("deleting orphaned chunk_files: %w", err)
}
rowsAffected, _ := result.RowsAffected()
log.Debug("[Temp DB Cleanup] Deleted chunk_files associations", "count", rowsAffected)
log.Debug("[Temp DB Cleanup] Deleted chunk_files associations",
"count", rowsAffected)
return nil
}
// deleteOrphanedBlobs deletes blobs not in the current snapshot
func (sm *SnapshotManager) deleteOrphanedBlobs(ctx context.Context, tx *sql.Tx, currentSnapshotID string) error {
log.Debug("[Temp DB Cleanup] Deleting blob records not referenced by current snapshot")
database.LogSQL("Execute", `DELETE FROM blobs WHERE NOT EXISTS (SELECT 1 FROM snapshot_blobs WHERE snapshot_blobs.blob_hash = blobs.blob_hash AND snapshot_blobs.snapshot_id = ?)`, currentSnapshotID)
func (sm *SnapshotManager) deleteOrphanedBlobs(
ctx context.Context, tx *sql.Tx, currentSnapshotID string,
) error {
log.Debug(
"[Temp DB Cleanup] Deleting blob records not referenced by current snapshot")
result, err := tx.ExecContext(ctx, `
DELETE FROM blobs
query := `
DELETE FROM blobs
WHERE NOT EXISTS (
SELECT 1 FROM snapshot_blobs
WHERE snapshot_blobs.blob_hash = blobs.blob_hash
SELECT 1 FROM snapshot_blobs
WHERE snapshot_blobs.blob_hash = blobs.blob_hash
AND snapshot_blobs.snapshot_id = ?
)`, currentSnapshotID)
)`
database.LogSQL("Execute", query, currentSnapshotID)
result, err := tx.ExecContext(ctx, query, currentSnapshotID)
if err != nil {
return fmt.Errorf("deleting orphaned blobs: %w", err)
}
rowsAffected, _ := result.RowsAffected()
log.Debug("[Temp DB Cleanup] Deleted blob records from database", "count", rowsAffected)
log.Debug("[Temp DB Cleanup] Deleted blob records from database",
"count", rowsAffected)
return nil
}
// deleteOrphanedBlobToChunkMappings deletes blob_chunks entries for deleted blobs
func (sm *SnapshotManager) deleteOrphanedBlobToChunkMappings(ctx context.Context, tx *sql.Tx) error {
func (sm *SnapshotManager) deleteOrphanedBlobToChunkMappings(
ctx context.Context, tx *sql.Tx,
) error {
log.Debug("[Temp DB Cleanup] Deleting orphaned blob_chunks associations")
database.LogSQL("Execute", `DELETE FROM blob_chunks WHERE NOT EXISTS (SELECT 1 FROM blobs WHERE blobs.id = blob_chunks.blob_id)`)
result, err := tx.ExecContext(ctx, `
DELETE FROM blob_chunks
query := `
DELETE FROM blob_chunks
WHERE NOT EXISTS (
SELECT 1 FROM blobs
SELECT 1 FROM blobs
WHERE blobs.id = blob_chunks.blob_id
)`)
)`
database.LogSQL("Execute", query)
result, err := tx.ExecContext(ctx, query)
if err != nil {
return fmt.Errorf("deleting orphaned blob_chunks: %w", err)
}
rowsAffected, _ := result.RowsAffected()
log.Debug("[Temp DB Cleanup] Deleted blob_chunks associations", "count", rowsAffected)
log.Debug("[Temp DB Cleanup] Deleted blob_chunks associations",
"count", rowsAffected)
return nil
}
@@ -1040,7 +1134,8 @@ func (sm *SnapshotManager) deleteOrphanedChunks(ctx context.Context, tx *sql.Tx)
}
rowsAffected, _ := result.RowsAffected()
log.Debug("[Temp DB Cleanup] Deleted chunk records from database", "count", rowsAffected)
log.Debug("[Temp DB Cleanup] Deleted chunk records from database",
"count", rowsAffected)
return nil
}

101
internal/snapshot/snapshot_test.go
View File

@@ -1,3 +1,4 @@
//nolint:testpackage // exercises unexported SnapshotManager internals
package snapshot
import (
@@ -37,9 +38,65 @@ func copyFile(fs afero.Fs, src, dst string) error {
return err
}
// verifyCleanedDB opens the cleaned database and checks that the kept
// snapshot survived while the orphan file and chunk were removed.
func verifyCleanedDB(
ctx context.Context,
t *testing.T,
tempDBPath, snapshotID string,
file *database.File,
chunk *database.Chunk,
) {
t.Helper()
cleanedDB, err := database.New(ctx, tempDBPath)
if err != nil {
t.Fatalf("failed to open cleaned database: %v", err)
}
defer func() {
err := cleanedDB.Close()
if err != nil {
t.Errorf("failed to close database: %v", err)
}
}()
cleanedRepos := database.NewRepositories(cleanedDB)
// Verify snapshot exists
verifySnapshot, err := cleanedRepos.Snapshots.GetByID(ctx, snapshotID)
if err != nil {
t.Fatalf("failed to get snapshot: %v", err)
}
if verifySnapshot == nil {
t.Error("snapshot should exist")
}
// Verify orphan file is gone
f, err := cleanedRepos.Files.GetByPath(ctx, file.Path.String())
if err != nil {
t.Fatalf("failed to check file: %v", err)
}
if f != nil {
t.Error("orphan file should not exist")
}
// Verify orphan chunk is gone
c, err := cleanedRepos.Chunks.GetByHash(ctx, chunk.ChunkHash.String())
if err != nil {
t.Fatalf("failed to check chunk: %v", err)
}
if c != nil {
t.Error("orphan chunk should not exist")
}
}
func TestCleanSnapshotDBEmptySnapshot(t *testing.T) {
// Initialize logger
log.Initialize(log.Config{})
t.Parallel()
ctx := context.Background()
fs := afero.NewOsFs()
@@ -115,53 +172,13 @@ func TestCleanSnapshotDBEmptySnapshot(t *testing.T) {
}
// Verify the cleaned database
cleanedDB, err := database.New(ctx, tempDBPath)
if err != nil {
t.Fatalf("failed to open cleaned database: %v", err)
}
defer func() {
err := cleanedDB.Close()
if err != nil {
t.Errorf("failed to close database: %v", err)
}
}()
cleanedRepos := database.NewRepositories(cleanedDB)
// Verify snapshot exists
verifySnapshot, err := cleanedRepos.Snapshots.GetByID(ctx, snapshot.ID.String())
if err != nil {
t.Fatalf("failed to get snapshot: %v", err)
}
if verifySnapshot == nil {
t.Error("snapshot should exist")
}
// Verify orphan file is gone
f, err := cleanedRepos.Files.GetByPath(ctx, file.Path.String())
if err != nil {
t.Fatalf("failed to check file: %v", err)
}
if f != nil {
t.Error("orphan file should not exist")
}
// Verify orphan chunk is gone
c, err := cleanedRepos.Chunks.GetByHash(ctx, chunk.ChunkHash.String())
if err != nil {
t.Fatalf("failed to check chunk: %v", err)
}
if c != nil {
t.Error("orphan chunk should not exist")
}
verifyCleanedDB(ctx, t, tempDBPath, snapshot.ID.String(), file, chunk)
}
func TestCleanSnapshotDBNonExistentSnapshot(t *testing.T) {
// Initialize logger
log.Initialize(log.Config{})
t.Parallel()
ctx := context.Background()
fs := afero.NewOsFs()