sneak/vaultik
1
0
Fork 0
Code Issues 19 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Remediate all lint findings under the canonical golangci-lint config

Browse Source 
Fix every finding surfaced by the canonical .golangci.yml with
golangci-lint v2.12.2 (refs #61), behavior-preserving throughout:

- err113: dynamic errors replaced with package-level sentinels and %w
  wrapping; direct comparisons converted to errors.Is
- goprintffuncname: printf-style helpers renamed with an f suffix
  (ui.Writer message methods, cli.ReportErrorf, database.Fatalf,
  vaultik stdoutf) and all call sites updated
- revive: stuttering type names renamed (blob.Handler, blob.WithReader,
  blob.ChunkPosition, storage.URL, storage.Info), doc comments added,
  unused parameters blanked, package comments added
- contextcheck/noctx: ctx threaded through blob.Packer
  (AddChunk/Flush/FinalizeBlob/PackChunks) and scanner call sites;
  context-aware exec and sql variants used
- funlen/cyclop/gocognit/nestif/dupl: oversized or duplicated
  functions split into focused helpers across production and test code
- paralleltest/tparallel/thelper/usetesting/testpackage: tests
  parallelized where safe (global log.Initialize kept in the serial
  phase), helpers marked, t.TempDir adopted, external test packages
  where only exported API is used
- gosec: integer conversions clamped or justified, header timeouts
  added, remaining findings suppressed with per-site justifications
- mnd/goconst/lll/wsl_v5/nlreturn/noinlineerr/errcheck and other
  mechanical findings fixed directly

Remove the deprecated log.LogOptions alias (callers migrated to
log.Options). make check is green.
This commit is contained in:
Jeffrey Paul
2026-08-07 18:51:21 +00:00
parent 6cf9211407
commit 7ae470e530
121 changed files with 8344 additions and 5406 deletions
Download Patch File Download Diff File Expand all files Collapse all files

524
internal/snapshot/backup_test.go
View File

@@ -1,4 +1,4 @@
package snapshot
package snapshot_test
import (
"context"
@@ -19,6 +19,12 @@ import (
"sneak.berlin/go/vaultik/internal/types"
)
// errBlobNotFound is returned by the mock S3 client for unknown hashes.
var errBlobNotFound = errors.New("blob not found")
// testFile1Name is the shared fixture filename used across backup tests.
const testFile1Name = "file1.txt"
// MockS3Client is a mock implementation of S3 operations for testing
type MockS3Client struct {
storage map[string][]byte
@@ -30,39 +36,149 @@ func NewMockS3Client() *MockS3Client {
}
}
func (m *MockS3Client) PutBlob(ctx context.Context, hash string, data []byte) error {
func (m *MockS3Client) PutBlob(_ context.Context, hash string, data []byte) error {
m.storage[hash] = data
return nil
}
func (m *MockS3Client) GetBlob(ctx context.Context, hash string) ([]byte, error) {
func (m *MockS3Client) GetBlob(_ context.Context, hash string) ([]byte, error) {
data, ok := m.storage[hash]
if !ok {
return nil, fmt.Errorf("blob not found: %s", hash)
return nil, fmt.Errorf("%w: %s", errBlobNotFound, hash)
}
return data, nil
}
func (m *MockS3Client) BlobExists(ctx context.Context, hash string) (bool, error) {
func (m *MockS3Client) BlobExists(_ context.Context, hash string) (bool, error) {
_, ok := m.storage[hash]
return ok, nil
}
func (m *MockS3Client) CreateBucket(ctx context.Context, bucket string) error {
func (m *MockS3Client) CreateBucket(_ context.Context, _ string) error {
return nil
}
// verifyBackupFiles checks the file records created by a backup against
// the fixture filesystem.
func verifyBackupFiles(
ctx context.Context,
t *testing.T,
repos *database.Repositories,
testFS fstest.MapFS,
) {
t.Helper()
files, err := repos.Files.ListByPrefix(ctx, "")
if err != nil {
t.Fatalf("Failed to list files: %v", err)
}
expectedFiles := map[string]bool{
testFile1Name: true,
"dir1/file2.txt": true,
"dir1/subdir/file3.txt": true,
"largefile.bin": true,
}
if len(files) != len(expectedFiles) {
t.Errorf("Expected %d files, got %d", len(expectedFiles), len(files))
}
for _, file := range files {
if !expectedFiles[file.Path.String()] {
t.Errorf("Unexpected file in database: %s", file.Path)
}
delete(expectedFiles, file.Path.String())
// Verify file metadata
fsFile := testFS[file.Path.String()]
if fsFile == nil {
t.Errorf("File %s not found in test filesystem", file.Path)
continue
}
if file.Size != int64(len(fsFile.Data)) {
t.Errorf("File %s: expected size %d, got %d",
file.Path, len(fsFile.Data), file.Size)
}
if file.Mode != uint32(fsFile.Mode) {
t.Errorf("File %s: expected mode %o, got %o",
file.Path, fsFile.Mode, file.Mode)
}
}
if len(expectedFiles) > 0 {
t.Errorf("Files not found in database: %v", expectedFiles)
}
}
// verifyBackupChunksAndBlobs checks that chunking produced the expected
// records and every referenced blob exists in the mock S3 store.
func verifyBackupChunksAndBlobs(
ctx context.Context,
t *testing.T,
repos *database.Repositories,
s3Client *MockS3Client,
snapshotID string,
) {
t.Helper()
chunks, err := repos.Chunks.List(ctx)
if err != nil {
t.Fatalf("Failed to list chunks: %v", err)
}
if len(chunks) == 0 {
t.Error("No chunks found in database")
}
// The large file should create 10 chunks (10MB / 1MB chunk size)
// Plus the small files
minExpectedChunks := 10 + 3
if len(chunks) < minExpectedChunks {
t.Errorf("Expected at least %d chunks, got %d", minExpectedChunks, len(chunks))
}
// Verify at least one blob was created and uploaded
// We can't list blobs directly, but we can check via snapshot blobs
blobHashes, err := repos.Snapshots.GetBlobHashes(ctx, snapshotID)
if err != nil {
t.Fatalf("Failed to get blob hashes: %v", err)
}
if len(blobHashes) == 0 {
t.Error("Expected at least one blob to be created")
}
for _, blobHash := range blobHashes {
// Check blob exists in mock S3
exists, err := s3Client.BlobExists(ctx, blobHash)
if err != nil {
t.Errorf("Failed to check blob %s: %v", blobHash, err)
}
if !exists {
t.Errorf("Blob %s not found in S3", blobHash)
}
}
}
func TestBackupWithInMemoryFS(t *testing.T) {
t.Parallel()
// Create a temporary directory for the database
tempDir := t.TempDir()
dbPath := filepath.Join(tempDir, "test.db")
// Create test filesystem
testFS := fstest.MapFS{
"file1.txt": &fstest.MapFile{
testFile1Name: &fstest.MapFile{
Data: []byte("Hello, World!"),
Mode: 0644,
ModTime: time.Now(),
@@ -129,100 +245,21 @@ func TestBackupWithInMemoryFS(t *testing.T) {
t.Error("Expected snapshot to have files")
}
// Verify files in database
files, err := repos.Files.ListByPrefix(ctx, "")
if err != nil {
t.Fatalf("Failed to list files: %v", err)
}
expectedFiles := map[string]bool{
"file1.txt": true,
"dir1/file2.txt": true,
"dir1/subdir/file3.txt": true,
"largefile.bin": true,
}
if len(files) != len(expectedFiles) {
t.Errorf("Expected %d files, got %d", len(expectedFiles), len(files))
}
for _, file := range files {
if !expectedFiles[file.Path.String()] {
t.Errorf("Unexpected file in database: %s", file.Path)
}
delete(expectedFiles, file.Path.String())
// Verify file metadata
fsFile := testFS[file.Path.String()]
if fsFile == nil {
t.Errorf("File %s not found in test filesystem", file.Path)
continue
}
if file.Size != int64(len(fsFile.Data)) {
t.Errorf("File %s: expected size %d, got %d", file.Path, len(fsFile.Data), file.Size)
}
if file.Mode != uint32(fsFile.Mode) {
t.Errorf("File %s: expected mode %o, got %o", file.Path, fsFile.Mode, file.Mode)
}
}
if len(expectedFiles) > 0 {
t.Errorf("Files not found in database: %v", expectedFiles)
}
// Verify chunks
chunks, err := repos.Chunks.List(ctx)
if err != nil {
t.Fatalf("Failed to list chunks: %v", err)
}
if len(chunks) == 0 {
t.Error("No chunks found in database")
}
// The large file should create 10 chunks (10MB / 1MB chunk size)
// Plus the small files
minExpectedChunks := 10 + 3
if len(chunks) < minExpectedChunks {
t.Errorf("Expected at least %d chunks, got %d", minExpectedChunks, len(chunks))
}
// Verify at least one blob was created and uploaded
// We can't list blobs directly, but we can check via snapshot blobs
blobHashes, err := repos.Snapshots.GetBlobHashes(ctx, snapshotID)
if err != nil {
t.Fatalf("Failed to get blob hashes: %v", err)
}
if len(blobHashes) == 0 {
t.Error("Expected at least one blob to be created")
}
for _, blobHash := range blobHashes {
// Check blob exists in mock S3
exists, err := s3Client.BlobExists(ctx, blobHash)
if err != nil {
t.Errorf("Failed to check blob %s: %v", blobHash, err)
}
if !exists {
t.Errorf("Blob %s not found in S3", blobHash)
}
}
// Verify files, chunks, and blob records
verifyBackupFiles(ctx, t, repos, testFS)
verifyBackupChunksAndBlobs(ctx, t, repos, s3Client, snapshotID)
}
func TestBackupDeduplication(t *testing.T) {
t.Parallel()
// Create a temporary directory for the database
tempDir := t.TempDir()
dbPath := filepath.Join(tempDir, "test.db")
// Create test filesystem with duplicate content
testFS := fstest.MapFS{
"file1.txt": &fstest.MapFile{
testFile1Name: &fstest.MapFile{
Data: []byte("Duplicate content"),
Mode: 0644,
ModTime: time.Now(),
@@ -290,7 +327,8 @@ func TestBackupDeduplication(t *testing.T) {
// The duplicate content chunk should be referenced by 2 files
if chunk.Size == int64(len("Duplicate content")) && len(files) != 2 {
t.Errorf("Expected duplicate chunk to be referenced by 2 files, got %d", len(files))
t.Errorf("Expected duplicate chunk to be referenced by 2 files, got %d",
len(files))
}
}
}
@@ -304,8 +342,19 @@ type BackupEngine struct {
}
}
// backupCounters accumulates statistics across a test backup run.
type backupCounters struct {
fileCount int64
chunkCount int64
blobCount int64
totalSize int64
blobSize int64
}
// Backup performs a backup of the given filesystem
func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (string, error) {
func (b *BackupEngine) Backup(
ctx context.Context, fsys fs.FS, root string,
) (string, error) {
// Create a new snapshot
hostname, _ := os.Hostname()
snapshotID := time.Now().Format(time.RFC3339)
@@ -325,8 +374,7 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
return "", err
}
// Track counters
var fileCount, chunkCount, blobCount, totalSize, blobSize int64
counters := &backupCounters{}
// Track which chunks we've seen to handle deduplication
processedChunks := make(map[string]bool)
@@ -354,122 +402,170 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
return nil
}
// Create file record in a short transaction
file := &database.File{
Path: types.FilePath(path),
Size: info.Size(),
Mode: uint32(info.Mode()),
MTime: info.ModTime(),
UID: 1000, // Default UID for test
GID: 1000, // Default GID for test
}
err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
return b.repos.Files.Create(ctx, tx, file)
})
if err != nil {
return err
}
fileCount++
totalSize += info.Size()
// Read and process file in chunks
f, err := fsys.Open(path)
if err != nil {
return err
}
defer func() {
err := f.Close()
if err != nil {
// Log but don't fail since we're already in an error path potentially
fmt.Fprintf(os.Stderr, "Failed to close file: %v\n", err)
}
}()
// Process file in chunks
chunkIndex := 0
buffer := make([]byte, defaultChunkSize)
for {
n, err := f.Read(buffer)
if err != nil && !errors.Is(err, io.EOF) {
return err
}
if n == 0 {
break
}
chunkData := buffer[:n]
chunkHash := calculateHash(chunkData)
// Check if chunk already exists (outside of transaction)
existingChunk, _ := b.repos.Chunks.GetByHash(ctx, chunkHash)
if existingChunk == nil {
// Create new chunk in a short transaction
err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
chunk := &database.Chunk{
ChunkHash: types.ChunkHash(chunkHash),
Size: int64(n),
}
return b.repos.Chunks.Create(ctx, tx, chunk)
})
if err != nil {
return err
}
processedChunks[chunkHash] = true
}
// Create file-chunk mapping in a short transaction
err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
fileChunk := &database.FileChunk{
FileID: file.ID,
Idx: chunkIndex,
ChunkHash: types.ChunkHash(chunkHash),
}
return b.repos.FileChunks.Create(ctx, tx, fileChunk)
})
if err != nil {
return err
}
// Create chunk-file mapping in a short transaction
err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
chunkFile := &database.ChunkFile{
ChunkHash: types.ChunkHash(chunkHash),
FileID: file.ID,
FileOffset: int64(chunkIndex * defaultChunkSize),
Length: int64(n),
}
return b.repos.ChunkFiles.Create(ctx, tx, chunkFile)
})
if err != nil {
return err
}
chunkIndex++
}
return nil
return b.backupOneFile(ctx, fsys, path, info, processedChunks, counters)
})
if err != nil {
return "", err
}
// After all files are processed, create blobs for new chunks
err = b.createBlobsForChunks(ctx, snapshotID, processedChunks, counters)
if err != nil {
return "", err
}
// Update snapshot with final counts
err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
return b.repos.Snapshots.UpdateCounts(ctx, tx, snapshotID,
counters.fileCount, counters.chunkCount, counters.blobCount,
counters.totalSize, counters.blobSize)
})
if err != nil {
return "", err
}
return snapshotID, nil
}
// backupOneFile records a single regular file and its chunks.
func (b *BackupEngine) backupOneFile(
ctx context.Context,
fsys fs.FS,
path string,
info fs.FileInfo,
processedChunks map[string]bool,
counters *backupCounters,
) error {
// Create file record in a short transaction
file := &database.File{
Path: types.FilePath(path),
Size: info.Size(),
Mode: uint32(info.Mode()),
MTime: info.ModTime(),
UID: 1000, // Default UID for test
GID: 1000, // Default GID for test
}
err := b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
return b.repos.Files.Create(ctx, tx, file)
})
if err != nil {
return err
}
counters.fileCount++
counters.totalSize += info.Size()
// Read and process file in chunks
f, err := fsys.Open(path)
if err != nil {
return err
}
defer func() {
err := f.Close()
if err != nil {
// Log but don't fail since we're already in an error path potentially
fmt.Fprintf(os.Stderr, "Failed to close file: %v\n", err)
}
}()
// Process file in chunks
chunkIndex := 0
buffer := make([]byte, defaultChunkSize)
for {
n, err := f.Read(buffer)
if err != nil && !errors.Is(err, io.EOF) {
return err
}
if n == 0 {
break
}
err = b.recordChunk(ctx, file, chunkIndex, buffer[:n], processedChunks)
if err != nil {
return err
}
chunkIndex++
}
return nil
}
// recordChunk creates the chunk record (if new) and its file associations.
func (b *BackupEngine) recordChunk(
ctx context.Context,
file *database.File,
chunkIndex int,
chunkData []byte,
processedChunks map[string]bool,
) error {
chunkHash := calculateHash(chunkData)
// Check if chunk already exists (outside of transaction)
existingChunk, _ := b.repos.Chunks.GetByHash(ctx, chunkHash)
if existingChunk == nil {
// Create new chunk in a short transaction
err := b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
chunk := &database.Chunk{
ChunkHash: types.ChunkHash(chunkHash),
Size: int64(len(chunkData)),
}
return b.repos.Chunks.Create(ctx, tx, chunk)
})
if err != nil {
return err
}
processedChunks[chunkHash] = true
}
// Create file-chunk mapping in a short transaction
err := b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
fileChunk := &database.FileChunk{
FileID: file.ID,
Idx: chunkIndex,
ChunkHash: types.ChunkHash(chunkHash),
}
return b.repos.FileChunks.Create(ctx, tx, fileChunk)
})
if err != nil {
return err
}
// Create chunk-file mapping in a short transaction
return b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
chunkFile := &database.ChunkFile{
ChunkHash: types.ChunkHash(chunkHash),
FileID: file.ID,
FileOffset: int64(chunkIndex * defaultChunkSize),
Length: int64(len(chunkData)),
}
return b.repos.ChunkFiles.Create(ctx, tx, chunkFile)
})
}
// createBlobsForChunks uploads one blob per new chunk and records the blob
// metadata and snapshot association.
func (b *BackupEngine) createBlobsForChunks(
ctx context.Context,
snapshotID string,
processedChunks map[string]bool,
counters *backupCounters,
) error {
for chunkHash := range processedChunks {
// Get chunk data (outside of transaction)
chunk, err := b.repos.Chunks.GetByHash(ctx, chunkHash)
if err != nil {
return "", err
return err
}
chunkCount++
counters.chunkCount++
// In a real system, blobs would contain multiple chunks and be encrypted
// For testing, we'll create a blob with a "blob-" prefix to differentiate
@@ -479,8 +575,9 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
dummyData := []byte(chunkHash)
// Upload to S3 as a blob
if err := b.s3Client.PutBlob(ctx, blobHash, dummyData); err != nil {
return "", err
err = b.s3Client.PutBlob(ctx, blobHash, dummyData)
if err != nil {
return err
}
// Create blob entry in a short transaction
@@ -496,11 +593,11 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
return b.repos.Blobs.Create(ctx, tx, blob)
})
if err != nil {
return "", err
return err
}
blobCount++
blobSize += chunk.Size
counters.blobCount++
counters.blobSize += chunk.Size
// Create blob-chunk mapping in a short transaction
err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
@@ -514,27 +611,20 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
return b.repos.BlobChunks.Create(ctx, tx, blobChunk)
})
if err != nil {
return "", err
return err
}
// Add blob to snapshot in a short transaction
err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
return b.repos.Snapshots.AddBlob(ctx, tx, snapshotID, blobID, types.BlobHash(blobHash))
return b.repos.Snapshots.AddBlob(ctx, tx, snapshotID, blobID,
types.BlobHash(blobHash))
})
if err != nil {
return "", err
return err
}
}
// Update snapshot with final counts
err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
return b.repos.Snapshots.UpdateCounts(ctx, tx, snapshotID, fileCount, chunkCount, blobCount, totalSize, blobSize)
})
if err != nil {
return "", err
}
return snapshotID, nil
return nil
}
func calculateHash(data []byte) string {

129
internal/snapshot/exclude_test.go
View File

@@ -10,16 +10,15 @@ import (
"github.com/spf13/afero"
"github.com/stretchr/testify/require"
"sneak.berlin/go/vaultik/internal/database"
"sneak.berlin/go/vaultik/internal/log"
"sneak.berlin/go/vaultik/internal/snapshot"
"sneak.berlin/go/vaultik/internal/types"
)
func setupExcludeTestFS(t *testing.T) afero.Fs {
func setupExcludeTestFS(t *testing.T) *afero.MemMapFs {
t.Helper()
// Create in-memory filesystem
fs := afero.NewMemMapFs()
fs := &afero.MemMapFs{}
// Create test directory structure:
// /backup/
@@ -77,12 +76,11 @@ func setupExcludeTestFS(t *testing.T) afero.Fs {
return fs
}
func createTestScanner(t *testing.T, fs afero.Fs, excludePatterns []string) (*snapshot.Scanner, *database.Repositories, func()) {
func createTestScanner(
t *testing.T, fs afero.Fs, excludePatterns []string,
) (*snapshot.Scanner, *database.Repositories, func()) {
t.Helper()
// Initialize logger
log.Initialize(log.Config{})
// Create test database
db, err := database.NewTestDB()
require.NoError(t, err)
@@ -95,8 +93,9 @@ func createTestScanner(t *testing.T, fs afero.Fs, excludePatterns []string) (*sn
Repositories: repos,
MaxBlobSize: 1024 * 1024,
CompressionLevel: 3,
AgeRecipients: []string{"age1ql3z7hjy54pw3hyww5ayyfg7zqgvc7w3j2elw8zmrj2kg5sfn9aqmcac8p"},
Exclude: excludePatterns,
AgeRecipients: []string{
"age1ql3z7hjy54pw3hyww5ayyfg7zqgvc7w3j2elw8zmrj2kg5sfn9aqmcac8p"},
Exclude: excludePatterns,
})
cleanup := func() {
@@ -106,14 +105,16 @@ func createTestScanner(t *testing.T, fs afero.Fs, excludePatterns []string) (*sn
return scanner, repos, cleanup
}
func createSnapshotRecord(t *testing.T, ctx context.Context, repos *database.Repositories, snapshotID string) {
func createSnapshotRecord(
ctx context.Context, t *testing.T, repos *database.Repositories, snapshotID string,
) {
t.Helper()
err := repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
snap := &database.Snapshot{
ID: types.SnapshotID(snapshotID),
Hostname: "test-host",
VaultikVersion: "test",
Hostname: testHost,
VaultikVersion: testVersion,
StartedAt: time.Now(),
CompletedAt: nil,
FileCount: 0,
@@ -130,6 +131,8 @@ func createSnapshotRecord(t *testing.T, ctx context.Context, repos *database.Rep
}
func TestExcludePatterns_ExcludeGitDirectory(t *testing.T) {
t.Parallel()
fs := setupExcludeTestFS(t)
scanner, repos, cleanup := createTestScanner(t, fs, []string{".git"})
@@ -138,13 +141,14 @@ func TestExcludePatterns_ExcludeGitDirectory(t *testing.T) {
require.NotNil(t, scanner)
ctx := context.Background()
createSnapshotRecord(t, ctx, repos, "test-snapshot")
createSnapshotRecord(ctx, t, repos, "test-snapshot")
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
require.NoError(t, err)
// Should have scanned files but NOT .git directory contents
// Expected: file1.txt, file2.log, src/main.go, src/test.go, node_modules/package/index.js,
// Expected: file1.txt, file2.log, src/main.go, src/test.go,
// node_modules/package/index.js,
// cache/temp.dat, build/output.bin, docs/readme.md, .DS_Store, thumbs.db,
// src/.hidden, important.log.bak
// Excluded: .git/config, .git/objects/pack/data.pack
@@ -152,6 +156,8 @@ func TestExcludePatterns_ExcludeGitDirectory(t *testing.T) {
}
func TestExcludePatterns_ExcludeByExtension(t *testing.T) {
t.Parallel()
fs := setupExcludeTestFS(t)
scanner, repos, cleanup := createTestScanner(t, fs, []string{"*.log"})
@@ -160,7 +166,7 @@ func TestExcludePatterns_ExcludeByExtension(t *testing.T) {
require.NotNil(t, scanner)
ctx := context.Background()
createSnapshotRecord(t, ctx, repos, "test-snapshot")
createSnapshotRecord(ctx, t, repos, "test-snapshot")
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
require.NoError(t, err)
@@ -171,6 +177,8 @@ func TestExcludePatterns_ExcludeByExtension(t *testing.T) {
}
func TestExcludePatterns_ExcludeNodeModules(t *testing.T) {
t.Parallel()
fs := setupExcludeTestFS(t)
scanner, repos, cleanup := createTestScanner(t, fs, []string{"node_modules"})
@@ -179,7 +187,7 @@ func TestExcludePatterns_ExcludeNodeModules(t *testing.T) {
require.NotNil(t, scanner)
ctx := context.Background()
createSnapshotRecord(t, ctx, repos, "test-snapshot")
createSnapshotRecord(ctx, t, repos, "test-snapshot")
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
require.NoError(t, err)
@@ -190,25 +198,32 @@ func TestExcludePatterns_ExcludeNodeModules(t *testing.T) {
}
func TestExcludePatterns_MultiplePatterns(t *testing.T) {
t.Parallel()
fs := setupExcludeTestFS(t)
scanner, repos, cleanup := createTestScanner(t, fs, []string{".git", "node_modules", "*.log", ".DS_Store", "thumbs.db", "cache", "build"})
scanner, repos, cleanup := createTestScanner(t, fs, []string{
".git", "node_modules", "*.log", ".DS_Store", "thumbs.db", "cache", "build"})
defer cleanup()
require.NotNil(t, scanner)
ctx := context.Background()
createSnapshotRecord(t, ctx, repos, "test-snapshot")
createSnapshotRecord(ctx, t, repos, "test-snapshot")
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
require.NoError(t, err)
// Should only have: file1.txt, src/main.go, src/test.go, docs/readme.md, src/.hidden, important.log.bak
// Excluded: .git/*, node_modules/*, *.log (file2.log), .DS_Store, thumbs.db, cache/*, build/*
// Should only have: file1.txt, src/main.go, src/test.go, docs/readme.md,
// src/.hidden, important.log.bak
// Excluded: .git/*, node_modules/*, *.log (file2.log), .DS_Store,
// thumbs.db, cache/*, build/*
require.Equal(t, 6, result.FilesScanned, "Should exclude multiple patterns")
}
func TestExcludePatterns_NoExclusions(t *testing.T) {
t.Parallel()
fs := setupExcludeTestFS(t)
scanner, repos, cleanup := createTestScanner(t, fs, []string{})
@@ -217,7 +232,7 @@ func TestExcludePatterns_NoExclusions(t *testing.T) {
require.NotNil(t, scanner)
ctx := context.Background()
createSnapshotRecord(t, ctx, repos, "test-snapshot")
createSnapshotRecord(ctx, t, repos, "test-snapshot")
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
require.NoError(t, err)
@@ -227,6 +242,8 @@ func TestExcludePatterns_NoExclusions(t *testing.T) {
}
func TestExcludePatterns_ExcludeHiddenFiles(t *testing.T) {
t.Parallel()
fs := setupExcludeTestFS(t)
scanner, repos, cleanup := createTestScanner(t, fs, []string{".*"})
@@ -235,17 +252,21 @@ func TestExcludePatterns_ExcludeHiddenFiles(t *testing.T) {
require.NotNil(t, scanner)
ctx := context.Background()
createSnapshotRecord(t, ctx, repos, "test-snapshot")
createSnapshotRecord(ctx, t, repos, "test-snapshot")
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
require.NoError(t, err)
// Should exclude: .git/*, .DS_Store, src/.hidden
// Total files: 14, excluded: 4 (.git/config, .git/objects/pack/data.pack, .DS_Store, src/.hidden)
require.Equal(t, 10, result.FilesScanned, "Should exclude hidden files and directories")
// Total files: 14, excluded: 4 (.git/config,
// .git/objects/pack/data.pack, .DS_Store, src/.hidden)
require.Equal(t, 10, result.FilesScanned,
"Should exclude hidden files and directories")
}
func TestExcludePatterns_DoubleStarGlob(t *testing.T) {
t.Parallel()
fs := setupExcludeTestFS(t)
scanner, repos, cleanup := createTestScanner(t, fs, []string{"**/*.pack"})
@@ -254,7 +275,7 @@ func TestExcludePatterns_DoubleStarGlob(t *testing.T) {
require.NotNil(t, scanner)
ctx := context.Background()
createSnapshotRecord(t, ctx, repos, "test-snapshot")
createSnapshotRecord(ctx, t, repos, "test-snapshot")
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
require.NoError(t, err)
@@ -265,6 +286,8 @@ func TestExcludePatterns_DoubleStarGlob(t *testing.T) {
}
func TestExcludePatterns_ExactFileName(t *testing.T) {
t.Parallel()
fs := setupExcludeTestFS(t)
scanner, repos, cleanup := createTestScanner(t, fs, []string{"thumbs.db", ".DS_Store"})
@@ -273,7 +296,7 @@ func TestExcludePatterns_ExactFileName(t *testing.T) {
require.NotNil(t, scanner)
ctx := context.Background()
createSnapshotRecord(t, ctx, repos, "test-snapshot")
createSnapshotRecord(ctx, t, repos, "test-snapshot")
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
require.NoError(t, err)
@@ -284,6 +307,8 @@ func TestExcludePatterns_ExactFileName(t *testing.T) {
}
func TestExcludePatterns_CaseSensitive(t *testing.T) {
t.Parallel()
// Pattern matching should be case-sensitive
fs := setupExcludeTestFS(t)
@@ -293,7 +318,7 @@ func TestExcludePatterns_CaseSensitive(t *testing.T) {
require.NotNil(t, scanner)
ctx := context.Background()
createSnapshotRecord(t, ctx, repos, "test-snapshot")
createSnapshotRecord(ctx, t, repos, "test-snapshot")
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
require.NoError(t, err)
@@ -304,6 +329,8 @@ func TestExcludePatterns_CaseSensitive(t *testing.T) {
}
func TestExcludePatterns_DirectoryWithTrailingSlash(t *testing.T) {
t.Parallel()
fs := setupExcludeTestFS(t)
// Some users might add trailing slashes to directory patterns
scanner, repos, cleanup := createTestScanner(t, fs, []string{"cache/", "build/"})
@@ -312,17 +339,20 @@ func TestExcludePatterns_DirectoryWithTrailingSlash(t *testing.T) {
require.NotNil(t, scanner)
ctx := context.Background()
createSnapshotRecord(t, ctx, repos, "test-snapshot")
createSnapshotRecord(ctx, t, repos, "test-snapshot")
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
require.NoError(t, err)
// Should exclude cache/temp.dat and build/output.bin
// Total files: 14, excluded: 2
require.Equal(t, 12, result.FilesScanned, "Should handle directory patterns with trailing slashes")
require.Equal(t, 12, result.FilesScanned,
"Should handle directory patterns with trailing slashes")
}
func TestExcludePatterns_PatternInSubdirectory(t *testing.T) {
t.Parallel()
fs := setupExcludeTestFS(t)
// Exclude .hidden file specifically in src directory
scanner, repos, cleanup := createTestScanner(t, fs, []string{"src/.hidden"})
@@ -331,7 +361,7 @@ func TestExcludePatterns_PatternInSubdirectory(t *testing.T) {
require.NotNil(t, scanner)
ctx := context.Background()
createSnapshotRecord(t, ctx, repos, "test-snapshot")
createSnapshotRecord(ctx, t, repos, "test-snapshot")
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
require.NoError(t, err)
@@ -350,13 +380,14 @@ func TestExcludePatterns_PatternInSubdirectory(t *testing.T) {
// file.txt (should be excluded with /projectname)
// otherproject/
// projectname/
// file.txt (should NOT be excluded with /projectname, only with projectname)
// file.txt (should NOT be excluded with /projectname,
// only with projectname)
// src/
// file.go
func setupAnchoredTestFS(t *testing.T) afero.Fs {
func setupAnchoredTestFS(t *testing.T) *afero.MemMapFs {
t.Helper()
fs := afero.NewMemMapFs()
fs := &afero.MemMapFs{}
files := map[string]string{
"/backup/projectname/file.txt": "root project file",
@@ -381,6 +412,8 @@ func setupAnchoredTestFS(t *testing.T) afero.Fs {
}
func TestExcludePatterns_AnchoredPattern(t *testing.T) {
t.Parallel()
// Pattern starting with / should only match from root of source dir
fs := setupAnchoredTestFS(t)
@@ -390,7 +423,7 @@ func TestExcludePatterns_AnchoredPattern(t *testing.T) {
require.NotNil(t, scanner)
ctx := context.Background()
createSnapshotRecord(t, ctx, repos, "test-snapshot")
createSnapshotRecord(ctx, t, repos, "test-snapshot")
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
require.NoError(t, err)
@@ -398,10 +431,13 @@ func TestExcludePatterns_AnchoredPattern(t *testing.T) {
// /projectname should ONLY exclude /backup/projectname/file.txt (1 file)
// /backup/otherproject/projectname/file.txt should NOT be excluded
// Total files: 4, excluded: 1
require.Equal(t, 3, result.FilesScanned, "Anchored pattern /projectname should only match at root of source dir")
require.Equal(t, 3, result.FilesScanned,
"Anchored pattern /projectname should only match at root of source dir")
}
func TestExcludePatterns_UnanchoredPattern(t *testing.T) {
t.Parallel()
// Pattern without leading / should match anywhere in path
fs := setupAnchoredTestFS(t)
@@ -411,7 +447,7 @@ func TestExcludePatterns_UnanchoredPattern(t *testing.T) {
require.NotNil(t, scanner)
ctx := context.Background()
createSnapshotRecord(t, ctx, repos, "test-snapshot")
createSnapshotRecord(ctx, t, repos, "test-snapshot")
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
require.NoError(t, err)
@@ -420,10 +456,13 @@ func TestExcludePatterns_UnanchoredPattern(t *testing.T) {
// - /backup/projectname/file.txt
// - /backup/otherproject/projectname/file.txt
// Total files: 4, excluded: 2
require.Equal(t, 2, result.FilesScanned, "Unanchored pattern should match anywhere in path")
require.Equal(t, 2, result.FilesScanned,
"Unanchored pattern should match anywhere in path")
}
func TestExcludePatterns_AnchoredPatternWithGlob(t *testing.T) {
t.Parallel()
// Anchored pattern with glob
fs := setupAnchoredTestFS(t)
@@ -433,7 +472,7 @@ func TestExcludePatterns_AnchoredPatternWithGlob(t *testing.T) {
require.NotNil(t, scanner)
ctx := context.Background()
createSnapshotRecord(t, ctx, repos, "test-snapshot")
createSnapshotRecord(ctx, t, repos, "test-snapshot")
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
require.NoError(t, err)
@@ -444,6 +483,8 @@ func TestExcludePatterns_AnchoredPatternWithGlob(t *testing.T) {
}
func TestExcludePatterns_AnchoredPatternFile(t *testing.T) {
t.Parallel()
// Anchored pattern for exact file at root
fs := setupAnchoredTestFS(t)
@@ -453,7 +494,7 @@ func TestExcludePatterns_AnchoredPatternFile(t *testing.T) {
require.NotNil(t, scanner)
ctx := context.Background()
createSnapshotRecord(t, ctx, repos, "test-snapshot")
createSnapshotRecord(ctx, t, repos, "test-snapshot")
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
require.NoError(t, err)
@@ -461,10 +502,13 @@ func TestExcludePatterns_AnchoredPatternFile(t *testing.T) {
// /file.txt should ONLY exclude /backup/file.txt
// NOT /backup/projectname/file.txt or /backup/otherproject/projectname/file.txt
// Total files: 4, excluded: 1
require.Equal(t, 3, result.FilesScanned, "Anchored pattern for file should only match at root")
require.Equal(t, 3, result.FilesScanned,
"Anchored pattern for file should only match at root")
}
func TestExcludePatterns_UnanchoredPatternFile(t *testing.T) {
t.Parallel()
// Unanchored pattern for file should match anywhere
fs := setupAnchoredTestFS(t)
@@ -474,7 +518,7 @@ func TestExcludePatterns_UnanchoredPatternFile(t *testing.T) {
require.NotNil(t, scanner)
ctx := context.Background()
createSnapshotRecord(t, ctx, repos, "test-snapshot")
createSnapshotRecord(ctx, t, repos, "test-snapshot")
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
require.NoError(t, err)
@@ -484,5 +528,6 @@ func TestExcludePatterns_UnanchoredPatternFile(t *testing.T) {
// - /backup/projectname/file.txt
// - /backup/otherproject/projectname/file.txt
// Total files: 4, excluded: 3
require.Equal(t, 1, result.FilesScanned, "Unanchored pattern for file should match anywhere")
require.Equal(t, 1, result.FilesScanned,
"Unanchored pattern for file should match anywhere")
}

136
internal/snapshot/file_change_test.go
View File

@@ -2,7 +2,6 @@ package snapshot_test
import (
"context"
"database/sql"
"testing"
"time"
@@ -15,11 +14,55 @@ import (
"sneak.berlin/go/vaultik/internal/types"
)
// verifyChunkChange checks that after a content change the file references
// the new chunk, the old chunk still exists, and the old chunk no longer
// maps to the modified file.
func verifyChunkChange(
ctx context.Context,
t *testing.T,
repos *database.Repositories,
oldChunkHash, newChunkHash types.ChunkHash,
) {
t.Helper()
// Verify the chunk hashes are different
assert.NotEqual(t, oldChunkHash, newChunkHash,
"Chunk hash should change when content changes")
// Get chunk files from second scan
chunkFiles2, err := repos.ChunkFiles.GetByFilePath(ctx, "/test.txt")
require.NoError(t, err)
assert.Len(t, chunkFiles2, 1)
assert.Equal(t, newChunkHash, chunkFiles2[0].ChunkHash)
// Verify old chunk still exists (it's still valid data)
oldChunk, err := repos.Chunks.GetByHash(ctx, oldChunkHash.String())
require.NoError(t, err)
assert.NotNil(t, oldChunk)
// Verify new chunk exists
newChunk, err := repos.Chunks.GetByHash(ctx, newChunkHash.String())
require.NoError(t, err)
assert.NotNil(t, newChunk)
// Verify that chunk_files for old chunk no longer references this file
oldChunkFiles, err := repos.ChunkFiles.GetByChunkHash(ctx, oldChunkHash)
require.NoError(t, err)
for _, cf := range oldChunkFiles {
file, err := repos.Files.GetByID(ctx, cf.FileID)
require.NoError(t, err)
assert.NotEqual(t, "/data/test.txt", file.Path,
"Old chunk should not be associated with the modified file")
}
}
// TestFileContentChange verifies that when a file's content changes,
// the old chunks are properly disassociated
func TestFileContentChange(t *testing.T) {
// Initialize logger for tests
log.Initialize(log.Config{})
t.Parallel()
// Create in-memory filesystem
fs := afero.NewMemMapFs()
@@ -48,23 +91,13 @@ func TestFileContentChange(t *testing.T) {
Repositories: repos,
MaxBlobSize: int64(1024 * 1024), // 1MB blobs
CompressionLevel: 3,
AgeRecipients: []string{"age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05gl0sjq9q9wjg"}, // Test public key
AgeRecipients: []string{testAgePublicKey},
})
// Create first snapshot
ctx := context.Background()
snapshotID1 := "snapshot1"
err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
snapshot := &database.Snapshot{
ID: types.SnapshotID(snapshotID1),
Hostname: "test-host",
VaultikVersion: "test",
StartedAt: time.Now(),
}
return repos.Snapshots.Create(ctx, tx, snapshot)
})
require.NoError(t, err)
createSnapshotRecord(ctx, t, repos, snapshotID1)
// First scan - should create chunks for initial content
result1, err := scanner.Scan(ctx, "/", snapshotID1)
@@ -85,22 +118,13 @@ func TestFileContentChange(t *testing.T) {
// Modify the file
time.Sleep(10 * time.Millisecond) // Ensure mtime changes
err = afero.WriteFile(fs, "/test.txt", []byte("Modified content with different data"), 0644)
err = afero.WriteFile(fs, "/test.txt",
[]byte("Modified content with different data"), 0644)
require.NoError(t, err)
// Create second snapshot
snapshotID2 := "snapshot2"
err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
snapshot := &database.Snapshot{
ID: types.SnapshotID(snapshotID2),
Hostname: "test-host",
VaultikVersion: "test",
StartedAt: time.Now(),
}
return repos.Snapshots.Create(ctx, tx, snapshot)
})
require.NoError(t, err)
createSnapshotRecord(ctx, t, repos, snapshotID2)
// Second scan - should create new chunks and remove old associations
result2, err := scanner.Scan(ctx, "/", snapshotID2)
@@ -113,40 +137,14 @@ func TestFileContentChange(t *testing.T) {
assert.Len(t, fileChunks2, 1) // Still 1 chunk but different hash
newChunkHash := fileChunks2[0].ChunkHash
// Verify the chunk hashes are different
assert.NotEqual(t, oldChunkHash, newChunkHash, "Chunk hash should change when content changes")
// Get chunk files from second scan
chunkFiles2, err := repos.ChunkFiles.GetByFilePath(ctx, "/test.txt")
require.NoError(t, err)
assert.Len(t, chunkFiles2, 1)
assert.Equal(t, newChunkHash, chunkFiles2[0].ChunkHash)
// Verify old chunk still exists (it's still valid data)
oldChunk, err := repos.Chunks.GetByHash(ctx, oldChunkHash.String())
require.NoError(t, err)
assert.NotNil(t, oldChunk)
// Verify new chunk exists
newChunk, err := repos.Chunks.GetByHash(ctx, newChunkHash.String())
require.NoError(t, err)
assert.NotNil(t, newChunk)
// Verify that chunk_files for old chunk no longer references this file
oldChunkFiles, err := repos.ChunkFiles.GetByChunkHash(ctx, oldChunkHash)
require.NoError(t, err)
for _, cf := range oldChunkFiles {
file, err := repos.Files.GetByID(ctx, cf.FileID)
require.NoError(t, err)
assert.NotEqual(t, "/data/test.txt", file.Path, "Old chunk should not be associated with the modified file")
}
verifyChunkChange(ctx, t, repos, oldChunkHash, newChunkHash)
}
// TestMultipleFileChanges verifies handling of multiple file changes in one scan
func TestMultipleFileChanges(t *testing.T) {
// Initialize logger for tests
log.Initialize(log.Config{})
t.Parallel()
// Create in-memory filesystem
fs := afero.NewMemMapFs()
@@ -183,23 +181,13 @@ func TestMultipleFileChanges(t *testing.T) {
Repositories: repos,
MaxBlobSize: int64(1024 * 1024), // 1MB blobs
CompressionLevel: 3,
AgeRecipients: []string{"age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05gl0sjq9q9wjg"}, // Test public key
AgeRecipients: []string{testAgePublicKey},
})
// Create first snapshot
ctx := context.Background()
snapshotID1 := "snapshot1"
err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
snapshot := &database.Snapshot{
ID: types.SnapshotID(snapshotID1),
Hostname: "test-host",
VaultikVersion: "test",
StartedAt: time.Now(),
}
return repos.Snapshots.Create(ctx, tx, snapshot)
})
require.NoError(t, err)
createSnapshotRecord(ctx, t, repos, snapshotID1)
// First scan
result1, err := scanner.Scan(ctx, "/", snapshotID1)
@@ -217,17 +205,7 @@ func TestMultipleFileChanges(t *testing.T) {
// Create second snapshot
snapshotID2 := "snapshot2"
err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
snapshot := &database.Snapshot{
ID: types.SnapshotID(snapshotID2),
Hostname: "test-host",
VaultikVersion: "test",
StartedAt: time.Now(),
}
return repos.Snapshots.Create(ctx, tx, snapshot)
})
require.NoError(t, err)
createSnapshotRecord(ctx, t, repos, snapshotID2)
// Second scan
result2, err := scanner.Scan(ctx, "/", snapshotID2)
@@ -240,10 +218,12 @@ func TestMultipleFileChanges(t *testing.T) {
for path := range files {
fileChunks, err := repos.FileChunks.GetByPath(ctx, path)
require.NoError(t, err)
assert.Len(t, fileChunks, 1, "File %s should have exactly 1 chunk association", path)
assert.Len(t, fileChunks, 1,
"File %s should have exactly 1 chunk association", path)
chunkFiles, err := repos.ChunkFiles.GetByFilePath(ctx, path)
require.NoError(t, err)
assert.Len(t, chunkFiles, 1, "File %s should have exactly 1 chunk-file association", path)
assert.Len(t, chunkFiles, 1,
"File %s should have exactly 1 chunk-file association", path)
}
}

17
internal/snapshot/manifest.go
View File

@@ -10,6 +10,8 @@ import (
)
// Manifest represents the structure of a snapshot's blob manifest
//
//nolint:tagliatelle // snake_case is the established on-disk manifest format
type Manifest struct {
SnapshotID string `json:"snapshot_id"`
Timestamp string `json:"timestamp"`
@@ -19,6 +21,8 @@ type Manifest struct {
}
// BlobInfo represents information about a single blob in the manifest
//
//nolint:tagliatelle // snake_case is the established on-disk manifest format
type BlobInfo struct {
Hash string `json:"hash"`
CompressedSize int64 `json:"compressed_size"`
@@ -35,7 +39,9 @@ func DecodeManifest(r io.Reader) (*Manifest, error) {
// Decode JSON manifest
var manifest Manifest
if err := json.NewDecoder(zr).Decode(&manifest); err != nil {
err = json.NewDecoder(zr).Decode(&manifest)
if err != nil {
return nil, fmt.Errorf("decoding manifest: %w", err)
}
@@ -53,18 +59,21 @@ func EncodeManifest(manifest *Manifest, compressionLevel int) ([]byte, error) {
// Compress using zstd
var compressedBuf bytes.Buffer
writer, err := zstd.NewWriter(&compressedBuf, zstd.WithEncoderLevel(zstd.EncoderLevelFromZstd(compressionLevel)))
writer, err := zstd.NewWriter(&compressedBuf,
zstd.WithEncoderLevel(zstd.EncoderLevelFromZstd(compressionLevel)))
if err != nil {
return nil, fmt.Errorf("creating zstd writer: %w", err)
}
if _, err := writer.Write(jsonData); err != nil {
_, err = writer.Write(jsonData)
if err != nil {
_ = writer.Close()
return nil, fmt.Errorf("writing compressed data: %w", err)
}
if err := writer.Close(); err != nil {
err = writer.Close()
if err != nil {
return nil, fmt.Errorf("closing zstd writer: %w", err)
}

6
internal/snapshot/module.go
View File

@@ -21,6 +21,8 @@ type ScannerParams struct {
// Module exports backup functionality as an fx module.
// It provides a ScannerFactory that can create Scanner instances
// with custom parameters while sharing common dependencies.
//
//nolint:gochecknoglobals // fx module definitions are conventionally globals
var Module = fx.Module("backup",
fx.Provide(
provideScannerFactory,
@@ -31,7 +33,9 @@ var Module = fx.Module("backup",
// ScannerFactory creates scanners with custom parameters
type ScannerFactory func(params ScannerParams) *Scanner
func provideScannerFactory(cfg *config.Config, repos *database.Repositories, storer storage.Storer) ScannerFactory {
func provideScannerFactory(
cfg *config.Config, repos *database.Repositories, storer storage.Storer,
) ScannerFactory {
return func(params ScannerParams) *Scanner {
// Use provided excludes, or fall back to global config excludes
excludes := params.Exclude

27
internal/snapshot/permission_error_test.go
View File

@@ -1,3 +1,4 @@
//nolint:testpackage // needs access to unexported wrapPermissionError
package snapshot
import (
@@ -9,12 +10,15 @@ import (
"testing"
)
func TestWrapPermissionError(t *testing.T) {
// Non-permission errors pass through unchanged.
plain := errors.New("disk on fire")
// errDiskOnFire is a non-permission sentinel used to verify pass-through.
var errDiskOnFire = errors.New("disk on fire")
got := wrapPermissionError("/some/path", plain)
if !errors.Is(got, plain) {
func TestWrapPermissionError(t *testing.T) {
t.Parallel()
// Non-permission errors pass through unchanged.
got := wrapPermissionError("/some/path", errDiskOnFire)
if !errors.Is(got, errDiskOnFire) {
t.Errorf("non-permission error should pass through, got %v", got)
}
@@ -32,15 +36,16 @@ func TestWrapPermissionError(t *testing.T) {
if runtime.GOOS == "darwin" {
if !strings.Contains(wrapped.Error(), "Full Disk Access") {
t.Errorf("macOS permission error should mention Full Disk Access:\n%s", wrapped.Error())
t.Errorf("macOS permission error should mention Full Disk Access:\n%s",
wrapped.Error())
}
if !strings.Contains(wrapped.Error(), "System Settings") {
t.Errorf("macOS permission error should point at System Settings:\n%s", wrapped.Error())
}
} else {
if !strings.Contains(wrapped.Error(), "--skip-errors") {
t.Errorf("non-macOS permission error should mention --skip-errors:\n%s", wrapped.Error())
t.Errorf("macOS permission error should point at System Settings:\n%s",
wrapped.Error())
}
} else if !strings.Contains(wrapped.Error(), "--skip-errors") {
t.Errorf("non-macOS permission error should mention --skip-errors:\n%s",
wrapped.Error())
}
}

357
internal/snapshot/progress.go
View File

@@ -19,14 +19,41 @@ const (
// These updates show current progress, ETA, and the file being processed.
SummaryInterval = 10 * time.Second
// DetailInterval defines how often multi-line detailed status reports are printed.
// These reports include comprehensive statistics about files, chunks, blobs, and uploads.
// DetailInterval defines how often multi-line detailed status reports are
// printed. These reports include comprehensive statistics about files,
// chunks, blobs, and uploads.
DetailInterval = 60 * time.Second
// UploadProgressInterval defines how often upload progress messages are logged.
UploadProgressInterval = 15 * time.Second
)
const (
// bitsPerByte converts byte counts to bit counts for speed display.
bitsPerByte = 8
// percentScale converts a ratio to a percentage.
percentScale = 100
// currentFileMaxLen is the display width used for current-file paths.
currentFileMaxLen = 40
// secondsPerMinute and minutesPerHour are used for duration formatting.
secondsPerMinute = 60
minutesPerHour = 60
// Bit-rate thresholds for human-readable upload speed formatting.
bitsPerGbit = 1e9
bitsPerMbit = 1e6
bitsPerKbit = 1e3
// ellipsis prefixes truncated paths and suffixes shortened hashes.
ellipsis = "..."
// hashPrefixLen is how many hex characters of a blob hash to show in logs.
hashPrefixLen = 8
)
// ProgressStats holds atomic counters for progress tracking
type ProgressStats struct {
FilesScanned atomic.Int64 // Total files seen during scan (includes skipped)
@@ -64,7 +91,7 @@ type UploadInfo struct {
// ProgressReporter handles periodic progress reporting
type ProgressReporter struct {
stats *ProgressStats
ctx context.Context
ctx context.Context //nolint:containedctx // bound at construction
cancel context.CancelFunc
wg sync.WaitGroup
detailTicker *time.Ticker
@@ -127,6 +154,161 @@ func (pr *ProgressReporter) SetTotalSize(size int64) {
pr.stats.ProcessStartTime.Store(time.Now().UTC())
}
// Helper functions
func formatDuration(d time.Duration) string {
if d < 0 {
return "unknown"
}
if d < time.Minute {
return fmt.Sprintf("%ds", int(d.Seconds()))
}
if d < time.Hour {
return fmt.Sprintf("%dm%ds", int(d.Minutes()), int(d.Seconds())%secondsPerMinute)
}
return fmt.Sprintf("%dh%dm", int(d.Hours()), int(d.Minutes())%minutesPerHour)
}
func formatPercent(numerator, denominator int64) string {
if denominator == 0 {
return "0.0%"
}
return fmt.Sprintf("%.1f%%", float64(numerator)/float64(denominator)*percentScale)
}
func formatRatio(compressed, uncompressed int64) string {
if uncompressed == 0 {
return "1.00"
}
ratio := float64(compressed) / float64(uncompressed)
return fmt.Sprintf("%.2f", ratio)
}
func truncatePath(path string, maxLen int) string {
if len(path) <= maxLen {
return path
}
// Keep the last maxLen-len(ellipsis) characters and prepend the ellipsis.
return ellipsis + path[len(path)-(maxLen-len(ellipsis)):]
}
// safeUint64 converts a non-negative int64 counter to uint64 for display,
// clamping negative values to zero.
func safeUint64(n int64) uint64 {
if n < 0 {
return 0
}
return uint64(n)
}
// ReportUploadStart marks the beginning of a blob upload
func (pr *ProgressReporter) ReportUploadStart(blobHash string, size int64) {
info := &UploadInfo{
BlobHash: blobHash,
Size: size,
StartTime: time.Now().UTC(),
}
pr.stats.CurrentUpload.Store(info)
// Log the start of upload
log.Info("Starting blob upload",
"hash", blobHash[:hashPrefixLen]+ellipsis,
"size", humanize.Bytes(safeUint64(size)))
}
// ReportUploadComplete marks the completion of a blob upload
func (pr *ProgressReporter) ReportUploadComplete(
blobHash string, size int64, duration time.Duration,
) {
// Clear current upload
pr.stats.CurrentUpload.Store((*UploadInfo)(nil))
// Add to total upload duration
pr.stats.UploadDurationMs.Add(duration.Milliseconds())
// Calculate speed
if duration < time.Millisecond {
duration = time.Millisecond
}
bytesPerSec := float64(size) / duration.Seconds()
bitsPerSec := bytesPerSec * bitsPerByte
// Format speed
var speedStr string
switch {
case bitsPerSec >= bitsPerGbit:
speedStr = fmt.Sprintf("%.1fGbit/sec", bitsPerSec/bitsPerGbit)
case bitsPerSec >= bitsPerMbit:
speedStr = fmt.Sprintf("%.0fMbit/sec", bitsPerSec/bitsPerMbit)
case bitsPerSec >= bitsPerKbit:
speedStr = fmt.Sprintf("%.0fKbit/sec", bitsPerSec/bitsPerKbit)
default:
speedStr = fmt.Sprintf("%.0fbit/sec", bitsPerSec)
}
log.Info("Blob upload completed",
"hash", blobHash[:hashPrefixLen]+ellipsis,
"size", humanize.Bytes(safeUint64(size)),
"duration", formatDuration(duration),
"speed", speedStr)
}
// UpdateChunkingActivity updates the last chunking time
func (pr *ProgressReporter) UpdateChunkingActivity() {
pr.stats.mu.Lock()
pr.stats.lastChunkingTime = time.Now().UTC()
pr.stats.mu.Unlock()
}
// ReportUploadProgress reports current upload progress with instantaneous speed
func (pr *ProgressReporter) ReportUploadProgress(
blobHash string, bytesUploaded, totalSize int64, instantSpeed float64,
) {
// Update the current upload info with progress
uploadInfo, ok := pr.stats.CurrentUpload.Load().(*UploadInfo)
if ok && uploadInfo != nil {
now := time.Now()
// Only log at the configured interval
if now.Sub(uploadInfo.LastLogTime) >= UploadProgressInterval {
// Format speed in bits/second using humanize
bitsPerSec := instantSpeed * bitsPerByte
speedStr := humanize.SI(bitsPerSec, "bit/sec")
percent := float64(bytesUploaded) / float64(totalSize) * percentScale
// Calculate ETA based on current speed
etaStr := "unknown"
if instantSpeed > 0 && bytesUploaded < totalSize {
remainingBytes := totalSize - bytesUploaded
remainingSeconds := float64(remainingBytes) / instantSpeed
eta := time.Duration(remainingSeconds * float64(time.Second))
etaStr = formatDuration(eta)
}
log.Info("Blob upload progress",
"hash", blobHash[:hashPrefixLen]+ellipsis,
"progress", fmt.Sprintf("%.1f%%", percent),
"uploaded", humanize.Bytes(safeUint64(bytesUploaded)),
"total", humanize.Bytes(safeUint64(totalSize)),
"speed", speedStr,
"eta", etaStr)
uploadInfo.LastLogTime = now
}
}
}
// run is the main progress reporting loop
func (pr *ProgressReporter) run() {
defer pr.wg.Done()
@@ -150,7 +332,8 @@ func (pr *ProgressReporter) run() {
// printSummaryStatus prints a one-line status update
func (pr *ProgressReporter) printSummaryStatus() {
// Check if we're currently uploading
if uploadInfo, ok := pr.stats.CurrentUpload.Load().(*UploadInfo); ok && uploadInfo != nil {
uploadInfo, ok := pr.stats.CurrentUpload.Load().(*UploadInfo)
if ok && uploadInfo != nil {
// Show upload progress instead
pr.printUploadProgress(uploadInfo)
@@ -172,7 +355,7 @@ func (pr *ProgressReporter) printSummaryStatus() {
bytesSkipped := pr.stats.BytesSkipped.Load()
bytesProcessed := pr.stats.BytesProcessed.Load()
totalSize := pr.stats.TotalSize.Load()
currentFile := pr.stats.CurrentFile.Load().(string)
currentFile, _ := pr.stats.CurrentFile.Load().(string)
// Calculate ETA if we have total size and are processing
etaStr := ""
@@ -201,15 +384,15 @@ func (pr *ProgressReporter) printSummaryStatus() {
status := fmt.Sprintf("Snapshot progress: %d/%d files, %s/%s (%.1f%%), %s/s%s",
filesProcessed,
totalFiles,
humanize.Bytes(uint64(bytesProcessed)),
humanize.Bytes(uint64(totalSize)),
float64(bytesProcessed)/float64(totalSize)*100,
humanize.Bytes(safeUint64(bytesProcessed)),
humanize.Bytes(safeUint64(totalSize)),
float64(bytesProcessed)/float64(totalSize)*percentScale,
humanize.Bytes(uint64(rate)),
etaStr,
)
if currentFile != "" {
status += " | Current: " + truncatePath(currentFile, 40)
status += " | Current: " + truncatePath(currentFile, currentFileMaxLen)
}
log.Info(status)
@@ -232,7 +415,7 @@ func (pr *ProgressReporter) printDetailedStatus() {
blobsCreated := pr.stats.BlobsCreated.Load()
blobsUploaded := pr.stats.BlobsUploaded.Load()
bytesUploaded := pr.stats.BytesUploaded.Load()
currentFile := pr.stats.CurrentFile.Load().(string)
currentFile, _ := pr.stats.CurrentFile.Load().(string)
totalBytes := bytesScanned + bytesSkipped
rate := float64(totalBytes) / elapsed.Seconds()
@@ -251,11 +434,11 @@ func (pr *ProgressReporter) printDetailedStatus() {
remainingBytes := totalSize - bytesProcessed
remainingSeconds := float64(remainingBytes) / processRate
eta := time.Duration(remainingSeconds * float64(time.Second))
percentComplete := float64(bytesProcessed) / float64(totalSize) * 100
percentComplete := float64(bytesProcessed) / float64(totalSize) * percentScale
log.Info("Overall progress",
"percent", fmt.Sprintf("%.1f%%", percentComplete),
"processed", humanize.Bytes(uint64(bytesProcessed)),
"total", humanize.Bytes(uint64(totalSize)),
"processed", humanize.Bytes(safeUint64(bytesProcessed)),
"total", humanize.Bytes(safeUint64(totalSize)),
"rate", humanize.Bytes(uint64(processRate))+"/s",
"eta", formatDuration(eta))
}
@@ -268,9 +451,9 @@ func (pr *ProgressReporter) printDetailedStatus() {
"total", filesScanned,
"skip_rate", formatPercent(filesSkipped, filesScanned))
log.Info("Data scanned",
"new", humanize.Bytes(uint64(bytesScanned)),
"skipped", humanize.Bytes(uint64(bytesSkipped)),
"total", humanize.Bytes(uint64(totalBytes)),
"new", humanize.Bytes(safeUint64(bytesScanned)),
"skipped", humanize.Bytes(safeUint64(bytesSkipped)),
"total", humanize.Bytes(safeUint64(totalBytes)),
"scan_rate", humanize.Bytes(uint64(rate))+"/s")
log.Info("Chunks created", "count", chunksCreated)
log.Info("Blobs status",
@@ -278,7 +461,7 @@ func (pr *ProgressReporter) printDetailedStatus() {
"uploaded", blobsUploaded,
"pending", blobsCreated-blobsUploaded)
log.Info("Total uploaded to remote",
"uploaded", humanize.Bytes(uint64(bytesUploaded)),
"uploaded", humanize.Bytes(safeUint64(bytesUploaded)),
"compression_ratio", formatRatio(bytesUploaded, bytesScanned))
if currentFile != "" {
@@ -288,146 +471,8 @@ func (pr *ProgressReporter) printDetailedStatus() {
log.Notice("=============================")
}
// Helper functions
func formatDuration(d time.Duration) string {
if d < 0 {
return "unknown"
}
if d < time.Minute {
return fmt.Sprintf("%ds", int(d.Seconds()))
}
if d < time.Hour {
return fmt.Sprintf("%dm%ds", int(d.Minutes()), int(d.Seconds())%60)
}
return fmt.Sprintf("%dh%dm", int(d.Hours()), int(d.Minutes())%60)
}
func formatPercent(numerator, denominator int64) string {
if denominator == 0 {
return "0.0%"
}
return fmt.Sprintf("%.1f%%", float64(numerator)/float64(denominator)*100)
}
func formatRatio(compressed, uncompressed int64) string {
if uncompressed == 0 {
return "1.00"
}
ratio := float64(compressed) / float64(uncompressed)
return fmt.Sprintf("%.2f", ratio)
}
func truncatePath(path string, maxLen int) string {
if len(path) <= maxLen {
return path
}
// Keep the last maxLen-3 characters and prepend "..."
return "..." + path[len(path)-(maxLen-3):]
}
// printUploadProgress prints upload progress
func (pr *ProgressReporter) printUploadProgress(info *UploadInfo) {
func (pr *ProgressReporter) printUploadProgress(_ *UploadInfo) {
// This function is called repeatedly during upload, not just at start
// Don't print anything here - the actual progress is shown by ReportUploadProgress
}
// ReportUploadStart marks the beginning of a blob upload
func (pr *ProgressReporter) ReportUploadStart(blobHash string, size int64) {
info := &UploadInfo{
BlobHash: blobHash,
Size: size,
StartTime: time.Now().UTC(),
}
pr.stats.CurrentUpload.Store(info)
// Log the start of upload
log.Info("Starting blob upload",
"hash", blobHash[:8]+"...",
"size", humanize.Bytes(uint64(size)))
}
// ReportUploadComplete marks the completion of a blob upload
func (pr *ProgressReporter) ReportUploadComplete(blobHash string, size int64, duration time.Duration) {
// Clear current upload
pr.stats.CurrentUpload.Store((*UploadInfo)(nil))
// Add to total upload duration
pr.stats.UploadDurationMs.Add(duration.Milliseconds())
// Calculate speed
if duration < time.Millisecond {
duration = time.Millisecond
}
bytesPerSec := float64(size) / duration.Seconds()
bitsPerSec := bytesPerSec * 8
// Format speed
var speedStr string
if bitsPerSec >= 1e9 {
speedStr = fmt.Sprintf("%.1fGbit/sec", bitsPerSec/1e9)
} else if bitsPerSec >= 1e6 {
speedStr = fmt.Sprintf("%.0fMbit/sec", bitsPerSec/1e6)
} else if bitsPerSec >= 1e3 {
speedStr = fmt.Sprintf("%.0fKbit/sec", bitsPerSec/1e3)
} else {
speedStr = fmt.Sprintf("%.0fbit/sec", bitsPerSec)
}
log.Info("Blob upload completed",
"hash", blobHash[:8]+"...",
"size", humanize.Bytes(uint64(size)),
"duration", formatDuration(duration),
"speed", speedStr)
}
// UpdateChunkingActivity updates the last chunking time
func (pr *ProgressReporter) UpdateChunkingActivity() {
pr.stats.mu.Lock()
pr.stats.lastChunkingTime = time.Now().UTC()
pr.stats.mu.Unlock()
}
// ReportUploadProgress reports current upload progress with instantaneous speed
func (pr *ProgressReporter) ReportUploadProgress(blobHash string, bytesUploaded, totalSize int64, instantSpeed float64) {
// Update the current upload info with progress
if uploadInfo, ok := pr.stats.CurrentUpload.Load().(*UploadInfo); ok && uploadInfo != nil {
now := time.Now()
// Only log at the configured interval
if now.Sub(uploadInfo.LastLogTime) >= UploadProgressInterval {
// Format speed in bits/second using humanize
bitsPerSec := instantSpeed * 8
speedStr := humanize.SI(bitsPerSec, "bit/sec")
percent := float64(bytesUploaded) / float64(totalSize) * 100
// Calculate ETA based on current speed
etaStr := "unknown"
if instantSpeed > 0 && bytesUploaded < totalSize {
remainingBytes := totalSize - bytesUploaded
remainingSeconds := float64(remainingBytes) / instantSpeed
eta := time.Duration(remainingSeconds * float64(time.Second))
etaStr = formatDuration(eta)
}
log.Info("Blob upload progress",
"hash", blobHash[:8]+"...",
"progress", fmt.Sprintf("%.1f%%", percent),
"uploaded", humanize.Bytes(uint64(bytesUploaded)),
"total", humanize.Bytes(uint64(totalSize)),
"speed", speedStr,
"eta", etaStr)
uploadInfo.LastLogTime = now
}
}
}

897
internal/snapshot/scanner.go
View File

File diff suppressed because it is too large Load Diff

219
internal/snapshot/scanner_test.go
View File

@@ -3,6 +3,7 @@ package snapshot_test
import (
"context"
"database/sql"
"os"
"path/filepath"
"testing"
"time"
@@ -14,9 +15,114 @@ import (
"sneak.berlin/go/vaultik/internal/types"
)
// Shared test fixture values for the snapshot_test package.
const (
// testHost is the hostname recorded on test snapshot rows.
testHost = "test-host"
// testVersion is the vaultik version recorded on test snapshot rows.
testVersion = "test"
// testAgePublicKey is the fixed age public key used for test encryption.
testAgePublicKey = "age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05gl0sjq9q9wjg"
)
// TestMain initializes the shared logger once, before any tests run, so
// parallel tests never race on the logger's global state.
func TestMain(m *testing.M) {
log.Initialize(log.Config{})
os.Exit(m.Run())
}
// createTestSnapshotRecord inserts an empty snapshot row used as the
// association target for scan tests.
func createTestSnapshotRecord(
ctx context.Context, t *testing.T, repos *database.Repositories, snapshotID string,
) {
t.Helper()
err := repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
snapshot := &database.Snapshot{
ID: types.SnapshotID(snapshotID),
Hostname: testHost,
VaultikVersion: testVersion,
StartedAt: time.Now(),
CompletedAt: nil,
FileCount: 0,
ChunkCount: 0,
BlobCount: 0,
TotalSize: 0,
BlobSize: 0,
CompressionRatio: 1.0,
}
return repos.Snapshots.Create(ctx, tx, snapshot)
})
if err != nil {
t.Fatalf("failed to create snapshot: %v", err)
}
}
// verifySimpleScanDatabase checks the database contents produced by
// TestScannerSimpleDirectory's scan.
func verifySimpleScanDatabase(
ctx context.Context, t *testing.T, repos *database.Repositories,
) {
t.Helper()
// Verify files in database - includes regular files and directories
files, err := repos.Files.ListByPrefix(ctx, "/source")
if err != nil {
t.Fatalf("failed to list files: %v", err)
}
// 6 regular files + 3 directories (/source, /source/subdir, /source/subdir2)
if len(files) != 9 {
t.Errorf("expected 9 entries in database (6 files + 3 dirs), got %d", len(files))
}
// Verify specific file
file1, err := repos.Files.GetByPath(ctx, "/source/file1.txt")
if err != nil {
t.Fatalf("failed to get file1.txt: %v", err)
}
if file1.Size != 13 {
t.Errorf("expected file1.txt size 13, got %d", file1.Size)
}
if file1.Mode != 0644 {
t.Errorf("expected file1.txt mode 0644, got %o", file1.Mode)
}
// Verify chunks were created
chunks, err := repos.FileChunks.GetByFile(ctx, "/source/file1.txt")
if err != nil {
t.Fatalf("failed to get chunks for file1.txt: %v", err)
}
if len(chunks) != 1 { // Small file should be one chunk
t.Errorf("expected 1 chunk for file1.txt, got %d", len(chunks))
}
// Verify deduplication - file3.txt and file4.txt have different content
// but we should still have the correct number of unique chunks
allChunks, err := repos.Chunks.List(ctx)
if err != nil {
t.Fatalf("failed to list all chunks: %v", err)
}
// We should have at most 6 chunks (one per unique file content)
// Empty file might not create a chunk
if len(allChunks) > 6 {
t.Errorf("expected at most 6 chunks, got %d", len(allChunks))
}
}
func TestScannerSimpleDirectory(t *testing.T) {
// Initialize logger for tests
log.Initialize(log.Config{})
t.Parallel()
// Create in-memory filesystem
fs := afero.NewMemMapFs()
@@ -74,38 +180,17 @@ func TestScannerSimpleDirectory(t *testing.T) {
Repositories: repos,
MaxBlobSize: int64(1024 * 1024), // 1MB blobs
CompressionLevel: 3,
AgeRecipients: []string{"age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05gl0sjq9q9wjg"}, // Test public key
AgeRecipients: []string{testAgePublicKey},
})
// Create a snapshot record for testing
ctx := context.Background()
snapshotID := "test-snapshot-001"
err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
snapshot := &database.Snapshot{
ID: types.SnapshotID(snapshotID),
Hostname: "test-host",
VaultikVersion: "test",
StartedAt: time.Now(),
CompletedAt: nil,
FileCount: 0,
ChunkCount: 0,
BlobCount: 0,
TotalSize: 0,
BlobSize: 0,
CompressionRatio: 1.0,
}
return repos.Snapshots.Create(ctx, tx, snapshot)
})
if err != nil {
t.Fatalf("failed to create snapshot: %v", err)
}
createTestSnapshotRecord(ctx, t, repos, snapshotID)
// Scan the directory
var result *snapshot.ScanResult
result, err = scanner.Scan(ctx, "/source", snapshotID)
result, err := scanner.Scan(ctx, "/source", snapshotID)
if err != nil {
t.Fatalf("scan failed: %v", err)
}
@@ -120,58 +205,13 @@ func TestScannerSimpleDirectory(t *testing.T) {
t.Errorf("expected at least 97 bytes scanned, got %d", result.BytesScanned)
}
// Verify files in database - includes regular files and directories
files, err := repos.Files.ListByPrefix(ctx, "/source")
if err != nil {
t.Fatalf("failed to list files: %v", err)
}
// 6 regular files + 3 directories (/source, /source/subdir, /source/subdir2)
if len(files) != 9 {
t.Errorf("expected 9 entries in database (6 files + 3 dirs), got %d", len(files))
}
// Verify specific file
file1, err := repos.Files.GetByPath(ctx, "/source/file1.txt")
if err != nil {
t.Fatalf("failed to get file1.txt: %v", err)
}
if file1.Size != 13 {
t.Errorf("expected file1.txt size 13, got %d", file1.Size)
}
if file1.Mode != 0644 {
t.Errorf("expected file1.txt mode 0644, got %o", file1.Mode)
}
// Verify chunks were created
chunks, err := repos.FileChunks.GetByFile(ctx, "/source/file1.txt")
if err != nil {
t.Fatalf("failed to get chunks for file1.txt: %v", err)
}
if len(chunks) != 1 { // Small file should be one chunk
t.Errorf("expected 1 chunk for file1.txt, got %d", len(chunks))
}
// Verify deduplication - file3.txt and file4.txt have different content
// but we should still have the correct number of unique chunks
allChunks, err := repos.Chunks.List(ctx)
if err != nil {
t.Fatalf("failed to list all chunks: %v", err)
}
// We should have at most 6 chunks (one per unique file content)
// Empty file might not create a chunk
if len(allChunks) > 6 {
t.Errorf("expected at most 6 chunks, got %d", len(allChunks))
}
verifySimpleScanDatabase(ctx, t, repos)
}
func TestScannerLargeFile(t *testing.T) {
// Initialize logger for tests
log.Initialize(log.Config{})
t.Parallel()
// Create in-memory filesystem
fs := afero.NewMemMapFs()
@@ -182,14 +222,17 @@ func TestScannerLargeFile(t *testing.T) {
// Fill with pseudo-random data to ensure chunk boundaries
for i := range largeContent {
// Simple pseudo-random generator for deterministic tests
//nolint:gosec // G115: intentional byte truncation of test data
largeContent[i] = byte((i * 7919) ^ (i >> 3))
}
if err := fs.MkdirAll("/source", 0755); err != nil {
err := fs.MkdirAll("/source", 0755)
if err != nil {
t.Fatal(err)
}
if err := afero.WriteFile(fs, "/source/large.bin", largeContent, 0644); err != nil {
err = afero.WriteFile(fs, "/source/large.bin", largeContent, 0644)
if err != nil {
t.Fatal(err)
}
@@ -214,38 +257,17 @@ func TestScannerLargeFile(t *testing.T) {
Repositories: repos,
MaxBlobSize: int64(1024 * 1024),
CompressionLevel: 3,
AgeRecipients: []string{"age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05gl0sjq9q9wjg"}, // Test public key
AgeRecipients: []string{testAgePublicKey},
})
// Create a snapshot record for testing
ctx := context.Background()
snapshotID := "test-snapshot-001"
err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
snapshot := &database.Snapshot{
ID: types.SnapshotID(snapshotID),
Hostname: "test-host",
VaultikVersion: "test",
StartedAt: time.Now(),
CompletedAt: nil,
FileCount: 0,
ChunkCount: 0,
BlobCount: 0,
TotalSize: 0,
BlobSize: 0,
CompressionRatio: 1.0,
}
return repos.Snapshots.Create(ctx, tx, snapshot)
})
if err != nil {
t.Fatalf("failed to create snapshot: %v", err)
}
createTestSnapshotRecord(ctx, t, repos, snapshotID)
// Scan the directory
var result *snapshot.ScanResult
result, err = scanner.Scan(ctx, "/source", snapshotID)
result, err := scanner.Scan(ctx, "/source", snapshotID)
if err != nil {
t.Fatalf("scan failed: %v", err)
}
@@ -257,7 +279,8 @@ func TestScannerLargeFile(t *testing.T) {
// The file size should be at least 1MB
if result.BytesScanned < 1024*1024 {
t.Errorf("expected at least %d bytes scanned, got %d", 1024*1024, result.BytesScanned)
t.Errorf("expected at least %d bytes scanned, got %d",
1024*1024, result.BytesScanned)
}
// Verify chunks

557
internal/snapshot/snapshot.go
View File

@@ -1,3 +1,6 @@
// Package snapshot implements snapshot creation: scanning source
// directories, chunking and deduplicating file data, packing chunks into
// encrypted blobs, and exporting per-snapshot metadata to remote storage.
package snapshot
// Snapshot Metadata Export Process
@@ -58,6 +61,8 @@ import (
)
// SnapshotManager handles snapshot creation and metadata export
//
//nolint:revive // renaming snapshot.SnapshotManager is a cross-package API change
type SnapshotManager struct {
repos *database.Repositories
storage storage.Storer
@@ -66,6 +71,8 @@ type SnapshotManager struct {
}
// SnapshotManagerParams holds dependencies for NewSnapshotManager
//
//nolint:revive // renaming this alongside SnapshotManager is a cross-package API change
type SnapshotManagerParams struct {
fx.In
@@ -88,15 +95,22 @@ func (sm *SnapshotManager) SetFilesystem(fs afero.Fs) {
sm.fs = fs
}
// CreateSnapshot creates a new snapshot record in the database at the start of a backup.
// CreateSnapshot creates a new snapshot record in the database at the
// start of a backup.
//
// Deprecated: Use CreateSnapshotWithName instead for multi-snapshot support.
func (sm *SnapshotManager) CreateSnapshot(ctx context.Context, hostname, version, gitRevision string) (string, error) {
func (sm *SnapshotManager) CreateSnapshot(
ctx context.Context, hostname, version, gitRevision string,
) (string, error) {
return sm.CreateSnapshotWithName(ctx, hostname, "", version, gitRevision)
}
// CreateSnapshotWithName creates a new snapshot record with an optional snapshot name.
// The snapshot ID format is: hostname_name_timestamp or hostname_timestamp if name is empty.
func (sm *SnapshotManager) CreateSnapshotWithName(ctx context.Context, hostname, name, version, gitRevision string) (string, error) {
// CreateSnapshotWithName creates a new snapshot record with an optional
// snapshot name. The snapshot ID format is: hostname_name_timestamp or
// hostname_timestamp if name is empty.
func (sm *SnapshotManager) CreateSnapshotWithName(
ctx context.Context, hostname, name, version, gitRevision string,
) (string, error) {
// Use short hostname (strip domain if present)
shortHostname := hostname
if before, _, ok := strings.Cut(hostname, "."); ok {
@@ -141,7 +155,9 @@ func (sm *SnapshotManager) CreateSnapshotWithName(ctx context.Context, hostname,
}
// UpdateSnapshotStats updates the statistics for a snapshot during backup
func (sm *SnapshotManager) UpdateSnapshotStats(ctx context.Context, snapshotID string, stats BackupStats) error {
func (sm *SnapshotManager) UpdateSnapshotStats(
ctx context.Context, snapshotID string, stats BackupStats,
) error {
err := sm.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
return sm.repos.Snapshots.UpdateCounts(ctx, tx, snapshotID,
int64(stats.FilesScanned),
@@ -160,7 +176,9 @@ func (sm *SnapshotManager) UpdateSnapshotStats(ctx context.Context, snapshotID s
// UpdateSnapshotStatsExtended updates snapshot statistics with extended metrics.
// This includes compression level, uncompressed blob size, and upload duration.
func (sm *SnapshotManager) UpdateSnapshotStatsExtended(ctx context.Context, snapshotID string, stats ExtendedBackupStats) error {
func (sm *SnapshotManager) UpdateSnapshotStatsExtended(
ctx context.Context, snapshotID string, stats ExtendedBackupStats,
) error {
return sm.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
// First update basic stats
err := sm.repos.Snapshots.UpdateCounts(ctx, tx, snapshotID,
@@ -187,7 +205,9 @@ func (sm *SnapshotManager) UpdateSnapshotStatsExtended(ctx context.Context, snap
// is populated with every blob holding any chunk referenced by the
// snapshot's files (including deduplicated blobs uploaded by prior
// snapshots). Without this, fully-deduplicated snapshots are unrestorable.
func (sm *SnapshotManager) CompleteSnapshot(ctx context.Context, snapshotID string) error {
func (sm *SnapshotManager) CompleteSnapshot(
ctx context.Context, snapshotID string,
) error {
err := sm.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
added, err := sm.repos.Snapshots.PopulateReferencedBlobs(ctx, tx, snapshotID)
if err != nil {
@@ -226,8 +246,11 @@ func (sm *SnapshotManager) CompleteSnapshot(ctx context.Context, snapshotID stri
// - Reopening the main database after this method returns
//
// This ensures database consistency during the copy operation.
func (sm *SnapshotManager) ExportSnapshotMetadata(ctx context.Context, dbPath string, snapshotID string) error {
log.Info("Phase 3/3: Exporting snapshot metadata", "snapshot_id", snapshotID, "source_db", dbPath)
func (sm *SnapshotManager) ExportSnapshotMetadata(
ctx context.Context, dbPath string, snapshotID string,
) error {
log.Info("Phase 3/3: Exporting snapshot metadata",
"snapshot_id", snapshotID, "source_db", dbPath)
// Create temp directory for all temporary files
tempDir, err := afero.TempDir(sm.fs, "", "vaultik-snapshot-*")
@@ -258,7 +281,8 @@ func (sm *SnapshotManager) ExportSnapshotMetadata(ctx context.Context, dbPath st
}
// Step 7: Upload to S3 in snapshot subdirectory
if err := sm.uploadSnapshotArtifacts(ctx, snapshotID, finalData, blobManifest); err != nil {
err = sm.uploadSnapshotArtifacts(ctx, snapshotID, finalData, blobManifest)
if err != nil {
return err
}
@@ -270,15 +294,130 @@ func (sm *SnapshotManager) ExportSnapshotMetadata(ctx context.Context, dbPath st
return nil
}
// prepareExportDB copies, cleans, vacuums, and compresses the snapshot database for export.
// Returns the compressed data and the path to the temporary database (needed for manifest generation).
func (sm *SnapshotManager) prepareExportDB(ctx context.Context, dbPath, snapshotID, tempDir string) ([]byte, string, error) {
// CleanupIncompleteSnapshots removes incomplete snapshots that don't have
// metadata in S3. This is critical for data safety: incomplete snapshots
// can cause deduplication to skip files that were never successfully
// backed up, resulting in data loss.
func (sm *SnapshotManager) CleanupIncompleteSnapshots(
ctx context.Context, hostname string,
) error {
log.Info("Checking for incomplete snapshots", "hostname", hostname)
// Get all incomplete snapshots for this hostname
incompleteSnapshots, err := sm.repos.Snapshots.GetIncompleteByHostname(ctx, hostname)
if err != nil {
return fmt.Errorf("getting incomplete snapshots: %w", err)
}
if len(incompleteSnapshots) == 0 {
log.Debug("No incomplete snapshots found")
return nil
}
log.Info("Found incomplete snapshots", "count", len(incompleteSnapshots))
// Check each incomplete snapshot for metadata in storage
for _, snapshot := range incompleteSnapshots {
// Check if metadata exists in storage (paths use the hashed
// remote key so we don't leak host info to the listing).
metadataKey := fmt.Sprintf("metadata/%s/db.zst",
RemoteSnapshotKey(snapshot.ID.String()))
_, err := sm.storage.Stat(ctx, metadataKey)
if err != nil {
// Metadata doesn't exist in S3 - this is an incomplete snapshot
log.Info("Cleaning up incomplete snapshot record",
"snapshot_id", snapshot.ID, "started_at", snapshot.StartedAt)
// Delete the snapshot and all its associations
err := sm.deleteSnapshot(ctx, snapshot.ID.String())
if err != nil {
return fmt.Errorf("deleting incomplete snapshot %s: %w",
snapshot.ID, err)
}
log.Info("Deleted incomplete snapshot record and associated data",
"snapshot_id", snapshot.ID)
} else {
// Metadata exists - this snapshot was completed but database wasn't updated
// This shouldn't happen in normal operation, but mark it complete
log.Warn("Found snapshot with remote metadata but incomplete in database",
"snapshot_id", snapshot.ID)
err := sm.repos.Snapshots.MarkComplete(ctx, nil, snapshot.ID.String())
if err != nil {
log.Error("Failed to mark snapshot as complete in database",
"snapshot_id", snapshot.ID, "error", err)
}
}
}
return nil
}
// CleanupOrphanedData removes files, chunks, and blobs that are no longer
// referenced by any snapshot. This should be called periodically to clean
// up data from deleted or incomplete snapshots.
func (sm *SnapshotManager) CleanupOrphanedData(ctx context.Context) error {
// Order is important to respect foreign key constraints:
// 1. Delete orphaned files (will cascade delete file_chunks)
// 2. Delete orphaned blobs (will cascade delete blob_chunks for deleted blobs)
// 3. Delete orphaned blob_chunks (where blob exists but chunk doesn't)
// 4. Delete orphaned chunks (now safe after all blob_chunks are gone)
// Delete orphaned files (files not in any snapshot)
log.Debug("Deleting orphaned file records from database")
err := sm.repos.Files.DeleteOrphaned(ctx)
if err != nil {
return fmt.Errorf("deleting orphaned files: %w", err)
}
// Delete orphaned blobs (blobs not in any snapshot)
// This will cascade delete blob_chunks for deleted blobs
log.Debug("Deleting orphaned blob records from database")
err = sm.repos.Blobs.DeleteOrphaned(ctx)
if err != nil {
return fmt.Errorf("deleting orphaned blobs: %w", err)
}
// Delete orphaned blob_chunks entries
// This handles cases where the blob still exists but chunks were deleted
log.Debug("Deleting orphaned blob_chunks associations from database")
err = sm.repos.BlobChunks.DeleteOrphaned(ctx)
if err != nil {
return fmt.Errorf("deleting orphaned blob_chunks: %w", err)
}
// Delete orphaned chunks (chunks not referenced by any file)
// This must come after cleaning up blob_chunks to avoid foreign key violations
log.Debug("Deleting orphaned chunk records from database")
err = sm.repos.Chunks.DeleteOrphaned(ctx)
if err != nil {
return fmt.Errorf("deleting orphaned chunks: %w", err)
}
return nil
}
// prepareExportDB copies, cleans, vacuums, and compresses the snapshot
// database for export. Returns the compressed data and the path to the
// temporary database (needed for manifest generation).
func (sm *SnapshotManager) prepareExportDB(
ctx context.Context, dbPath, snapshotID, tempDir string,
) ([]byte, string, error) {
// Step 1: Copy database to temp file
// The main database should be closed at this point
tempDBPath := filepath.Join(tempDir, "snapshot.db")
log.Debug("Copying database to temporary location", "source", dbPath, "destination", tempDBPath)
log.Debug("Copying database to temporary location",
"source", dbPath, "destination", tempDBPath)
if err := sm.copyFile(dbPath, tempDBPath); err != nil {
err := sm.copyFile(dbPath, tempDBPath)
if err != nil {
return nil, "", fmt.Errorf("copying database: %w", err)
}
@@ -294,31 +433,36 @@ func (sm *SnapshotManager) prepareExportDB(ctx context.Context, dbPath, snapshot
log.Info("Temporary database cleanup complete",
"db_path", tempDBPath,
"size_after_clean", humanize.Bytes(uint64(sm.getFileSize(tempDBPath))),
"size_after_clean", humanize.Bytes(safeUint64(sm.getFileSize(tempDBPath))),
"files", stats.FileCount,
"chunks", stats.ChunkCount,
"blobs", stats.BlobCount,
"total_compressed_size", humanize.Bytes(uint64(stats.CompressedSize)),
"total_uncompressed_size", humanize.Bytes(uint64(stats.UncompressedSize)),
"compression_ratio", fmt.Sprintf("%.2fx", float64(stats.UncompressedSize)/float64(stats.CompressedSize)))
"total_compressed_size", humanize.Bytes(safeUint64(stats.CompressedSize)),
"total_uncompressed_size", humanize.Bytes(safeUint64(stats.UncompressedSize)),
"compression_ratio", fmt.Sprintf("%.2fx",
float64(stats.UncompressedSize)/float64(stats.CompressedSize)))
// Step 3: VACUUM the database to remove deleted data and compact
// This is critical for security - ensures no stale/deleted data is uploaded
if err := sm.vacuumDatabase(tempDBPath); err != nil {
err = sm.vacuumDatabase(ctx, tempDBPath)
if err != nil {
return nil, "", fmt.Errorf("vacuuming database: %w", err)
}
log.Debug("Database vacuumed", "size", humanize.Bytes(uint64(sm.getFileSize(tempDBPath))))
log.Debug("Database vacuumed",
"size", humanize.Bytes(safeUint64(sm.getFileSize(tempDBPath))))
// Step 4: Compress and encrypt the binary database file
compressedPath := filepath.Join(tempDir, "db.zst.age")
if err := sm.compressFile(tempDBPath, compressedPath); err != nil {
err = sm.compressFile(tempDBPath, compressedPath)
if err != nil {
return nil, "", fmt.Errorf("compressing database: %w", err)
}
log.Debug("Compression complete",
"original_size", humanize.Bytes(uint64(sm.getFileSize(tempDBPath))),
"compressed_size", humanize.Bytes(uint64(sm.getFileSize(compressedPath))))
"original_size", humanize.Bytes(safeUint64(sm.getFileSize(tempDBPath))),
"compressed_size", humanize.Bytes(safeUint64(sm.getFileSize(compressedPath))))
// Step 5: Read compressed and encrypted data for upload
finalData, err := afero.ReadFile(sm.fs, compressedPath)
@@ -335,7 +479,9 @@ func (sm *SnapshotManager) prepareExportDB(ctx context.Context, dbPath, snapshot
// We never write the human-readable snapshot ID into any unencrypted
// part of remote storage so a listing of the destination bucket leaks
// no host, configuration, or scheduling information.
func (sm *SnapshotManager) uploadSnapshotArtifacts(ctx context.Context, snapshotID string, dbData, manifestData []byte) error {
func (sm *SnapshotManager) uploadSnapshotArtifacts(
ctx context.Context, snapshotID string, dbData, manifestData []byte,
) error {
remoteKey := RemoteSnapshotKey(snapshotID)
// Upload database backup (compressed and encrypted)
@@ -349,7 +495,8 @@ func (sm *SnapshotManager) uploadSnapshotArtifacts(ctx context.Context, snapshot
}
dbUploadDuration := time.Since(dbUploadStart)
dbUploadSpeed := float64(len(dbData)) * 8 / dbUploadDuration.Seconds() // bits per second
// bits per second
dbUploadSpeed := float64(len(dbData)) * bitsPerByte / dbUploadDuration.Seconds()
log.Info("Uploaded snapshot database",
"path", dbKey,
"size", humanize.Bytes(uint64(len(dbData))),
@@ -366,7 +513,9 @@ func (sm *SnapshotManager) uploadSnapshotArtifacts(ctx context.Context, snapshot
}
manifestUploadDuration := time.Since(manifestUploadStart)
manifestUploadSpeed := float64(len(manifestData)) * 8 / manifestUploadDuration.Seconds() // bits per second
// bits per second
manifestUploadSpeed := float64(len(manifestData)) * bitsPerByte /
manifestUploadDuration.Seconds()
log.Info("Uploaded blob manifest",
"path", manifestKey,
"size", humanize.Bytes(uint64(len(manifestData))),
@@ -388,16 +537,19 @@ type CleanupStats struct {
// cleanSnapshotDB removes all data except for the specified snapshot
//
// The cleanup is performed in a specific order to maintain referential integrity:
// 1. Delete other snapshots
// 2. Delete orphaned snapshot associations (snapshot_files, snapshot_blobs) for deleted snapshots
// 3. Delete orphaned files (not in the current snapshot)
// 4. Delete orphaned chunk-to-file mappings (references to deleted files)
// 5. Delete orphaned blobs (not in the current snapshot)
// 6. Delete orphaned blob-to-chunk mappings (references to deleted chunks)
// 7. Delete orphaned chunks (not referenced by any file)
// 1. Delete other snapshots
// 2. Delete orphaned snapshot associations (snapshot_files, snapshot_blobs)
// for deleted snapshots
// 3. Delete orphaned files (not in the current snapshot)
// 4. Delete orphaned chunk-to-file mappings (references to deleted files)
// 5. Delete orphaned blobs (not in the current snapshot)
// 6. Delete orphaned blob-to-chunk mappings (references to deleted chunks)
// 7. Delete orphaned chunks (not referenced by any file)
//
// Each step is implemented as a separate method for clarity and maintainability.
func (sm *SnapshotManager) cleanSnapshotDB(ctx context.Context, dbPath string, snapshotID string) (*CleanupStats, error) {
func (sm *SnapshotManager) cleanSnapshotDB(
ctx context.Context, dbPath string, snapshotID string,
) (*CleanupStats, error) {
// Open the temp database
db, err := database.New(ctx, dbPath)
if err != nil {
@@ -423,48 +575,54 @@ func (sm *SnapshotManager) cleanSnapshotDB(ctx context.Context, dbPath string, s
}()
// Execute cleanup steps in order
if err := sm.deleteOtherSnapshots(ctx, tx, snapshotID); err != nil {
return nil, fmt.Errorf("step 1 - delete other snapshots: %w", err)
steps := []struct {
name string
fn func() error
}{
{"delete other snapshots",
func() error { return sm.deleteOtherSnapshots(ctx, tx, snapshotID) }},
{"delete orphaned snapshot associations",
func() error { return sm.deleteOrphanedSnapshotAssociations(ctx, tx, snapshotID) }},
{"delete orphaned files",
func() error { return sm.deleteOrphanedFiles(ctx, tx, snapshotID) }},
{"delete orphaned chunk-to-file mappings",
func() error { return sm.deleteOrphanedChunkToFileMappings(ctx, tx) }},
{"delete orphaned blobs",
func() error { return sm.deleteOrphanedBlobs(ctx, tx, snapshotID) }},
{"delete orphaned blob-to-chunk mappings",
func() error { return sm.deleteOrphanedBlobToChunkMappings(ctx, tx) }},
{"delete orphaned chunks",
func() error { return sm.deleteOrphanedChunks(ctx, tx) }},
}
if err := sm.deleteOrphanedSnapshotAssociations(ctx, tx, snapshotID); err != nil {
return nil, fmt.Errorf("step 2 - delete orphaned snapshot associations: %w", err)
}
if err := sm.deleteOrphanedFiles(ctx, tx, snapshotID); err != nil {
return nil, fmt.Errorf("step 3 - delete orphaned files: %w", err)
}
if err := sm.deleteOrphanedChunkToFileMappings(ctx, tx); err != nil {
return nil, fmt.Errorf("step 4 - delete orphaned chunk-to-file mappings: %w", err)
}
if err := sm.deleteOrphanedBlobs(ctx, tx, snapshotID); err != nil {
return nil, fmt.Errorf("step 5 - delete orphaned blobs: %w", err)
}
if err := sm.deleteOrphanedBlobToChunkMappings(ctx, tx); err != nil {
return nil, fmt.Errorf("step 6 - delete orphaned blob-to-chunk mappings: %w", err)
}
if err := sm.deleteOrphanedChunks(ctx, tx); err != nil {
return nil, fmt.Errorf("step 7 - delete orphaned chunks: %w", err)
for i, step := range steps {
err = step.fn()
if err != nil {
return nil, fmt.Errorf("step %d - %s: %w", i+1, step.name, err)
}
}
// Commit transaction
log.Debug("[Temp DB Cleanup] Committing cleanup transaction")
if err := tx.Commit(); err != nil {
err = tx.Commit()
if err != nil {
return nil, fmt.Errorf("committing transaction: %w", err)
}
// Collect statistics about the cleaned database
return sm.collectCleanupStats(ctx, db, snapshotID)
}
// collectCleanupStats gathers statistics about the cleaned database.
func (sm *SnapshotManager) collectCleanupStats(
ctx context.Context, db *database.DB, snapshotID string,
) (*CleanupStats, error) {
stats := &CleanupStats{}
// Count files
var fileCount int
err = db.QueryRowWithLog(ctx, "SELECT COUNT(*) FROM files").Scan(&fileCount)
err := db.QueryRowWithLog(ctx, "SELECT COUNT(*) FROM files").Scan(&fileCount)
if err != nil {
return nil, fmt.Errorf("counting files: %w", err)
}
@@ -488,9 +646,12 @@ func (sm *SnapshotManager) cleanSnapshotDB(ctx context.Context, dbPath string, s
)
err = db.QueryRowWithLog(ctx, `
SELECT COUNT(*), COALESCE(SUM(compressed_size), 0), COALESCE(SUM(uncompressed_size), 0)
FROM blobs
WHERE blob_hash IN (SELECT blob_hash FROM snapshot_blobs WHERE snapshot_id = ?)
SELECT COUNT(*),
COALESCE(SUM(compressed_size), 0),
COALESCE(SUM(uncompressed_size), 0)
FROM blobs
WHERE blob_hash IN
(SELECT blob_hash FROM snapshot_blobs WHERE snapshot_id = ?)
`, snapshotID).Scan(&blobCount, &compressedSize, &uncompressedSize)
if err != nil {
return nil, fmt.Errorf("counting blobs and sizes: %w", err)
@@ -505,11 +666,13 @@ func (sm *SnapshotManager) cleanSnapshotDB(ctx context.Context, dbPath string, s
// vacuumDatabase runs VACUUM on the database to remove deleted data and compact
// This is critical for security - ensures no stale/deleted data pages are uploaded
func (sm *SnapshotManager) vacuumDatabase(dbPath string) error {
func (sm *SnapshotManager) vacuumDatabase(ctx context.Context, dbPath string) error {
log.Debug("Running VACUUM on database", "path", dbPath)
cmd := exec.Command("sqlite3", dbPath, "VACUUM;")
//nolint:gosec // G204: fixed argv; dbPath is our own temp file path
cmd := exec.CommandContext(ctx, "sqlite3", dbPath, "VACUUM;")
if output, err := cmd.CombinedOutput(); err != nil {
output, err := cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("running VACUUM: %w (output: %s)", err, string(output))
}
@@ -543,7 +706,8 @@ func (sm *SnapshotManager) compressFile(inputPath, outputPath string) error {
// Use blobgen for compression and encryption
log.Debug("Compressing and encrypting data")
writer, err := blobgen.NewWriter(output, sm.config.CompressionLevel, sm.config.AgeRecipients)
writer, err := blobgen.NewWriter(output, sm.config.CompressionLevel,
sm.config.AgeRecipients)
if err != nil {
return fmt.Errorf("creating blobgen writer: %w", err)
}
@@ -559,12 +723,14 @@ func (sm *SnapshotManager) compressFile(inputPath, outputPath string) error {
}
}()
if _, err := io.Copy(writer, input); err != nil {
_, err = io.Copy(writer, input)
if err != nil {
return fmt.Errorf("compressing data: %w", err)
}
// Close writer to flush all data
if err := writer.Close(); err != nil {
err = writer.Close()
if err != nil {
return fmt.Errorf("closing writer: %w", err)
}
@@ -620,7 +786,9 @@ func (sm *SnapshotManager) copyFile(src, dst string) error {
}
// generateBlobManifest creates a compressed JSON list of all blobs in the snapshot
func (sm *SnapshotManager) generateBlobManifest(ctx context.Context, dbPath string, snapshotID string) ([]byte, error) {
func (sm *SnapshotManager) generateBlobManifest(
ctx context.Context, dbPath string, snapshotID string,
) ([]byte, error) {
// Open the cleaned database using the database package
db, err := database.New(ctx, dbPath)
if err != nil {
@@ -718,61 +886,10 @@ type ExtendedBackupStats struct {
UploadDurationMs int64 // Total milliseconds spent uploading to S3
}
// CleanupIncompleteSnapshots removes incomplete snapshots that don't have metadata in S3.
// This is critical for data safety: incomplete snapshots can cause deduplication to skip
// files that were never successfully backed up, resulting in data loss.
func (sm *SnapshotManager) CleanupIncompleteSnapshots(ctx context.Context, hostname string) error {
log.Info("Checking for incomplete snapshots", "hostname", hostname)
// Get all incomplete snapshots for this hostname
incompleteSnapshots, err := sm.repos.Snapshots.GetIncompleteByHostname(ctx, hostname)
if err != nil {
return fmt.Errorf("getting incomplete snapshots: %w", err)
}
if len(incompleteSnapshots) == 0 {
log.Debug("No incomplete snapshots found")
return nil
}
log.Info("Found incomplete snapshots", "count", len(incompleteSnapshots))
// Check each incomplete snapshot for metadata in storage
for _, snapshot := range incompleteSnapshots {
// Check if metadata exists in storage (paths use the hashed
// remote key so we don't leak host info to the listing).
metadataKey := fmt.Sprintf("metadata/%s/db.zst", RemoteSnapshotKey(snapshot.ID.String()))
_, err := sm.storage.Stat(ctx, metadataKey)
if err != nil {
// Metadata doesn't exist in S3 - this is an incomplete snapshot
log.Info("Cleaning up incomplete snapshot record", "snapshot_id", snapshot.ID, "started_at", snapshot.StartedAt)
// Delete the snapshot and all its associations
err := sm.deleteSnapshot(ctx, snapshot.ID.String())
if err != nil {
return fmt.Errorf("deleting incomplete snapshot %s: %w", snapshot.ID, err)
}
log.Info("Deleted incomplete snapshot record and associated data", "snapshot_id", snapshot.ID)
} else {
// Metadata exists - this snapshot was completed but database wasn't updated
// This shouldn't happen in normal operation, but mark it complete
log.Warn("Found snapshot with remote metadata but incomplete in database", "snapshot_id", snapshot.ID)
err := sm.repos.Snapshots.MarkComplete(ctx, nil, snapshot.ID.String())
if err != nil {
log.Error("Failed to mark snapshot as complete in database", "snapshot_id", snapshot.ID, "error", err)
}
}
}
return nil
}
// deleteSnapshot removes a snapshot and all its associations from the database
func (sm *SnapshotManager) deleteSnapshot(ctx context.Context, snapshotID string) error {
func (sm *SnapshotManager) deleteSnapshot(
ctx context.Context, snapshotID string,
) error {
// Delete snapshot_files entries
err := sm.repos.Snapshots.DeleteSnapshotFiles(ctx, snapshotID)
if err != nil {
@@ -808,61 +925,20 @@ func (sm *SnapshotManager) deleteSnapshot(ctx context.Context, snapshotID string
return nil
}
// CleanupOrphanedData removes files, chunks, and blobs that are no longer referenced by any snapshot.
// This should be called periodically to clean up data from deleted or incomplete snapshots.
func (sm *SnapshotManager) CleanupOrphanedData(ctx context.Context) error {
// Order is important to respect foreign key constraints:
// 1. Delete orphaned files (will cascade delete file_chunks)
// 2. Delete orphaned blobs (will cascade delete blob_chunks for deleted blobs)
// 3. Delete orphaned blob_chunks (where blob exists but chunk doesn't)
// 4. Delete orphaned chunks (now safe after all blob_chunks are gone)
// Delete orphaned files (files not in any snapshot)
log.Debug("Deleting orphaned file records from database")
err := sm.repos.Files.DeleteOrphaned(ctx)
if err != nil {
return fmt.Errorf("deleting orphaned files: %w", err)
}
// Delete orphaned blobs (blobs not in any snapshot)
// This will cascade delete blob_chunks for deleted blobs
log.Debug("Deleting orphaned blob records from database")
err = sm.repos.Blobs.DeleteOrphaned(ctx)
if err != nil {
return fmt.Errorf("deleting orphaned blobs: %w", err)
}
// Delete orphaned blob_chunks entries
// This handles cases where the blob still exists but chunks were deleted
log.Debug("Deleting orphaned blob_chunks associations from database")
err = sm.repos.BlobChunks.DeleteOrphaned(ctx)
if err != nil {
return fmt.Errorf("deleting orphaned blob_chunks: %w", err)
}
// Delete orphaned chunks (chunks not referenced by any file)
// This must come after cleaning up blob_chunks to avoid foreign key violations
log.Debug("Deleting orphaned chunk records from database")
err = sm.repos.Chunks.DeleteOrphaned(ctx)
if err != nil {
return fmt.Errorf("deleting orphaned chunks: %w", err)
}
return nil
}
// deleteOtherSnapshots deletes all snapshots except the current one
func (sm *SnapshotManager) deleteOtherSnapshots(ctx context.Context, tx *sql.Tx, currentSnapshotID string) error {
log.Debug("[Temp DB Cleanup] Deleting all snapshot records except current", "keeping", currentSnapshotID)
func (sm *SnapshotManager) deleteOtherSnapshots(
ctx context.Context, tx *sql.Tx, currentSnapshotID string,
) error {
log.Debug("[Temp DB Cleanup] Deleting all snapshot records except current",
"keeping", currentSnapshotID)
// First delete uploads that reference other snapshots (no CASCADE DELETE on this FK)
database.LogSQL("Execute", "DELETE FROM uploads WHERE snapshot_id != ?", currentSnapshotID)
// First delete uploads that reference other snapshots (no CASCADE DELETE
// on this FK)
database.LogSQL("Execute", "DELETE FROM uploads WHERE snapshot_id != ?",
currentSnapshotID)
uploadResult, err := tx.ExecContext(ctx, "DELETE FROM uploads WHERE snapshot_id != ?", currentSnapshotID)
uploadResult, err := tx.ExecContext(ctx,
"DELETE FROM uploads WHERE snapshot_id != ?", currentSnapshotID)
if err != nil {
return fmt.Errorf("deleting uploads for other snapshots: %w", err)
}
@@ -871,66 +947,84 @@ func (sm *SnapshotManager) deleteOtherSnapshots(ctx context.Context, tx *sql.Tx,
log.Debug("[Temp DB Cleanup] Deleted upload records", "count", uploadsDeleted)
// Now we can safely delete the snapshots
database.LogSQL("Execute", "DELETE FROM snapshots WHERE id != ?", currentSnapshotID)
database.LogSQL("Execute", "DELETE FROM snapshots WHERE id != ?",
currentSnapshotID)
result, err := tx.ExecContext(ctx, "DELETE FROM snapshots WHERE id != ?", currentSnapshotID)
result, err := tx.ExecContext(ctx, "DELETE FROM snapshots WHERE id != ?",
currentSnapshotID)
if err != nil {
return fmt.Errorf("deleting other snapshots: %w", err)
}
rowsAffected, _ := result.RowsAffected()
log.Debug("[Temp DB Cleanup] Deleted snapshot records from database", "count", rowsAffected)
log.Debug("[Temp DB Cleanup] Deleted snapshot records from database",
"count", rowsAffected)
return nil
}
// deleteOrphanedSnapshotAssociations deletes snapshot_files and snapshot_blobs for deleted snapshots
func (sm *SnapshotManager) deleteOrphanedSnapshotAssociations(ctx context.Context, tx *sql.Tx, currentSnapshotID string) error {
// deleteOrphanedSnapshotAssociations deletes snapshot_files and
// snapshot_blobs for deleted snapshots
func (sm *SnapshotManager) deleteOrphanedSnapshotAssociations(
ctx context.Context, tx *sql.Tx, currentSnapshotID string,
) error {
// Delete orphaned snapshot_files
log.Debug("[Temp DB Cleanup] Deleting orphaned snapshot_files associations")
database.LogSQL("Execute", "DELETE FROM snapshot_files WHERE snapshot_id != ?", currentSnapshotID)
database.LogSQL("Execute", "DELETE FROM snapshot_files WHERE snapshot_id != ?",
currentSnapshotID)
result, err := tx.ExecContext(ctx, "DELETE FROM snapshot_files WHERE snapshot_id != ?", currentSnapshotID)
result, err := tx.ExecContext(ctx,
"DELETE FROM snapshot_files WHERE snapshot_id != ?", currentSnapshotID)
if err != nil {
return fmt.Errorf("deleting orphaned snapshot_files: %w", err)
}
rowsAffected, _ := result.RowsAffected()
log.Debug("[Temp DB Cleanup] Deleted snapshot_files associations", "count", rowsAffected)
log.Debug("[Temp DB Cleanup] Deleted snapshot_files associations",
"count", rowsAffected)
// Delete orphaned snapshot_blobs
log.Debug("[Temp DB Cleanup] Deleting orphaned snapshot_blobs associations")
database.LogSQL("Execute", "DELETE FROM snapshot_blobs WHERE snapshot_id != ?", currentSnapshotID)
database.LogSQL("Execute", "DELETE FROM snapshot_blobs WHERE snapshot_id != ?",
currentSnapshotID)
result, err = tx.ExecContext(ctx, "DELETE FROM snapshot_blobs WHERE snapshot_id != ?", currentSnapshotID)
result, err = tx.ExecContext(ctx,
"DELETE FROM snapshot_blobs WHERE snapshot_id != ?", currentSnapshotID)
if err != nil {
return fmt.Errorf("deleting orphaned snapshot_blobs: %w", err)
}
rowsAffected, _ = result.RowsAffected()
log.Debug("[Temp DB Cleanup] Deleted snapshot_blobs associations", "count", rowsAffected)
log.Debug("[Temp DB Cleanup] Deleted snapshot_blobs associations",
"count", rowsAffected)
return nil
}
// deleteOrphanedFiles deletes files not in the current snapshot
func (sm *SnapshotManager) deleteOrphanedFiles(ctx context.Context, tx *sql.Tx, currentSnapshotID string) error {
log.Debug("[Temp DB Cleanup] Deleting file records not referenced by current snapshot")
database.LogSQL("Execute", `DELETE FROM files WHERE NOT EXISTS (SELECT 1 FROM snapshot_files WHERE snapshot_files.file_id = files.id AND snapshot_files.snapshot_id = ?)`, currentSnapshotID)
func (sm *SnapshotManager) deleteOrphanedFiles(
ctx context.Context, tx *sql.Tx, currentSnapshotID string,
) error {
log.Debug(
"[Temp DB Cleanup] Deleting file records not referenced by current snapshot")
result, err := tx.ExecContext(ctx, `
DELETE FROM files
query := `
DELETE FROM files
WHERE NOT EXISTS (
SELECT 1 FROM snapshot_files
WHERE snapshot_files.file_id = files.id
SELECT 1 FROM snapshot_files
WHERE snapshot_files.file_id = files.id
AND snapshot_files.snapshot_id = ?
)`, currentSnapshotID)
)`
database.LogSQL("Execute", query, currentSnapshotID)
result, err := tx.ExecContext(ctx, query, currentSnapshotID)
if err != nil {
return fmt.Errorf("deleting orphaned files: %w", err)
}
rowsAffected, _ := result.RowsAffected()
log.Debug("[Temp DB Cleanup] Deleted file records from database", "count", rowsAffected)
log.Debug("[Temp DB Cleanup] Deleted file records from database",
"count", rowsAffected)
// Note: file_chunks will be deleted via CASCADE
log.Debug("[Temp DB Cleanup] file_chunks associations deleted via CASCADE")
@@ -939,65 +1033,81 @@ func (sm *SnapshotManager) deleteOrphanedFiles(ctx context.Context, tx *sql.Tx,
}
// deleteOrphanedChunkToFileMappings deletes chunk_files entries for deleted files
func (sm *SnapshotManager) deleteOrphanedChunkToFileMappings(ctx context.Context, tx *sql.Tx) error {
func (sm *SnapshotManager) deleteOrphanedChunkToFileMappings(
ctx context.Context, tx *sql.Tx,
) error {
log.Debug("[Temp DB Cleanup] Deleting orphaned chunk_files associations")
database.LogSQL("Execute", `DELETE FROM chunk_files WHERE NOT EXISTS (SELECT 1 FROM files WHERE files.id = chunk_files.file_id)`)
result, err := tx.ExecContext(ctx, `
DELETE FROM chunk_files
query := `
DELETE FROM chunk_files
WHERE NOT EXISTS (
SELECT 1 FROM files
SELECT 1 FROM files
WHERE files.id = chunk_files.file_id
)`)
)`
database.LogSQL("Execute", query)
result, err := tx.ExecContext(ctx, query)
if err != nil {
return fmt.Errorf("deleting orphaned chunk_files: %w", err)
}
rowsAffected, _ := result.RowsAffected()
log.Debug("[Temp DB Cleanup] Deleted chunk_files associations", "count", rowsAffected)
log.Debug("[Temp DB Cleanup] Deleted chunk_files associations",
"count", rowsAffected)
return nil
}
// deleteOrphanedBlobs deletes blobs not in the current snapshot
func (sm *SnapshotManager) deleteOrphanedBlobs(ctx context.Context, tx *sql.Tx, currentSnapshotID string) error {
log.Debug("[Temp DB Cleanup] Deleting blob records not referenced by current snapshot")
database.LogSQL("Execute", `DELETE FROM blobs WHERE NOT EXISTS (SELECT 1 FROM snapshot_blobs WHERE snapshot_blobs.blob_hash = blobs.blob_hash AND snapshot_blobs.snapshot_id = ?)`, currentSnapshotID)
func (sm *SnapshotManager) deleteOrphanedBlobs(
ctx context.Context, tx *sql.Tx, currentSnapshotID string,
) error {
log.Debug(
"[Temp DB Cleanup] Deleting blob records not referenced by current snapshot")
result, err := tx.ExecContext(ctx, `
DELETE FROM blobs
query := `
DELETE FROM blobs
WHERE NOT EXISTS (
SELECT 1 FROM snapshot_blobs
WHERE snapshot_blobs.blob_hash = blobs.blob_hash
SELECT 1 FROM snapshot_blobs
WHERE snapshot_blobs.blob_hash = blobs.blob_hash
AND snapshot_blobs.snapshot_id = ?
)`, currentSnapshotID)
)`
database.LogSQL("Execute", query, currentSnapshotID)
result, err := tx.ExecContext(ctx, query, currentSnapshotID)
if err != nil {
return fmt.Errorf("deleting orphaned blobs: %w", err)
}
rowsAffected, _ := result.RowsAffected()
log.Debug("[Temp DB Cleanup] Deleted blob records from database", "count", rowsAffected)
log.Debug("[Temp DB Cleanup] Deleted blob records from database",
"count", rowsAffected)
return nil
}
// deleteOrphanedBlobToChunkMappings deletes blob_chunks entries for deleted blobs
func (sm *SnapshotManager) deleteOrphanedBlobToChunkMappings(ctx context.Context, tx *sql.Tx) error {
func (sm *SnapshotManager) deleteOrphanedBlobToChunkMappings(
ctx context.Context, tx *sql.Tx,
) error {
log.Debug("[Temp DB Cleanup] Deleting orphaned blob_chunks associations")
database.LogSQL("Execute", `DELETE FROM blob_chunks WHERE NOT EXISTS (SELECT 1 FROM blobs WHERE blobs.id = blob_chunks.blob_id)`)
result, err := tx.ExecContext(ctx, `
DELETE FROM blob_chunks
query := `
DELETE FROM blob_chunks
WHERE NOT EXISTS (
SELECT 1 FROM blobs
SELECT 1 FROM blobs
WHERE blobs.id = blob_chunks.blob_id
)`)
)`
database.LogSQL("Execute", query)
result, err := tx.ExecContext(ctx, query)
if err != nil {
return fmt.Errorf("deleting orphaned blob_chunks: %w", err)
}
rowsAffected, _ := result.RowsAffected()
log.Debug("[Temp DB Cleanup] Deleted blob_chunks associations", "count", rowsAffected)
log.Debug("[Temp DB Cleanup] Deleted blob_chunks associations",
"count", rowsAffected)
return nil
}
@@ -1024,7 +1134,8 @@ func (sm *SnapshotManager) deleteOrphanedChunks(ctx context.Context, tx *sql.Tx)
}
rowsAffected, _ := result.RowsAffected()
log.Debug("[Temp DB Cleanup] Deleted chunk records from database", "count", rowsAffected)
log.Debug("[Temp DB Cleanup] Deleted chunk records from database",
"count", rowsAffected)
return nil
}

119
internal/snapshot/snapshot_test.go
View File

@@ -1,3 +1,4 @@
//nolint:testpackage // exercises unexported SnapshotManager internals
package snapshot
import (
@@ -37,9 +38,65 @@ func copyFile(fs afero.Fs, src, dst string) error {
return err
}
// verifyCleanedDB opens the cleaned database and checks that the kept
// snapshot survived while the orphan file and chunk were removed.
func verifyCleanedDB(
ctx context.Context,
t *testing.T,
tempDBPath, snapshotID string,
file *database.File,
chunk *database.Chunk,
) {
t.Helper()
cleanedDB, err := database.New(ctx, tempDBPath)
if err != nil {
t.Fatalf("failed to open cleaned database: %v", err)
}
defer func() {
err := cleanedDB.Close()
if err != nil {
t.Errorf("failed to close database: %v", err)
}
}()
cleanedRepos := database.NewRepositories(cleanedDB)
// Verify snapshot exists
verifySnapshot, err := cleanedRepos.Snapshots.GetByID(ctx, snapshotID)
if err != nil {
t.Fatalf("failed to get snapshot: %v", err)
}
if verifySnapshot == nil {
t.Error("snapshot should exist")
}
// Verify orphan file is gone
f, err := cleanedRepos.Files.GetByPath(ctx, file.Path.String())
if err != nil {
t.Fatalf("failed to check file: %v", err)
}
if f != nil {
t.Error("orphan file should not exist")
}
// Verify orphan chunk is gone
c, err := cleanedRepos.Chunks.GetByHash(ctx, chunk.ChunkHash.String())
if err != nil {
t.Fatalf("failed to check chunk: %v", err)
}
if c != nil {
t.Error("orphan chunk should not exist")
}
}
func TestCleanSnapshotDBEmptySnapshot(t *testing.T) {
// Initialize logger
log.Initialize(log.Config{})
t.Parallel()
ctx := context.Background()
fs := afero.NewOsFs()
@@ -85,13 +142,16 @@ func TestCleanSnapshotDBEmptySnapshot(t *testing.T) {
}
// Close the database
if err := db.Close(); err != nil {
err = db.Close()
if err != nil {
t.Fatalf("failed to close database: %v", err)
}
// Copy database
tempDBPath := filepath.Join(tempDir, "temp.db")
if err := copyFile(fs, dbPath, tempDBPath); err != nil {
err = copyFile(fs, dbPath, tempDBPath)
if err != nil {
t.Fatalf("failed to copy database: %v", err)
}
@@ -105,58 +165,20 @@ func TestCleanSnapshotDBEmptySnapshot(t *testing.T) {
config: cfg,
fs: fs,
}
if _, err := sm.cleanSnapshotDB(ctx, tempDBPath, snapshot.ID.String()); err != nil {
_, err = sm.cleanSnapshotDB(ctx, tempDBPath, snapshot.ID.String())
if err != nil {
t.Fatalf("failed to clean snapshot database: %v", err)
}
// Verify the cleaned database
cleanedDB, err := database.New(ctx, tempDBPath)
if err != nil {
t.Fatalf("failed to open cleaned database: %v", err)
}
defer func() {
err := cleanedDB.Close()
if err != nil {
t.Errorf("failed to close database: %v", err)
}
}()
cleanedRepos := database.NewRepositories(cleanedDB)
// Verify snapshot exists
verifySnapshot, err := cleanedRepos.Snapshots.GetByID(ctx, snapshot.ID.String())
if err != nil {
t.Fatalf("failed to get snapshot: %v", err)
}
if verifySnapshot == nil {
t.Error("snapshot should exist")
}
// Verify orphan file is gone
f, err := cleanedRepos.Files.GetByPath(ctx, file.Path.String())
if err != nil {
t.Fatalf("failed to check file: %v", err)
}
if f != nil {
t.Error("orphan file should not exist")
}
// Verify orphan chunk is gone
c, err := cleanedRepos.Chunks.GetByHash(ctx, chunk.ChunkHash.String())
if err != nil {
t.Fatalf("failed to check chunk: %v", err)
}
if c != nil {
t.Error("orphan chunk should not exist")
}
verifyCleanedDB(ctx, t, tempDBPath, snapshot.ID.String(), file, chunk)
}
func TestCleanSnapshotDBNonExistentSnapshot(t *testing.T) {
// Initialize logger
log.Initialize(log.Config{})
t.Parallel()
ctx := context.Background()
fs := afero.NewOsFs()
@@ -171,13 +193,16 @@ func TestCleanSnapshotDBNonExistentSnapshot(t *testing.T) {
}
// Close immediately
if err := db.Close(); err != nil {
err = db.Close()
if err != nil {
t.Fatalf("failed to close database: %v", err)
}
// Copy database
tempDBPath := filepath.Join(tempDir, "temp.db")
if err := copyFile(fs, dbPath, tempDBPath); err != nil {
err = copyFile(fs, dbPath, tempDBPath)
if err != nil {
t.Fatalf("failed to copy database: %v", err)
}