sneak
417b25a5f5
- Add internal/types package with type-safe wrappers for IDs, hashes, paths, and credentials (FileID, BlobID, ChunkHash, etc.) - Implement driver.Valuer and sql.Scanner for UUID-based types - Add `vaultik version` command showing version, commit, go version - Add `--verify` flag to restore command that checksums all restored files against expected chunk hashes with progress bar - Remove fetch.go (dead code, functionality in restore) - Clean up TODO.md, remove completed items - Update all database and snapshot code to use new custom types
455 lines
15 KiB
Go
455 lines
15 KiB
Go
package snapshot_test
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"path/filepath"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.eeqj.de/sneak/vaultik/internal/database"
|
|
"git.eeqj.de/sneak/vaultik/internal/log"
|
|
"git.eeqj.de/sneak/vaultik/internal/snapshot"
|
|
"git.eeqj.de/sneak/vaultik/internal/types"
|
|
"github.com/spf13/afero"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
func setupExcludeTestFS(t *testing.T) afero.Fs {
|
|
t.Helper()
|
|
|
|
// Create in-memory filesystem
|
|
fs := afero.NewMemMapFs()
|
|
|
|
// Create test directory structure:
|
|
// /backup/
|
|
// file1.txt (should be backed up)
|
|
// file2.log (should be excluded if *.log is in patterns)
|
|
// .git/
|
|
// config (should be excluded if .git is in patterns)
|
|
// objects/
|
|
// pack/
|
|
// data.pack (should be excluded if .git is in patterns)
|
|
// src/
|
|
// main.go (should be backed up)
|
|
// test.go (should be backed up)
|
|
// node_modules/
|
|
// package/
|
|
// index.js (should be excluded if node_modules is in patterns)
|
|
// cache/
|
|
// temp.dat (should be excluded if cache/ is in patterns)
|
|
// build/
|
|
// output.bin (should be excluded if build is in patterns)
|
|
// docs/
|
|
// readme.md (should be backed up)
|
|
// .DS_Store (should be excluded if .DS_Store is in patterns)
|
|
// thumbs.db (should be excluded if thumbs.db is in patterns)
|
|
|
|
files := map[string]string{
|
|
"/backup/file1.txt": "content1",
|
|
"/backup/file2.log": "log content",
|
|
"/backup/.git/config": "git config",
|
|
"/backup/.git/objects/pack/data.pack": "pack data",
|
|
"/backup/src/main.go": "package main",
|
|
"/backup/src/test.go": "package main_test",
|
|
"/backup/node_modules/package/index.js": "module.exports = {}",
|
|
"/backup/cache/temp.dat": "cached data",
|
|
"/backup/build/output.bin": "binary data",
|
|
"/backup/docs/readme.md": "# Documentation",
|
|
"/backup/.DS_Store": "ds store data",
|
|
"/backup/thumbs.db": "thumbs data",
|
|
"/backup/src/.hidden": "hidden file",
|
|
"/backup/important.log.bak": "backup of log",
|
|
}
|
|
|
|
testTime := time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC)
|
|
for path, content := range files {
|
|
dir := filepath.Dir(path)
|
|
err := fs.MkdirAll(dir, 0755)
|
|
require.NoError(t, err)
|
|
err = afero.WriteFile(fs, path, []byte(content), 0644)
|
|
require.NoError(t, err)
|
|
err = fs.Chtimes(path, testTime, testTime)
|
|
require.NoError(t, err)
|
|
}
|
|
|
|
return fs
|
|
}
|
|
|
|
func createTestScanner(t *testing.T, fs afero.Fs, excludePatterns []string) (*snapshot.Scanner, *database.Repositories, func()) {
|
|
t.Helper()
|
|
|
|
// Initialize logger
|
|
log.Initialize(log.Config{})
|
|
|
|
// Create test database
|
|
db, err := database.NewTestDB()
|
|
require.NoError(t, err)
|
|
|
|
repos := database.NewRepositories(db)
|
|
|
|
scanner := snapshot.NewScanner(snapshot.ScannerConfig{
|
|
FS: fs,
|
|
ChunkSize: 64 * 1024,
|
|
Repositories: repos,
|
|
MaxBlobSize: 1024 * 1024,
|
|
CompressionLevel: 3,
|
|
AgeRecipients: []string{"age1ql3z7hjy54pw3hyww5ayyfg7zqgvc7w3j2elw8zmrj2kg5sfn9aqmcac8p"},
|
|
Exclude: excludePatterns,
|
|
})
|
|
|
|
cleanup := func() {
|
|
_ = db.Close()
|
|
}
|
|
|
|
return scanner, repos, cleanup
|
|
}
|
|
|
|
func createSnapshotRecord(t *testing.T, ctx context.Context, repos *database.Repositories, snapshotID string) {
|
|
t.Helper()
|
|
err := repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
|
|
snap := &database.Snapshot{
|
|
ID: types.SnapshotID(snapshotID),
|
|
Hostname: "test-host",
|
|
VaultikVersion: "test",
|
|
StartedAt: time.Now(),
|
|
CompletedAt: nil,
|
|
FileCount: 0,
|
|
ChunkCount: 0,
|
|
BlobCount: 0,
|
|
TotalSize: 0,
|
|
BlobSize: 0,
|
|
CompressionRatio: 1.0,
|
|
}
|
|
return repos.Snapshots.Create(ctx, tx, snap)
|
|
})
|
|
require.NoError(t, err)
|
|
}
|
|
|
|
func TestExcludePatterns_ExcludeGitDirectory(t *testing.T) {
|
|
fs := setupExcludeTestFS(t)
|
|
scanner, repos, cleanup := createTestScanner(t, fs, []string{".git"})
|
|
defer cleanup()
|
|
require.NotNil(t, scanner)
|
|
|
|
ctx := context.Background()
|
|
createSnapshotRecord(t, ctx, repos, "test-snapshot")
|
|
|
|
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
|
|
require.NoError(t, err)
|
|
|
|
// Should have scanned files but NOT .git directory contents
|
|
// Expected: file1.txt, file2.log, src/main.go, src/test.go, node_modules/package/index.js,
|
|
// cache/temp.dat, build/output.bin, docs/readme.md, .DS_Store, thumbs.db,
|
|
// src/.hidden, important.log.bak
|
|
// Excluded: .git/config, .git/objects/pack/data.pack
|
|
require.Equal(t, 12, result.FilesScanned, "Should exclude .git directory contents")
|
|
}
|
|
|
|
func TestExcludePatterns_ExcludeByExtension(t *testing.T) {
|
|
fs := setupExcludeTestFS(t)
|
|
scanner, repos, cleanup := createTestScanner(t, fs, []string{"*.log"})
|
|
defer cleanup()
|
|
require.NotNil(t, scanner)
|
|
|
|
ctx := context.Background()
|
|
createSnapshotRecord(t, ctx, repos, "test-snapshot")
|
|
|
|
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
|
|
require.NoError(t, err)
|
|
|
|
// Should exclude file2.log but NOT important.log.bak (different extension)
|
|
// Total files: 14, excluded: 1 (file2.log)
|
|
require.Equal(t, 13, result.FilesScanned, "Should exclude *.log files")
|
|
}
|
|
|
|
func TestExcludePatterns_ExcludeNodeModules(t *testing.T) {
|
|
fs := setupExcludeTestFS(t)
|
|
scanner, repos, cleanup := createTestScanner(t, fs, []string{"node_modules"})
|
|
defer cleanup()
|
|
require.NotNil(t, scanner)
|
|
|
|
ctx := context.Background()
|
|
createSnapshotRecord(t, ctx, repos, "test-snapshot")
|
|
|
|
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
|
|
require.NoError(t, err)
|
|
|
|
// Should exclude node_modules/package/index.js
|
|
// Total files: 14, excluded: 1
|
|
require.Equal(t, 13, result.FilesScanned, "Should exclude node_modules directory")
|
|
}
|
|
|
|
func TestExcludePatterns_MultiplePatterns(t *testing.T) {
|
|
fs := setupExcludeTestFS(t)
|
|
scanner, repos, cleanup := createTestScanner(t, fs, []string{".git", "node_modules", "*.log", ".DS_Store", "thumbs.db", "cache", "build"})
|
|
defer cleanup()
|
|
require.NotNil(t, scanner)
|
|
|
|
ctx := context.Background()
|
|
createSnapshotRecord(t, ctx, repos, "test-snapshot")
|
|
|
|
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
|
|
require.NoError(t, err)
|
|
|
|
// Should only have: file1.txt, src/main.go, src/test.go, docs/readme.md, src/.hidden, important.log.bak
|
|
// Excluded: .git/*, node_modules/*, *.log (file2.log), .DS_Store, thumbs.db, cache/*, build/*
|
|
require.Equal(t, 6, result.FilesScanned, "Should exclude multiple patterns")
|
|
}
|
|
|
|
func TestExcludePatterns_NoExclusions(t *testing.T) {
|
|
fs := setupExcludeTestFS(t)
|
|
scanner, repos, cleanup := createTestScanner(t, fs, []string{})
|
|
defer cleanup()
|
|
require.NotNil(t, scanner)
|
|
|
|
ctx := context.Background()
|
|
createSnapshotRecord(t, ctx, repos, "test-snapshot")
|
|
|
|
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
|
|
require.NoError(t, err)
|
|
|
|
// Should scan all 14 files
|
|
require.Equal(t, 14, result.FilesScanned, "Should scan all files when no exclusions")
|
|
}
|
|
|
|
func TestExcludePatterns_ExcludeHiddenFiles(t *testing.T) {
|
|
fs := setupExcludeTestFS(t)
|
|
scanner, repos, cleanup := createTestScanner(t, fs, []string{".*"})
|
|
defer cleanup()
|
|
require.NotNil(t, scanner)
|
|
|
|
ctx := context.Background()
|
|
createSnapshotRecord(t, ctx, repos, "test-snapshot")
|
|
|
|
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
|
|
require.NoError(t, err)
|
|
|
|
// Should exclude: .git/*, .DS_Store, src/.hidden
|
|
// Total files: 14, excluded: 4 (.git/config, .git/objects/pack/data.pack, .DS_Store, src/.hidden)
|
|
require.Equal(t, 10, result.FilesScanned, "Should exclude hidden files and directories")
|
|
}
|
|
|
|
func TestExcludePatterns_DoubleStarGlob(t *testing.T) {
|
|
fs := setupExcludeTestFS(t)
|
|
scanner, repos, cleanup := createTestScanner(t, fs, []string{"**/*.pack"})
|
|
defer cleanup()
|
|
require.NotNil(t, scanner)
|
|
|
|
ctx := context.Background()
|
|
createSnapshotRecord(t, ctx, repos, "test-snapshot")
|
|
|
|
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
|
|
require.NoError(t, err)
|
|
|
|
// Should exclude .git/objects/pack/data.pack
|
|
// Total files: 14, excluded: 1
|
|
require.Equal(t, 13, result.FilesScanned, "Should exclude **/*.pack files")
|
|
}
|
|
|
|
func TestExcludePatterns_ExactFileName(t *testing.T) {
|
|
fs := setupExcludeTestFS(t)
|
|
scanner, repos, cleanup := createTestScanner(t, fs, []string{"thumbs.db", ".DS_Store"})
|
|
defer cleanup()
|
|
require.NotNil(t, scanner)
|
|
|
|
ctx := context.Background()
|
|
createSnapshotRecord(t, ctx, repos, "test-snapshot")
|
|
|
|
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
|
|
require.NoError(t, err)
|
|
|
|
// Should exclude thumbs.db and .DS_Store
|
|
// Total files: 14, excluded: 2
|
|
require.Equal(t, 12, result.FilesScanned, "Should exclude exact file names")
|
|
}
|
|
|
|
func TestExcludePatterns_CaseSensitive(t *testing.T) {
|
|
// Pattern matching should be case-sensitive
|
|
fs := setupExcludeTestFS(t)
|
|
scanner, repos, cleanup := createTestScanner(t, fs, []string{"THUMBS.DB"})
|
|
defer cleanup()
|
|
require.NotNil(t, scanner)
|
|
|
|
ctx := context.Background()
|
|
createSnapshotRecord(t, ctx, repos, "test-snapshot")
|
|
|
|
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
|
|
require.NoError(t, err)
|
|
|
|
// Case-sensitive matching: THUMBS.DB should NOT match thumbs.db
|
|
// All 14 files should be scanned
|
|
require.Equal(t, 14, result.FilesScanned, "Pattern matching should be case-sensitive")
|
|
}
|
|
|
|
func TestExcludePatterns_DirectoryWithTrailingSlash(t *testing.T) {
|
|
fs := setupExcludeTestFS(t)
|
|
// Some users might add trailing slashes to directory patterns
|
|
scanner, repos, cleanup := createTestScanner(t, fs, []string{"cache/", "build/"})
|
|
defer cleanup()
|
|
require.NotNil(t, scanner)
|
|
|
|
ctx := context.Background()
|
|
createSnapshotRecord(t, ctx, repos, "test-snapshot")
|
|
|
|
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
|
|
require.NoError(t, err)
|
|
|
|
// Should exclude cache/temp.dat and build/output.bin
|
|
// Total files: 14, excluded: 2
|
|
require.Equal(t, 12, result.FilesScanned, "Should handle directory patterns with trailing slashes")
|
|
}
|
|
|
|
func TestExcludePatterns_PatternInSubdirectory(t *testing.T) {
|
|
fs := setupExcludeTestFS(t)
|
|
// Exclude .hidden file specifically in src directory
|
|
scanner, repos, cleanup := createTestScanner(t, fs, []string{"src/.hidden"})
|
|
defer cleanup()
|
|
require.NotNil(t, scanner)
|
|
|
|
ctx := context.Background()
|
|
createSnapshotRecord(t, ctx, repos, "test-snapshot")
|
|
|
|
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
|
|
require.NoError(t, err)
|
|
|
|
// Should exclude only src/.hidden
|
|
// Total files: 14, excluded: 1
|
|
require.Equal(t, 13, result.FilesScanned, "Should exclude specific subdirectory files")
|
|
}
|
|
|
|
// setupAnchoredTestFS creates a filesystem for testing anchored patterns
|
|
// Source dir: /backup
|
|
// Structure:
|
|
//
|
|
// /backup/
|
|
// projectname/
|
|
// file.txt (should be excluded with /projectname)
|
|
// otherproject/
|
|
// projectname/
|
|
// file.txt (should NOT be excluded with /projectname, only with projectname)
|
|
// src/
|
|
// file.go
|
|
func setupAnchoredTestFS(t *testing.T) afero.Fs {
|
|
t.Helper()
|
|
|
|
fs := afero.NewMemMapFs()
|
|
|
|
files := map[string]string{
|
|
"/backup/projectname/file.txt": "root project file",
|
|
"/backup/otherproject/projectname/file.txt": "nested project file",
|
|
"/backup/src/file.go": "source file",
|
|
"/backup/file.txt": "root file",
|
|
}
|
|
|
|
testTime := time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC)
|
|
for path, content := range files {
|
|
dir := filepath.Dir(path)
|
|
err := fs.MkdirAll(dir, 0755)
|
|
require.NoError(t, err)
|
|
err = afero.WriteFile(fs, path, []byte(content), 0644)
|
|
require.NoError(t, err)
|
|
err = fs.Chtimes(path, testTime, testTime)
|
|
require.NoError(t, err)
|
|
}
|
|
|
|
return fs
|
|
}
|
|
|
|
func TestExcludePatterns_AnchoredPattern(t *testing.T) {
|
|
// Pattern starting with / should only match from root of source dir
|
|
fs := setupAnchoredTestFS(t)
|
|
scanner, repos, cleanup := createTestScanner(t, fs, []string{"/projectname"})
|
|
defer cleanup()
|
|
require.NotNil(t, scanner)
|
|
|
|
ctx := context.Background()
|
|
createSnapshotRecord(t, ctx, repos, "test-snapshot")
|
|
|
|
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
|
|
require.NoError(t, err)
|
|
|
|
// /projectname should ONLY exclude /backup/projectname/file.txt (1 file)
|
|
// /backup/otherproject/projectname/file.txt should NOT be excluded
|
|
// Total files: 4, excluded: 1
|
|
require.Equal(t, 3, result.FilesScanned, "Anchored pattern /projectname should only match at root of source dir")
|
|
}
|
|
|
|
func TestExcludePatterns_UnanchoredPattern(t *testing.T) {
|
|
// Pattern without leading / should match anywhere in path
|
|
fs := setupAnchoredTestFS(t)
|
|
scanner, repos, cleanup := createTestScanner(t, fs, []string{"projectname"})
|
|
defer cleanup()
|
|
require.NotNil(t, scanner)
|
|
|
|
ctx := context.Background()
|
|
createSnapshotRecord(t, ctx, repos, "test-snapshot")
|
|
|
|
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
|
|
require.NoError(t, err)
|
|
|
|
// projectname (without /) should exclude BOTH:
|
|
// - /backup/projectname/file.txt
|
|
// - /backup/otherproject/projectname/file.txt
|
|
// Total files: 4, excluded: 2
|
|
require.Equal(t, 2, result.FilesScanned, "Unanchored pattern should match anywhere in path")
|
|
}
|
|
|
|
func TestExcludePatterns_AnchoredPatternWithGlob(t *testing.T) {
|
|
// Anchored pattern with glob
|
|
fs := setupAnchoredTestFS(t)
|
|
scanner, repos, cleanup := createTestScanner(t, fs, []string{"/src/*.go"})
|
|
defer cleanup()
|
|
require.NotNil(t, scanner)
|
|
|
|
ctx := context.Background()
|
|
createSnapshotRecord(t, ctx, repos, "test-snapshot")
|
|
|
|
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
|
|
require.NoError(t, err)
|
|
|
|
// /src/*.go should exclude /backup/src/file.go
|
|
// Total files: 4, excluded: 1
|
|
require.Equal(t, 3, result.FilesScanned, "Anchored pattern with glob should work")
|
|
}
|
|
|
|
func TestExcludePatterns_AnchoredPatternFile(t *testing.T) {
|
|
// Anchored pattern for exact file at root
|
|
fs := setupAnchoredTestFS(t)
|
|
scanner, repos, cleanup := createTestScanner(t, fs, []string{"/file.txt"})
|
|
defer cleanup()
|
|
require.NotNil(t, scanner)
|
|
|
|
ctx := context.Background()
|
|
createSnapshotRecord(t, ctx, repos, "test-snapshot")
|
|
|
|
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
|
|
require.NoError(t, err)
|
|
|
|
// /file.txt should ONLY exclude /backup/file.txt
|
|
// NOT /backup/projectname/file.txt or /backup/otherproject/projectname/file.txt
|
|
// Total files: 4, excluded: 1
|
|
require.Equal(t, 3, result.FilesScanned, "Anchored pattern for file should only match at root")
|
|
}
|
|
|
|
func TestExcludePatterns_UnanchoredPatternFile(t *testing.T) {
|
|
// Unanchored pattern for file should match anywhere
|
|
fs := setupAnchoredTestFS(t)
|
|
scanner, repos, cleanup := createTestScanner(t, fs, []string{"file.txt"})
|
|
defer cleanup()
|
|
require.NotNil(t, scanner)
|
|
|
|
ctx := context.Background()
|
|
createSnapshotRecord(t, ctx, repos, "test-snapshot")
|
|
|
|
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
|
|
require.NoError(t, err)
|
|
|
|
// file.txt should exclude ALL file.txt files:
|
|
// - /backup/file.txt
|
|
// - /backup/projectname/file.txt
|
|
// - /backup/otherproject/projectname/file.txt
|
|
// Total files: 4, excluded: 3
|
|
require.Equal(t, 1, result.FilesScanned, "Unanchored pattern for file should match anywhere")
|
|
}
|