Move StartTime initialization to application startup hook
- Remove StartTime initialization from globals.New() - Add setupGlobals function in app.go to set StartTime during fx OnStart - Simplify globals package to be just a key/value store - Remove fx dependencies from globals test
This commit is contained in:
6
internal/backup/module.go
Normal file
6
internal/backup/module.go
Normal file
@@ -0,0 +1,6 @@
|
||||
package backup
|
||||
|
||||
import "go.uber.org/fx"
|
||||
|
||||
// Module exports backup functionality
|
||||
var Module = fx.Module("backup")
|
||||
216
internal/backup/scanner.go
Normal file
216
internal/backup/scanner.go
Normal file
@@ -0,0 +1,216 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/database"
|
||||
"github.com/spf13/afero"
|
||||
)
|
||||
|
||||
// Scanner scans directories and populates the database with file and chunk information
|
||||
type Scanner struct {
|
||||
fs afero.Fs
|
||||
chunkSize int
|
||||
repos *database.Repositories
|
||||
}
|
||||
|
||||
// ScannerConfig contains configuration for the scanner
|
||||
type ScannerConfig struct {
|
||||
FS afero.Fs
|
||||
ChunkSize int
|
||||
Repositories *database.Repositories
|
||||
}
|
||||
|
||||
// ScanResult contains the results of a scan operation
|
||||
type ScanResult struct {
|
||||
FilesScanned int
|
||||
BytesScanned int64
|
||||
StartTime time.Time
|
||||
EndTime time.Time
|
||||
}
|
||||
|
||||
// NewScanner creates a new scanner instance
|
||||
func NewScanner(cfg ScannerConfig) *Scanner {
|
||||
return &Scanner{
|
||||
fs: cfg.FS,
|
||||
chunkSize: cfg.ChunkSize,
|
||||
repos: cfg.Repositories,
|
||||
}
|
||||
}
|
||||
|
||||
// Scan scans a directory and populates the database
|
||||
func (s *Scanner) Scan(ctx context.Context, path string) (*ScanResult, error) {
|
||||
result := &ScanResult{
|
||||
StartTime: time.Now(),
|
||||
}
|
||||
|
||||
// Start a transaction
|
||||
err := s.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
|
||||
return s.scanDirectory(ctx, tx, path, result)
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scan failed: %w", err)
|
||||
}
|
||||
|
||||
result.EndTime = time.Now()
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s *Scanner) scanDirectory(ctx context.Context, tx *sql.Tx, path string, result *ScanResult) error {
|
||||
return afero.Walk(s.fs, path, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Check context cancellation
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
// Skip directories
|
||||
if info.IsDir() {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Process the file
|
||||
if err := s.processFile(ctx, tx, path, info, result); err != nil {
|
||||
return fmt.Errorf("failed to process %s: %w", path, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Scanner) processFile(ctx context.Context, tx *sql.Tx, path string, info os.FileInfo, result *ScanResult) error {
|
||||
// Get file stats
|
||||
stat, ok := info.Sys().(interface {
|
||||
Uid() uint32
|
||||
Gid() uint32
|
||||
})
|
||||
|
||||
var uid, gid uint32
|
||||
if ok {
|
||||
uid = stat.Uid()
|
||||
gid = stat.Gid()
|
||||
}
|
||||
|
||||
// Check if it's a symlink
|
||||
var linkTarget string
|
||||
if info.Mode()&os.ModeSymlink != 0 {
|
||||
// Read the symlink target
|
||||
if linker, ok := s.fs.(afero.LinkReader); ok {
|
||||
linkTarget, _ = linker.ReadlinkIfPossible(path)
|
||||
}
|
||||
}
|
||||
|
||||
// Create file record
|
||||
file := &database.File{
|
||||
Path: path,
|
||||
MTime: info.ModTime(),
|
||||
CTime: info.ModTime(), // afero doesn't provide ctime
|
||||
Size: info.Size(),
|
||||
Mode: uint32(info.Mode()),
|
||||
UID: uid,
|
||||
GID: gid,
|
||||
LinkTarget: linkTarget,
|
||||
}
|
||||
|
||||
// Insert file
|
||||
if err := s.repos.Files.Create(ctx, tx, file); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
result.FilesScanned++
|
||||
result.BytesScanned += info.Size()
|
||||
|
||||
// Process chunks only for regular files
|
||||
if info.Mode().IsRegular() && info.Size() > 0 {
|
||||
if err := s.processFileChunks(ctx, tx, path, result); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Scanner) processFileChunks(ctx context.Context, tx *sql.Tx, path string, result *ScanResult) error {
|
||||
file, err := s.fs.Open(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
if err := file.Close(); err != nil {
|
||||
database.Fatal("failed to close file %s: %v", path, err)
|
||||
}
|
||||
}()
|
||||
|
||||
sequence := 0
|
||||
buffer := make([]byte, s.chunkSize)
|
||||
|
||||
for {
|
||||
n, err := io.ReadFull(file, buffer)
|
||||
if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
|
||||
return err
|
||||
}
|
||||
|
||||
if n == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
// Calculate chunk hash
|
||||
h := sha256.New()
|
||||
h.Write(buffer[:n])
|
||||
hash := hex.EncodeToString(h.Sum(nil))
|
||||
|
||||
// Create chunk if it doesn't exist
|
||||
chunk := &database.Chunk{
|
||||
ChunkHash: hash,
|
||||
SHA256: hash, // Using same hash for now
|
||||
Size: int64(n),
|
||||
}
|
||||
|
||||
// Try to insert chunk (ignore duplicate errors)
|
||||
_ = s.repos.Chunks.Create(ctx, tx, chunk)
|
||||
|
||||
// Create file-chunk mapping
|
||||
fileChunk := &database.FileChunk{
|
||||
Path: path,
|
||||
ChunkHash: hash,
|
||||
Idx: sequence,
|
||||
}
|
||||
|
||||
if err := s.repos.FileChunks.Create(ctx, tx, fileChunk); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Create chunk-file mapping
|
||||
chunkFile := &database.ChunkFile{
|
||||
ChunkHash: hash,
|
||||
FilePath: path,
|
||||
FileOffset: int64(sequence * s.chunkSize),
|
||||
Length: int64(n),
|
||||
}
|
||||
|
||||
if err := s.repos.ChunkFiles.Create(ctx, tx, chunkFile); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
sequence++
|
||||
|
||||
if err == io.EOF || err == io.ErrUnexpectedEOF {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
276
internal/backup/scanner_test.go
Normal file
276
internal/backup/scanner_test.go
Normal file
@@ -0,0 +1,276 @@
|
||||
package backup_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/backup"
|
||||
"git.eeqj.de/sneak/vaultik/internal/database"
|
||||
"github.com/spf13/afero"
|
||||
)
|
||||
|
||||
func TestScannerSimpleDirectory(t *testing.T) {
|
||||
// Create in-memory filesystem
|
||||
fs := afero.NewMemMapFs()
|
||||
|
||||
// Create test directory structure
|
||||
testFiles := map[string]string{
|
||||
"/source/file1.txt": "Hello, world!", // 13 bytes
|
||||
"/source/file2.txt": "This is another file", // 20 bytes
|
||||
"/source/subdir/file3.txt": "File in subdirectory", // 20 bytes
|
||||
"/source/subdir/file4.txt": "Another file in subdirectory", // 28 bytes
|
||||
"/source/empty.txt": "", // 0 bytes
|
||||
"/source/subdir2/file5.txt": "Yet another file", // 16 bytes
|
||||
}
|
||||
|
||||
// Create files with specific times
|
||||
testTime := time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC)
|
||||
for path, content := range testFiles {
|
||||
dir := filepath.Dir(path)
|
||||
if err := fs.MkdirAll(dir, 0755); err != nil {
|
||||
t.Fatalf("failed to create directory %s: %v", dir, err)
|
||||
}
|
||||
if err := afero.WriteFile(fs, path, []byte(content), 0644); err != nil {
|
||||
t.Fatalf("failed to write file %s: %v", path, err)
|
||||
}
|
||||
// Set times
|
||||
if err := fs.Chtimes(path, testTime, testTime); err != nil {
|
||||
t.Fatalf("failed to set times for %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Create test database
|
||||
db, err := database.NewTestDB()
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create test database: %v", err)
|
||||
}
|
||||
defer func() {
|
||||
if err := db.Close(); err != nil {
|
||||
t.Errorf("failed to close database: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
repos := database.NewRepositories(db)
|
||||
|
||||
// Create scanner
|
||||
scanner := backup.NewScanner(backup.ScannerConfig{
|
||||
FS: fs,
|
||||
ChunkSize: 1024 * 16, // 16KB chunks for testing
|
||||
Repositories: repos,
|
||||
})
|
||||
|
||||
// Scan the directory
|
||||
ctx := context.Background()
|
||||
result, err := scanner.Scan(ctx, "/source")
|
||||
if err != nil {
|
||||
t.Fatalf("scan failed: %v", err)
|
||||
}
|
||||
|
||||
// Verify results
|
||||
if result.FilesScanned != 6 {
|
||||
t.Errorf("expected 6 files scanned, got %d", result.FilesScanned)
|
||||
}
|
||||
|
||||
if result.BytesScanned != 97 { // Total size of all test files: 13 + 20 + 20 + 28 + 0 + 16 = 97
|
||||
t.Errorf("expected 97 bytes scanned, got %d", result.BytesScanned)
|
||||
}
|
||||
|
||||
// Verify files in database
|
||||
files, err := repos.Files.ListByPrefix(ctx, "/source")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to list files: %v", err)
|
||||
}
|
||||
|
||||
if len(files) != 6 {
|
||||
t.Errorf("expected 6 files in database, got %d", len(files))
|
||||
}
|
||||
|
||||
// Verify specific file
|
||||
file1, err := repos.Files.GetByPath(ctx, "/source/file1.txt")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get file1.txt: %v", err)
|
||||
}
|
||||
|
||||
if file1.Size != 13 {
|
||||
t.Errorf("expected file1.txt size 13, got %d", file1.Size)
|
||||
}
|
||||
|
||||
if file1.Mode != 0644 {
|
||||
t.Errorf("expected file1.txt mode 0644, got %o", file1.Mode)
|
||||
}
|
||||
|
||||
// Verify chunks were created
|
||||
chunks, err := repos.FileChunks.GetByFile(ctx, "/source/file1.txt")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get chunks for file1.txt: %v", err)
|
||||
}
|
||||
|
||||
if len(chunks) != 1 { // Small file should be one chunk
|
||||
t.Errorf("expected 1 chunk for file1.txt, got %d", len(chunks))
|
||||
}
|
||||
|
||||
// Verify deduplication - file3.txt and file4.txt have different content
|
||||
// but we should still have the correct number of unique chunks
|
||||
allChunks, err := repos.Chunks.List(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to list all chunks: %v", err)
|
||||
}
|
||||
|
||||
// We should have at most 6 chunks (one per unique file content)
|
||||
// Empty file might not create a chunk
|
||||
if len(allChunks) > 6 {
|
||||
t.Errorf("expected at most 6 chunks, got %d", len(allChunks))
|
||||
}
|
||||
}
|
||||
|
||||
func TestScannerWithSymlinks(t *testing.T) {
|
||||
// Create in-memory filesystem
|
||||
fs := afero.NewMemMapFs()
|
||||
|
||||
// Create test files
|
||||
if err := fs.MkdirAll("/source", 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := afero.WriteFile(fs, "/source/target.txt", []byte("target content"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := afero.WriteFile(fs, "/outside/file.txt", []byte("outside content"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Create symlinks (if supported by the filesystem)
|
||||
linker, ok := fs.(afero.Symlinker)
|
||||
if !ok {
|
||||
t.Skip("filesystem does not support symlinks")
|
||||
}
|
||||
|
||||
// Symlink to file in source
|
||||
if err := linker.SymlinkIfPossible("target.txt", "/source/link1.txt"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Symlink to file outside source
|
||||
if err := linker.SymlinkIfPossible("/outside/file.txt", "/source/link2.txt"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Create test database
|
||||
db, err := database.NewTestDB()
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create test database: %v", err)
|
||||
}
|
||||
defer func() {
|
||||
if err := db.Close(); err != nil {
|
||||
t.Errorf("failed to close database: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
repos := database.NewRepositories(db)
|
||||
|
||||
// Create scanner
|
||||
scanner := backup.NewScanner(backup.ScannerConfig{
|
||||
FS: fs,
|
||||
ChunkSize: 1024 * 16,
|
||||
Repositories: repos,
|
||||
})
|
||||
|
||||
// Scan the directory
|
||||
ctx := context.Background()
|
||||
result, err := scanner.Scan(ctx, "/source")
|
||||
if err != nil {
|
||||
t.Fatalf("scan failed: %v", err)
|
||||
}
|
||||
|
||||
// Should have scanned 3 files (target + 2 symlinks)
|
||||
if result.FilesScanned != 3 {
|
||||
t.Errorf("expected 3 files scanned, got %d", result.FilesScanned)
|
||||
}
|
||||
|
||||
// Check symlinks in database
|
||||
link1, err := repos.Files.GetByPath(ctx, "/source/link1.txt")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get link1.txt: %v", err)
|
||||
}
|
||||
|
||||
if link1.LinkTarget != "target.txt" {
|
||||
t.Errorf("expected link1.txt target 'target.txt', got %q", link1.LinkTarget)
|
||||
}
|
||||
|
||||
link2, err := repos.Files.GetByPath(ctx, "/source/link2.txt")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get link2.txt: %v", err)
|
||||
}
|
||||
|
||||
if link2.LinkTarget != "/outside/file.txt" {
|
||||
t.Errorf("expected link2.txt target '/outside/file.txt', got %q", link2.LinkTarget)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScannerLargeFile(t *testing.T) {
|
||||
// Create in-memory filesystem
|
||||
fs := afero.NewMemMapFs()
|
||||
|
||||
// Create a large file that will require multiple chunks
|
||||
largeContent := make([]byte, 1024*1024) // 1MB
|
||||
for i := range largeContent {
|
||||
largeContent[i] = byte(i % 256)
|
||||
}
|
||||
|
||||
if err := fs.MkdirAll("/source", 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := afero.WriteFile(fs, "/source/large.bin", largeContent, 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Create test database
|
||||
db, err := database.NewTestDB()
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create test database: %v", err)
|
||||
}
|
||||
defer func() {
|
||||
if err := db.Close(); err != nil {
|
||||
t.Errorf("failed to close database: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
repos := database.NewRepositories(db)
|
||||
|
||||
// Create scanner with 64KB chunks
|
||||
scanner := backup.NewScanner(backup.ScannerConfig{
|
||||
FS: fs,
|
||||
ChunkSize: 1024 * 64, // 64KB chunks
|
||||
Repositories: repos,
|
||||
})
|
||||
|
||||
// Scan the directory
|
||||
ctx := context.Background()
|
||||
result, err := scanner.Scan(ctx, "/source")
|
||||
if err != nil {
|
||||
t.Fatalf("scan failed: %v", err)
|
||||
}
|
||||
|
||||
if result.BytesScanned != 1024*1024 {
|
||||
t.Errorf("expected %d bytes scanned, got %d", 1024*1024, result.BytesScanned)
|
||||
}
|
||||
|
||||
// Verify chunks
|
||||
chunks, err := repos.FileChunks.GetByFile(ctx, "/source/large.bin")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get chunks: %v", err)
|
||||
}
|
||||
|
||||
expectedChunks := 16 // 1MB / 64KB
|
||||
if len(chunks) != expectedChunks {
|
||||
t.Errorf("expected %d chunks, got %d", expectedChunks, len(chunks))
|
||||
}
|
||||
|
||||
// Verify chunk sequence
|
||||
for i, fc := range chunks {
|
||||
if fc.Idx != i {
|
||||
t.Errorf("chunk %d has incorrect sequence %d", i, fc.Idx)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user