Move StartTime initialization to application startup hook

- Remove StartTime initialization from globals.New()
- Add setupGlobals function in app.go to set StartTime during fx OnStart
- Simplify globals package to be just a key/value store
- Remove fx dependencies from globals test

internal/backup/module.go

@@ -0,0 +1,6 @@
+package backup
+
+import "go.uber.org/fx"
+
+// Module exports backup functionality
+var Module = fx.Module("backup")

internal/backup/scanner.go

@@ -0,0 +1,216 @@
+package backup
+
+import (
+	"context"
+	"crypto/sha256"
+	"database/sql"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"os"
+	"time"
+
+	"git.eeqj.de/sneak/vaultik/internal/database"
+	"github.com/spf13/afero"
+)
+
+// Scanner scans directories and populates the database with file and chunk information
+type Scanner struct {
+	fs        afero.Fs
+	chunkSize int
+	repos     *database.Repositories
+}
+
+// ScannerConfig contains configuration for the scanner
+type ScannerConfig struct {
+	FS           afero.Fs
+	ChunkSize    int
+	Repositories *database.Repositories
+}
+
+// ScanResult contains the results of a scan operation
+type ScanResult struct {
+	FilesScanned int
+	BytesScanned int64
+	StartTime    time.Time
+	EndTime      time.Time
+}
+
+// NewScanner creates a new scanner instance
+func NewScanner(cfg ScannerConfig) *Scanner {
+	return &Scanner{
+		fs:        cfg.FS,
+		chunkSize: cfg.ChunkSize,
+		repos:     cfg.Repositories,
+	}
+}
+
+// Scan scans a directory and populates the database
+func (s *Scanner) Scan(ctx context.Context, path string) (*ScanResult, error) {
+	result := &ScanResult{
+		StartTime: time.Now(),
+	}
+
+	// Start a transaction
+	err := s.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
+		return s.scanDirectory(ctx, tx, path, result)
+	})
+
+	if err != nil {
+		return nil, fmt.Errorf("scan failed: %w", err)
+	}
+
+	result.EndTime = time.Now()
+	return result, nil
+}
+
+func (s *Scanner) scanDirectory(ctx context.Context, tx *sql.Tx, path string, result *ScanResult) error {
+	return afero.Walk(s.fs, path, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		// Check context cancellation
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		default:
+		}
+
+		// Skip directories
+		if info.IsDir() {
+			return nil
+		}
+
+		// Process the file
+		if err := s.processFile(ctx, tx, path, info, result); err != nil {
+			return fmt.Errorf("failed to process %s: %w", path, err)
+		}
+
+		return nil
+	})
+}
+
+func (s *Scanner) processFile(ctx context.Context, tx *sql.Tx, path string, info os.FileInfo, result *ScanResult) error {
+	// Get file stats
+	stat, ok := info.Sys().(interface {
+		Uid() uint32
+		Gid() uint32
+	})
+
+	var uid, gid uint32
+	if ok {
+		uid = stat.Uid()
+		gid = stat.Gid()
+	}
+
+	// Check if it's a symlink
+	var linkTarget string
+	if info.Mode()&os.ModeSymlink != 0 {
+		// Read the symlink target
+		if linker, ok := s.fs.(afero.LinkReader); ok {
+			linkTarget, _ = linker.ReadlinkIfPossible(path)
+		}
+	}
+
+	// Create file record
+	file := &database.File{
+		Path:       path,
+		MTime:      info.ModTime(),
+		CTime:      info.ModTime(), // afero doesn't provide ctime
+		Size:       info.Size(),
+		Mode:       uint32(info.Mode()),
+		UID:        uid,
+		GID:        gid,
+		LinkTarget: linkTarget,
+	}
+
+	// Insert file
+	if err := s.repos.Files.Create(ctx, tx, file); err != nil {
+		return err
+	}
+
+	result.FilesScanned++
+	result.BytesScanned += info.Size()
+
+	// Process chunks only for regular files
+	if info.Mode().IsRegular() && info.Size() > 0 {
+		if err := s.processFileChunks(ctx, tx, path, result); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (s *Scanner) processFileChunks(ctx context.Context, tx *sql.Tx, path string, result *ScanResult) error {
+	file, err := s.fs.Open(path)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if err := file.Close(); err != nil {
+			database.Fatal("failed to close file %s: %v", path, err)
+		}
+	}()
+
+	sequence := 0
+	buffer := make([]byte, s.chunkSize)
+
+	for {
+		n, err := io.ReadFull(file, buffer)
+		if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
+			return err
+		}
+
+		if n == 0 {
+			break
+		}
+
+		// Calculate chunk hash
+		h := sha256.New()
+		h.Write(buffer[:n])
+		hash := hex.EncodeToString(h.Sum(nil))
+
+		// Create chunk if it doesn't exist
+		chunk := &database.Chunk{
+			ChunkHash: hash,
+			SHA256:    hash, // Using same hash for now
+			Size:      int64(n),
+		}
+
+		// Try to insert chunk (ignore duplicate errors)
+		_ = s.repos.Chunks.Create(ctx, tx, chunk)
+
+		// Create file-chunk mapping
+		fileChunk := &database.FileChunk{
+			Path:      path,
+			ChunkHash: hash,
+			Idx:       sequence,
+		}
+
+		if err := s.repos.FileChunks.Create(ctx, tx, fileChunk); err != nil {
+			return err
+		}
+
+		// Create chunk-file mapping
+		chunkFile := &database.ChunkFile{
+			ChunkHash:  hash,
+			FilePath:   path,
+			FileOffset: int64(sequence * s.chunkSize),
+			Length:     int64(n),
+		}
+
+		if err := s.repos.ChunkFiles.Create(ctx, tx, chunkFile); err != nil {
+			return err
+		}
+
+		sequence++
+
+		if err == io.EOF || err == io.ErrUnexpectedEOF {
+			break
+		}
+	}
+
+	return nil
+}

internal/backup/scanner_test.go

@@ -0,0 +1,276 @@
+package backup_test
+
+import (
+	"context"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"git.eeqj.de/sneak/vaultik/internal/backup"
+	"git.eeqj.de/sneak/vaultik/internal/database"
+	"github.com/spf13/afero"
+)
+
+func TestScannerSimpleDirectory(t *testing.T) {
+	// Create in-memory filesystem
+	fs := afero.NewMemMapFs()
+
+	// Create test directory structure
+	testFiles := map[string]string{
+		"/source/file1.txt":         "Hello, world!",                // 13 bytes
+		"/source/file2.txt":         "This is another file",         // 20 bytes
+		"/source/subdir/file3.txt":  "File in subdirectory",         // 20 bytes
+		"/source/subdir/file4.txt":  "Another file in subdirectory", // 28 bytes
+		"/source/empty.txt":         "",                             // 0 bytes
+		"/source/subdir2/file5.txt": "Yet another file",             // 16 bytes
+	}
+
+	// Create files with specific times
+	testTime := time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC)
+	for path, content := range testFiles {
+		dir := filepath.Dir(path)
+		if err := fs.MkdirAll(dir, 0755); err != nil {
+			t.Fatalf("failed to create directory %s: %v", dir, err)
+		}
+		if err := afero.WriteFile(fs, path, []byte(content), 0644); err != nil {
+			t.Fatalf("failed to write file %s: %v", path, err)
+		}
+		// Set times
+		if err := fs.Chtimes(path, testTime, testTime); err != nil {
+			t.Fatalf("failed to set times for %s: %v", path, err)
+		}
+	}
+
+	// Create test database
+	db, err := database.NewTestDB()
+	if err != nil {
+		t.Fatalf("failed to create test database: %v", err)
+	}
+	defer func() {
+		if err := db.Close(); err != nil {
+			t.Errorf("failed to close database: %v", err)
+		}
+	}()
+
+	repos := database.NewRepositories(db)
+
+	// Create scanner
+	scanner := backup.NewScanner(backup.ScannerConfig{
+		FS:           fs,
+		ChunkSize:    1024 * 16, // 16KB chunks for testing
+		Repositories: repos,
+	})
+
+	// Scan the directory
+	ctx := context.Background()
+	result, err := scanner.Scan(ctx, "/source")
+	if err != nil {
+		t.Fatalf("scan failed: %v", err)
+	}
+
+	// Verify results
+	if result.FilesScanned != 6 {
+		t.Errorf("expected 6 files scanned, got %d", result.FilesScanned)
+	}
+
+	if result.BytesScanned != 97 { // Total size of all test files: 13 + 20 + 20 + 28 + 0 + 16 = 97
+		t.Errorf("expected 97 bytes scanned, got %d", result.BytesScanned)
+	}
+
+	// Verify files in database
+	files, err := repos.Files.ListByPrefix(ctx, "/source")
+	if err != nil {
+		t.Fatalf("failed to list files: %v", err)
+	}
+
+	if len(files) != 6 {
+		t.Errorf("expected 6 files in database, got %d", len(files))
+	}
+
+	// Verify specific file
+	file1, err := repos.Files.GetByPath(ctx, "/source/file1.txt")
+	if err != nil {
+		t.Fatalf("failed to get file1.txt: %v", err)
+	}
+
+	if file1.Size != 13 {
+		t.Errorf("expected file1.txt size 13, got %d", file1.Size)
+	}
+
+	if file1.Mode != 0644 {
+		t.Errorf("expected file1.txt mode 0644, got %o", file1.Mode)
+	}
+
+	// Verify chunks were created
+	chunks, err := repos.FileChunks.GetByFile(ctx, "/source/file1.txt")
+	if err != nil {
+		t.Fatalf("failed to get chunks for file1.txt: %v", err)
+	}
+
+	if len(chunks) != 1 { // Small file should be one chunk
+		t.Errorf("expected 1 chunk for file1.txt, got %d", len(chunks))
+	}
+
+	// Verify deduplication - file3.txt and file4.txt have different content
+	// but we should still have the correct number of unique chunks
+	allChunks, err := repos.Chunks.List(ctx)
+	if err != nil {
+		t.Fatalf("failed to list all chunks: %v", err)
+	}
+
+	// We should have at most 6 chunks (one per unique file content)
+	// Empty file might not create a chunk
+	if len(allChunks) > 6 {
+		t.Errorf("expected at most 6 chunks, got %d", len(allChunks))
+	}
+}
+
+func TestScannerWithSymlinks(t *testing.T) {
+	// Create in-memory filesystem
+	fs := afero.NewMemMapFs()
+
+	// Create test files
+	if err := fs.MkdirAll("/source", 0755); err != nil {
+		t.Fatal(err)
+	}
+	if err := afero.WriteFile(fs, "/source/target.txt", []byte("target content"), 0644); err != nil {
+		t.Fatal(err)
+	}
+	if err := afero.WriteFile(fs, "/outside/file.txt", []byte("outside content"), 0644); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create symlinks (if supported by the filesystem)
+	linker, ok := fs.(afero.Symlinker)
+	if !ok {
+		t.Skip("filesystem does not support symlinks")
+	}
+
+	// Symlink to file in source
+	if err := linker.SymlinkIfPossible("target.txt", "/source/link1.txt"); err != nil {
+		t.Fatal(err)
+	}
+
+	// Symlink to file outside source
+	if err := linker.SymlinkIfPossible("/outside/file.txt", "/source/link2.txt"); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create test database
+	db, err := database.NewTestDB()
+	if err != nil {
+		t.Fatalf("failed to create test database: %v", err)
+	}
+	defer func() {
+		if err := db.Close(); err != nil {
+			t.Errorf("failed to close database: %v", err)
+		}
+	}()
+
+	repos := database.NewRepositories(db)
+
+	// Create scanner
+	scanner := backup.NewScanner(backup.ScannerConfig{
+		FS:           fs,
+		ChunkSize:    1024 * 16,
+		Repositories: repos,
+	})
+
+	// Scan the directory
+	ctx := context.Background()
+	result, err := scanner.Scan(ctx, "/source")
+	if err != nil {
+		t.Fatalf("scan failed: %v", err)
+	}
+
+	// Should have scanned 3 files (target + 2 symlinks)
+	if result.FilesScanned != 3 {
+		t.Errorf("expected 3 files scanned, got %d", result.FilesScanned)
+	}
+
+	// Check symlinks in database
+	link1, err := repos.Files.GetByPath(ctx, "/source/link1.txt")
+	if err != nil {
+		t.Fatalf("failed to get link1.txt: %v", err)
+	}
+
+	if link1.LinkTarget != "target.txt" {
+		t.Errorf("expected link1.txt target 'target.txt', got %q", link1.LinkTarget)
+	}
+
+	link2, err := repos.Files.GetByPath(ctx, "/source/link2.txt")
+	if err != nil {
+		t.Fatalf("failed to get link2.txt: %v", err)
+	}
+
+	if link2.LinkTarget != "/outside/file.txt" {
+		t.Errorf("expected link2.txt target '/outside/file.txt', got %q", link2.LinkTarget)
+	}
+}
+
+func TestScannerLargeFile(t *testing.T) {
+	// Create in-memory filesystem
+	fs := afero.NewMemMapFs()
+
+	// Create a large file that will require multiple chunks
+	largeContent := make([]byte, 1024*1024) // 1MB
+	for i := range largeContent {
+		largeContent[i] = byte(i % 256)
+	}
+
+	if err := fs.MkdirAll("/source", 0755); err != nil {
+		t.Fatal(err)
+	}
+	if err := afero.WriteFile(fs, "/source/large.bin", largeContent, 0644); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create test database
+	db, err := database.NewTestDB()
+	if err != nil {
+		t.Fatalf("failed to create test database: %v", err)
+	}
+	defer func() {
+		if err := db.Close(); err != nil {
+			t.Errorf("failed to close database: %v", err)
+		}
+	}()
+
+	repos := database.NewRepositories(db)
+
+	// Create scanner with 64KB chunks
+	scanner := backup.NewScanner(backup.ScannerConfig{
+		FS:           fs,
+		ChunkSize:    1024 * 64, // 64KB chunks
+		Repositories: repos,
+	})
+
+	// Scan the directory
+	ctx := context.Background()
+	result, err := scanner.Scan(ctx, "/source")
+	if err != nil {
+		t.Fatalf("scan failed: %v", err)
+	}
+
+	if result.BytesScanned != 1024*1024 {
+		t.Errorf("expected %d bytes scanned, got %d", 1024*1024, result.BytesScanned)
+	}
+
+	// Verify chunks
+	chunks, err := repos.FileChunks.GetByFile(ctx, "/source/large.bin")
+	if err != nil {
+		t.Fatalf("failed to get chunks: %v", err)
+	}
+
+	expectedChunks := 16 // 1MB / 64KB
+	if len(chunks) != expectedChunks {
+		t.Errorf("expected %d chunks, got %d", expectedChunks, len(chunks))
+	}
+
+	// Verify chunk sequence
+	for i, fc := range chunks {
+		if fc.Idx != i {
+			t.Errorf("chunk %d has incorrect sequence %d", i, fc.Idx)
+		}
+	}
+}

internal/cli/app.go

@@ -3,6 +3,7 @@ package cli
 import (
 	"context"
 	"fmt"
+	"time"
 
 	"git.eeqj.de/sneak/vaultik/internal/config"
 	"git.eeqj.de/sneak/vaultik/internal/database"
@@ -17,6 +18,16 @@ type AppOptions struct {
 	Invokes    []fx.Option
 }
 
+// setupGlobals sets up the globals with application startup time
+func setupGlobals(lc fx.Lifecycle, g *globals.Globals) {
+	lc.Append(fx.Hook{
+		OnStart: func(ctx context.Context) error {
+			g.StartTime = time.Now()
+			return nil
+		},
+	})
+}
+
 // NewApp creates a new fx application with common modules
 func NewApp(opts AppOptions) *fx.App {
 	baseModules := []fx.Option{
@@ -24,6 +35,7 @@ func NewApp(opts AppOptions) *fx.App {
 		fx.Provide(globals.New),
 		config.Module,
 		database.Module,
+		fx.Invoke(setupGlobals),
 		fx.NopLogger,
 	}
 

internal/database/chunks_ext.go

@@ -0,0 +1,38 @@
+package database
+
+import (
+	"context"
+	"fmt"
+)
+
+func (r *ChunkRepository) List(ctx context.Context) ([]*Chunk, error) {
+	query := `
+		SELECT chunk_hash, sha256, size
+		FROM chunks
+		ORDER BY chunk_hash
+	`
+
+	rows, err := r.db.conn.QueryContext(ctx, query)
+	if err != nil {
+		return nil, fmt.Errorf("querying chunks: %w", err)
+	}
+	defer CloseRows(rows)
+
+	var chunks []*Chunk
+	for rows.Next() {
+		var chunk Chunk
+
+		err := rows.Scan(
+			&chunk.ChunkHash,
+			&chunk.SHA256,
+			&chunk.Size,
+		)
+		if err != nil {
+			return nil, fmt.Errorf("scanning chunk: %w", err)
+		}
+
+		chunks = append(chunks, &chunk)
+	}
+
+	return chunks, rows.Err()
+}

internal/database/database.go

@@ -141,3 +141,8 @@ func (db *DB) createSchema(ctx context.Context) error {
 	_, err := db.conn.ExecContext(ctx, schema)
 	return err
 }
+
+// NewTestDB creates an in-memory SQLite database for testing
+func NewTestDB() (*DB, error) {
+	return New(context.Background(), ":memory:")
+}

internal/database/file_chunks.go

@@ -78,3 +78,8 @@ func (r *FileChunkRepository) DeleteByPath(ctx context.Context, tx *sql.Tx, path
 
 	return nil
 }
+
+// GetByFile is an alias for GetByPath for compatibility
+func (r *FileChunkRepository) GetByFile(ctx context.Context, path string) ([]*FileChunk, error) {
+	return r.GetByPath(ctx, path)
+}

internal/database/files.go

@@ -143,3 +143,49 @@ func (r *FileRepository) Delete(ctx context.Context, tx *sql.Tx, path string) er
 
 	return nil
 }
+
+func (r *FileRepository) ListByPrefix(ctx context.Context, prefix string) ([]*File, error) {
+	query := `
+		SELECT path, mtime, ctime, size, mode, uid, gid, link_target
+		FROM files
+		WHERE path LIKE ? || '%'
+		ORDER BY path
+	`
+
+	rows, err := r.db.conn.QueryContext(ctx, query, prefix)
+	if err != nil {
+		return nil, fmt.Errorf("querying files: %w", err)
+	}
+	defer CloseRows(rows)
+
+	var files []*File
+	for rows.Next() {
+		var file File
+		var mtimeUnix, ctimeUnix int64
+		var linkTarget sql.NullString
+
+		err := rows.Scan(
+			&file.Path,
+			&mtimeUnix,
+			&ctimeUnix,
+			&file.Size,
+			&file.Mode,
+			&file.UID,
+			&file.GID,
+			&linkTarget,
+		)
+		if err != nil {
+			return nil, fmt.Errorf("scanning file: %w", err)
+		}
+
+		file.MTime = time.Unix(mtimeUnix, 0)
+		file.CTime = time.Unix(ctimeUnix, 0)
+		if linkTarget.Valid {
+			file.LinkTarget = linkTarget.String
+		}
+
+		files = append(files, &file)
+	}
+
+	return files, rows.Err()
+}

internal/globals/globals.go

@@ -19,12 +19,9 @@ type Globals struct {
 }
 
 func New() (*Globals, error) {
-	n := &Globals{
-		Appname:   Appname,
-		Version:   Version,
-		Commit:    Commit,
-		StartTime: time.Now(),
-	}
-
-	return n, nil
+	return &Globals{
+		Appname: Appname,
+		Version: Version,
+		Commit:  Commit,
+	}, nil
 }

internal/globals/globals_test.go

@@ -2,35 +2,29 @@ package globals
 
 import (
 	"testing"
-
-	"go.uber.org/fx"
-	"go.uber.org/fx/fxtest"
 )
 
 // TestGlobalsNew ensures the globals package initializes correctly
 func TestGlobalsNew(t *testing.T) {
-	app := fxtest.New(t,
-		fx.Provide(New),
-		fx.Invoke(func(g *Globals) {
-			if g == nil {
-				t.Fatal("Globals instance is nil")
-			}
+	g, err := New()
+	if err != nil {
+		t.Fatalf("Failed to create Globals: %v", err)
+	}
 
-			if g.Appname != "vaultik" {
-				t.Errorf("Expected Appname to be 'vaultik', got '%s'", g.Appname)
-			}
+	if g == nil {
+		t.Fatal("Globals instance is nil")
+	}
 
-			// Version and Commit will be "dev" and "unknown" by default
-			if g.Version == "" {
-				t.Error("Version should not be empty")
-			}
+	if g.Appname != "vaultik" {
+		t.Errorf("Expected Appname to be 'vaultik', got '%s'", g.Appname)
+	}
 
-			if g.Commit == "" {
-				t.Error("Commit should not be empty")
-			}
-		}),
-	)
+	// Version and Commit will be "dev" and "unknown" by default
+	if g.Version == "" {
+		t.Error("Version should not be empty")
+	}
 
-	app.RequireStart()
-	app.RequireStop()
+	if g.Commit == "" {
+		t.Error("Commit should not be empty")
+	}
 }

internal/s3/s3_test.go

@@ -3,6 +3,7 @@ package s3_test
 import (
 	"bytes"
 	"context"
+	"fmt"
 	"io"
 	"net/http"
 	"os"
@@ -14,6 +15,7 @@ import (
 	"github.com/aws/aws-sdk-go-v2/config"
 	"github.com/aws/aws-sdk-go-v2/credentials"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/smithy-go/logging"
 	"github.com/johannesboyne/gofakes3"
 	"github.com/johannesboyne/gofakes3/backend/s3mem"
 )
@@ -32,6 +34,7 @@ type TestServer struct {
 	backend  gofakes3.Backend
 	s3Client *s3.Client
 	tempDir  string
+	logBuf   *bytes.Buffer
 }
 
 // NewTestServer creates and starts a new test server
@@ -62,7 +65,10 @@ func NewTestServer(t *testing.T) *TestServer {
 	// Wait for server to be ready
 	time.Sleep(100 * time.Millisecond)
 
-	// Create S3 client
+	// Create a buffer to capture logs
+	logBuf := &bytes.Buffer{}
+
+	// Create S3 client with custom logger
 	cfg, err := config.LoadDefaultConfig(context.Background(),
 		config.WithRegion(testRegion),
 		config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(
@@ -71,6 +77,13 @@ func NewTestServer(t *testing.T) *TestServer {
 			"",
 		)),
 		config.WithClientLogMode(aws.LogRetries|aws.LogRequestWithBody|aws.LogResponseWithBody),
+		config.WithLogger(logging.LoggerFunc(func(classification logging.Classification, format string, v ...interface{}) {
+			// Capture logs to buffer instead of stdout
+			fmt.Fprintf(logBuf, "SDK %s %s %s\n",
+				time.Now().Format("2006/01/02 15:04:05"),
+				string(classification),
+				fmt.Sprintf(format, v...))
+		})),
 	)
 	if err != nil {
 		t.Fatalf("failed to create AWS config: %v", err)
@@ -86,8 +99,16 @@ func NewTestServer(t *testing.T) *TestServer {
 		backend:  backend,
 		s3Client: s3Client,
 		tempDir:  tempDir,
+		logBuf:   logBuf,
 	}
 
+	// Register cleanup to show logs on test failure
+	t.Cleanup(func() {
+		if t.Failed() && logBuf.Len() > 0 {
+			t.Logf("S3 SDK Debug Output:\n%s", logBuf.String())
+		}
+	})
+
 	// Create test bucket
 	_, err = s3Client.CreateBucket(context.Background(), &s3.CreateBucketInput{
 		Bucket: aws.String(testBucket),