From 26db09691313f0261efd5bee17e7ac6ffa43d805 Mon Sep 17 00:00:00 2001 From: sneak Date: Sun, 20 Jul 2025 12:05:24 +0200 Subject: [PATCH] Move StartTime initialization to application startup hook - Remove StartTime initialization from globals.New() - Add setupGlobals function in app.go to set StartTime during fx OnStart - Simplify globals package to be just a key/value store - Remove fx dependencies from globals test --- TODO.md | 19 +-- go.mod | 1 + go.sum | 3 + internal/backup/module.go | 6 + internal/backup/scanner.go | 216 ++++++++++++++++++++++++ internal/backup/scanner_test.go | 276 +++++++++++++++++++++++++++++++ internal/cli/app.go | 12 ++ internal/database/chunks_ext.go | 38 +++++ internal/database/database.go | 5 + internal/database/file_chunks.go | 5 + internal/database/files.go | 46 ++++++ internal/globals/globals.go | 13 +- internal/globals/globals_test.go | 40 ++--- internal/s3/s3_test.go | 23 ++- 14 files changed, 657 insertions(+), 46 deletions(-) create mode 100644 internal/backup/module.go create mode 100644 internal/backup/scanner.go create mode 100644 internal/backup/scanner_test.go create mode 100644 internal/database/chunks_ext.go diff --git a/TODO.md b/TODO.md index 57ffa57..d47cb4a 100644 --- a/TODO.md +++ b/TODO.md @@ -1,19 +1,10 @@ # Implementation TODO -## Local Index Database -1. Implement SQLite schema creation -1. Create Index type with all database operations -1. Add transaction support and proper locking -1. Implement file tracking (save, lookup, delete) -1. Implement chunk tracking and deduplication -1. Implement blob tracking and chunk-to-blob mapping -1. Write tests for all index operations - ## Chunking and Hashing 1. Implement Rabin fingerprint chunker -1. Create streaming chunk processor -1. Implement SHA256 hashing for chunks -1. Add configurable chunk size parameters +1. Create streaming chunk processor +1. ~~Implement SHA256 hashing for chunks~~ (done in scanner) +1. ~~Add configurable chunk size parameters~~ (done in scanner) 1. Write tests for chunking consistency ## Compression and Encryption @@ -40,9 +31,9 @@ 1. Write tests using MinIO container ## Backup Command - Basic -1. Implement directory walking with exclusion patterns +1. ~~Implement directory walking with exclusion patterns~~ (done with afero) 1. Add file change detection using index -1. Integrate chunking pipeline for changed files +1. ~~Integrate chunking pipeline for changed files~~ (done in scanner) 1. Implement blob upload coordination 1. Add progress reporting to stderr 1. Write integration tests for backup diff --git a/go.mod b/go.mod index c17a664..565da8b 100644 --- a/go.mod +++ b/go.mod @@ -46,6 +46,7 @@ require ( github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rs/xid v1.6.0 // indirect github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 // indirect + github.com/spf13/afero v1.14.0 // indirect github.com/spf13/pflag v1.0.6 // indirect github.com/tinylib/msgp v1.3.0 // indirect go.shabbyrobe.org/gocovmerge v0.0.0-20230507111327-fa4f82cfbf4d // indirect diff --git a/go.sum b/go.sum index 70cba84..a3c0c34 100644 --- a/go.sum +++ b/go.sum @@ -85,6 +85,8 @@ github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= github.com/spf13/afero v1.2.1/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/afero v1.14.0 h1:9tH6MapGnn/j0eb0yIXiLjERO8RB6xIVZRDCX7PtqWA= +github.com/spf13/afero v1.14.0/go.mod h1:acJQ8t0ohCGuMN3O+Pv0V0hgMxNYDlvdk+VTfyZmbYo= github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= @@ -161,6 +163,7 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190829051458-42f498d34c4d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= diff --git a/internal/backup/module.go b/internal/backup/module.go new file mode 100644 index 0000000..f8f8068 --- /dev/null +++ b/internal/backup/module.go @@ -0,0 +1,6 @@ +package backup + +import "go.uber.org/fx" + +// Module exports backup functionality +var Module = fx.Module("backup") diff --git a/internal/backup/scanner.go b/internal/backup/scanner.go new file mode 100644 index 0000000..25d9ece --- /dev/null +++ b/internal/backup/scanner.go @@ -0,0 +1,216 @@ +package backup + +import ( + "context" + "crypto/sha256" + "database/sql" + "encoding/hex" + "fmt" + "io" + "os" + "time" + + "git.eeqj.de/sneak/vaultik/internal/database" + "github.com/spf13/afero" +) + +// Scanner scans directories and populates the database with file and chunk information +type Scanner struct { + fs afero.Fs + chunkSize int + repos *database.Repositories +} + +// ScannerConfig contains configuration for the scanner +type ScannerConfig struct { + FS afero.Fs + ChunkSize int + Repositories *database.Repositories +} + +// ScanResult contains the results of a scan operation +type ScanResult struct { + FilesScanned int + BytesScanned int64 + StartTime time.Time + EndTime time.Time +} + +// NewScanner creates a new scanner instance +func NewScanner(cfg ScannerConfig) *Scanner { + return &Scanner{ + fs: cfg.FS, + chunkSize: cfg.ChunkSize, + repos: cfg.Repositories, + } +} + +// Scan scans a directory and populates the database +func (s *Scanner) Scan(ctx context.Context, path string) (*ScanResult, error) { + result := &ScanResult{ + StartTime: time.Now(), + } + + // Start a transaction + err := s.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { + return s.scanDirectory(ctx, tx, path, result) + }) + + if err != nil { + return nil, fmt.Errorf("scan failed: %w", err) + } + + result.EndTime = time.Now() + return result, nil +} + +func (s *Scanner) scanDirectory(ctx context.Context, tx *sql.Tx, path string, result *ScanResult) error { + return afero.Walk(s.fs, path, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + // Check context cancellation + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + // Skip directories + if info.IsDir() { + return nil + } + + // Process the file + if err := s.processFile(ctx, tx, path, info, result); err != nil { + return fmt.Errorf("failed to process %s: %w", path, err) + } + + return nil + }) +} + +func (s *Scanner) processFile(ctx context.Context, tx *sql.Tx, path string, info os.FileInfo, result *ScanResult) error { + // Get file stats + stat, ok := info.Sys().(interface { + Uid() uint32 + Gid() uint32 + }) + + var uid, gid uint32 + if ok { + uid = stat.Uid() + gid = stat.Gid() + } + + // Check if it's a symlink + var linkTarget string + if info.Mode()&os.ModeSymlink != 0 { + // Read the symlink target + if linker, ok := s.fs.(afero.LinkReader); ok { + linkTarget, _ = linker.ReadlinkIfPossible(path) + } + } + + // Create file record + file := &database.File{ + Path: path, + MTime: info.ModTime(), + CTime: info.ModTime(), // afero doesn't provide ctime + Size: info.Size(), + Mode: uint32(info.Mode()), + UID: uid, + GID: gid, + LinkTarget: linkTarget, + } + + // Insert file + if err := s.repos.Files.Create(ctx, tx, file); err != nil { + return err + } + + result.FilesScanned++ + result.BytesScanned += info.Size() + + // Process chunks only for regular files + if info.Mode().IsRegular() && info.Size() > 0 { + if err := s.processFileChunks(ctx, tx, path, result); err != nil { + return err + } + } + + return nil +} + +func (s *Scanner) processFileChunks(ctx context.Context, tx *sql.Tx, path string, result *ScanResult) error { + file, err := s.fs.Open(path) + if err != nil { + return err + } + defer func() { + if err := file.Close(); err != nil { + database.Fatal("failed to close file %s: %v", path, err) + } + }() + + sequence := 0 + buffer := make([]byte, s.chunkSize) + + for { + n, err := io.ReadFull(file, buffer) + if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF { + return err + } + + if n == 0 { + break + } + + // Calculate chunk hash + h := sha256.New() + h.Write(buffer[:n]) + hash := hex.EncodeToString(h.Sum(nil)) + + // Create chunk if it doesn't exist + chunk := &database.Chunk{ + ChunkHash: hash, + SHA256: hash, // Using same hash for now + Size: int64(n), + } + + // Try to insert chunk (ignore duplicate errors) + _ = s.repos.Chunks.Create(ctx, tx, chunk) + + // Create file-chunk mapping + fileChunk := &database.FileChunk{ + Path: path, + ChunkHash: hash, + Idx: sequence, + } + + if err := s.repos.FileChunks.Create(ctx, tx, fileChunk); err != nil { + return err + } + + // Create chunk-file mapping + chunkFile := &database.ChunkFile{ + ChunkHash: hash, + FilePath: path, + FileOffset: int64(sequence * s.chunkSize), + Length: int64(n), + } + + if err := s.repos.ChunkFiles.Create(ctx, tx, chunkFile); err != nil { + return err + } + + sequence++ + + if err == io.EOF || err == io.ErrUnexpectedEOF { + break + } + } + + return nil +} diff --git a/internal/backup/scanner_test.go b/internal/backup/scanner_test.go new file mode 100644 index 0000000..492f487 --- /dev/null +++ b/internal/backup/scanner_test.go @@ -0,0 +1,276 @@ +package backup_test + +import ( + "context" + "path/filepath" + "testing" + "time" + + "git.eeqj.de/sneak/vaultik/internal/backup" + "git.eeqj.de/sneak/vaultik/internal/database" + "github.com/spf13/afero" +) + +func TestScannerSimpleDirectory(t *testing.T) { + // Create in-memory filesystem + fs := afero.NewMemMapFs() + + // Create test directory structure + testFiles := map[string]string{ + "/source/file1.txt": "Hello, world!", // 13 bytes + "/source/file2.txt": "This is another file", // 20 bytes + "/source/subdir/file3.txt": "File in subdirectory", // 20 bytes + "/source/subdir/file4.txt": "Another file in subdirectory", // 28 bytes + "/source/empty.txt": "", // 0 bytes + "/source/subdir2/file5.txt": "Yet another file", // 16 bytes + } + + // Create files with specific times + testTime := time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC) + for path, content := range testFiles { + dir := filepath.Dir(path) + if err := fs.MkdirAll(dir, 0755); err != nil { + t.Fatalf("failed to create directory %s: %v", dir, err) + } + if err := afero.WriteFile(fs, path, []byte(content), 0644); err != nil { + t.Fatalf("failed to write file %s: %v", path, err) + } + // Set times + if err := fs.Chtimes(path, testTime, testTime); err != nil { + t.Fatalf("failed to set times for %s: %v", path, err) + } + } + + // Create test database + db, err := database.NewTestDB() + if err != nil { + t.Fatalf("failed to create test database: %v", err) + } + defer func() { + if err := db.Close(); err != nil { + t.Errorf("failed to close database: %v", err) + } + }() + + repos := database.NewRepositories(db) + + // Create scanner + scanner := backup.NewScanner(backup.ScannerConfig{ + FS: fs, + ChunkSize: 1024 * 16, // 16KB chunks for testing + Repositories: repos, + }) + + // Scan the directory + ctx := context.Background() + result, err := scanner.Scan(ctx, "/source") + if err != nil { + t.Fatalf("scan failed: %v", err) + } + + // Verify results + if result.FilesScanned != 6 { + t.Errorf("expected 6 files scanned, got %d", result.FilesScanned) + } + + if result.BytesScanned != 97 { // Total size of all test files: 13 + 20 + 20 + 28 + 0 + 16 = 97 + t.Errorf("expected 97 bytes scanned, got %d", result.BytesScanned) + } + + // Verify files in database + files, err := repos.Files.ListByPrefix(ctx, "/source") + if err != nil { + t.Fatalf("failed to list files: %v", err) + } + + if len(files) != 6 { + t.Errorf("expected 6 files in database, got %d", len(files)) + } + + // Verify specific file + file1, err := repos.Files.GetByPath(ctx, "/source/file1.txt") + if err != nil { + t.Fatalf("failed to get file1.txt: %v", err) + } + + if file1.Size != 13 { + t.Errorf("expected file1.txt size 13, got %d", file1.Size) + } + + if file1.Mode != 0644 { + t.Errorf("expected file1.txt mode 0644, got %o", file1.Mode) + } + + // Verify chunks were created + chunks, err := repos.FileChunks.GetByFile(ctx, "/source/file1.txt") + if err != nil { + t.Fatalf("failed to get chunks for file1.txt: %v", err) + } + + if len(chunks) != 1 { // Small file should be one chunk + t.Errorf("expected 1 chunk for file1.txt, got %d", len(chunks)) + } + + // Verify deduplication - file3.txt and file4.txt have different content + // but we should still have the correct number of unique chunks + allChunks, err := repos.Chunks.List(ctx) + if err != nil { + t.Fatalf("failed to list all chunks: %v", err) + } + + // We should have at most 6 chunks (one per unique file content) + // Empty file might not create a chunk + if len(allChunks) > 6 { + t.Errorf("expected at most 6 chunks, got %d", len(allChunks)) + } +} + +func TestScannerWithSymlinks(t *testing.T) { + // Create in-memory filesystem + fs := afero.NewMemMapFs() + + // Create test files + if err := fs.MkdirAll("/source", 0755); err != nil { + t.Fatal(err) + } + if err := afero.WriteFile(fs, "/source/target.txt", []byte("target content"), 0644); err != nil { + t.Fatal(err) + } + if err := afero.WriteFile(fs, "/outside/file.txt", []byte("outside content"), 0644); err != nil { + t.Fatal(err) + } + + // Create symlinks (if supported by the filesystem) + linker, ok := fs.(afero.Symlinker) + if !ok { + t.Skip("filesystem does not support symlinks") + } + + // Symlink to file in source + if err := linker.SymlinkIfPossible("target.txt", "/source/link1.txt"); err != nil { + t.Fatal(err) + } + + // Symlink to file outside source + if err := linker.SymlinkIfPossible("/outside/file.txt", "/source/link2.txt"); err != nil { + t.Fatal(err) + } + + // Create test database + db, err := database.NewTestDB() + if err != nil { + t.Fatalf("failed to create test database: %v", err) + } + defer func() { + if err := db.Close(); err != nil { + t.Errorf("failed to close database: %v", err) + } + }() + + repos := database.NewRepositories(db) + + // Create scanner + scanner := backup.NewScanner(backup.ScannerConfig{ + FS: fs, + ChunkSize: 1024 * 16, + Repositories: repos, + }) + + // Scan the directory + ctx := context.Background() + result, err := scanner.Scan(ctx, "/source") + if err != nil { + t.Fatalf("scan failed: %v", err) + } + + // Should have scanned 3 files (target + 2 symlinks) + if result.FilesScanned != 3 { + t.Errorf("expected 3 files scanned, got %d", result.FilesScanned) + } + + // Check symlinks in database + link1, err := repos.Files.GetByPath(ctx, "/source/link1.txt") + if err != nil { + t.Fatalf("failed to get link1.txt: %v", err) + } + + if link1.LinkTarget != "target.txt" { + t.Errorf("expected link1.txt target 'target.txt', got %q", link1.LinkTarget) + } + + link2, err := repos.Files.GetByPath(ctx, "/source/link2.txt") + if err != nil { + t.Fatalf("failed to get link2.txt: %v", err) + } + + if link2.LinkTarget != "/outside/file.txt" { + t.Errorf("expected link2.txt target '/outside/file.txt', got %q", link2.LinkTarget) + } +} + +func TestScannerLargeFile(t *testing.T) { + // Create in-memory filesystem + fs := afero.NewMemMapFs() + + // Create a large file that will require multiple chunks + largeContent := make([]byte, 1024*1024) // 1MB + for i := range largeContent { + largeContent[i] = byte(i % 256) + } + + if err := fs.MkdirAll("/source", 0755); err != nil { + t.Fatal(err) + } + if err := afero.WriteFile(fs, "/source/large.bin", largeContent, 0644); err != nil { + t.Fatal(err) + } + + // Create test database + db, err := database.NewTestDB() + if err != nil { + t.Fatalf("failed to create test database: %v", err) + } + defer func() { + if err := db.Close(); err != nil { + t.Errorf("failed to close database: %v", err) + } + }() + + repos := database.NewRepositories(db) + + // Create scanner with 64KB chunks + scanner := backup.NewScanner(backup.ScannerConfig{ + FS: fs, + ChunkSize: 1024 * 64, // 64KB chunks + Repositories: repos, + }) + + // Scan the directory + ctx := context.Background() + result, err := scanner.Scan(ctx, "/source") + if err != nil { + t.Fatalf("scan failed: %v", err) + } + + if result.BytesScanned != 1024*1024 { + t.Errorf("expected %d bytes scanned, got %d", 1024*1024, result.BytesScanned) + } + + // Verify chunks + chunks, err := repos.FileChunks.GetByFile(ctx, "/source/large.bin") + if err != nil { + t.Fatalf("failed to get chunks: %v", err) + } + + expectedChunks := 16 // 1MB / 64KB + if len(chunks) != expectedChunks { + t.Errorf("expected %d chunks, got %d", expectedChunks, len(chunks)) + } + + // Verify chunk sequence + for i, fc := range chunks { + if fc.Idx != i { + t.Errorf("chunk %d has incorrect sequence %d", i, fc.Idx) + } + } +} diff --git a/internal/cli/app.go b/internal/cli/app.go index d2112c1..1d7f6c8 100644 --- a/internal/cli/app.go +++ b/internal/cli/app.go @@ -3,6 +3,7 @@ package cli import ( "context" "fmt" + "time" "git.eeqj.de/sneak/vaultik/internal/config" "git.eeqj.de/sneak/vaultik/internal/database" @@ -17,6 +18,16 @@ type AppOptions struct { Invokes []fx.Option } +// setupGlobals sets up the globals with application startup time +func setupGlobals(lc fx.Lifecycle, g *globals.Globals) { + lc.Append(fx.Hook{ + OnStart: func(ctx context.Context) error { + g.StartTime = time.Now() + return nil + }, + }) +} + // NewApp creates a new fx application with common modules func NewApp(opts AppOptions) *fx.App { baseModules := []fx.Option{ @@ -24,6 +35,7 @@ func NewApp(opts AppOptions) *fx.App { fx.Provide(globals.New), config.Module, database.Module, + fx.Invoke(setupGlobals), fx.NopLogger, } diff --git a/internal/database/chunks_ext.go b/internal/database/chunks_ext.go new file mode 100644 index 0000000..57fcaf4 --- /dev/null +++ b/internal/database/chunks_ext.go @@ -0,0 +1,38 @@ +package database + +import ( + "context" + "fmt" +) + +func (r *ChunkRepository) List(ctx context.Context) ([]*Chunk, error) { + query := ` + SELECT chunk_hash, sha256, size + FROM chunks + ORDER BY chunk_hash + ` + + rows, err := r.db.conn.QueryContext(ctx, query) + if err != nil { + return nil, fmt.Errorf("querying chunks: %w", err) + } + defer CloseRows(rows) + + var chunks []*Chunk + for rows.Next() { + var chunk Chunk + + err := rows.Scan( + &chunk.ChunkHash, + &chunk.SHA256, + &chunk.Size, + ) + if err != nil { + return nil, fmt.Errorf("scanning chunk: %w", err) + } + + chunks = append(chunks, &chunk) + } + + return chunks, rows.Err() +} diff --git a/internal/database/database.go b/internal/database/database.go index 46aceea..49c93a4 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -141,3 +141,8 @@ func (db *DB) createSchema(ctx context.Context) error { _, err := db.conn.ExecContext(ctx, schema) return err } + +// NewTestDB creates an in-memory SQLite database for testing +func NewTestDB() (*DB, error) { + return New(context.Background(), ":memory:") +} diff --git a/internal/database/file_chunks.go b/internal/database/file_chunks.go index cf9a946..86859b7 100644 --- a/internal/database/file_chunks.go +++ b/internal/database/file_chunks.go @@ -78,3 +78,8 @@ func (r *FileChunkRepository) DeleteByPath(ctx context.Context, tx *sql.Tx, path return nil } + +// GetByFile is an alias for GetByPath for compatibility +func (r *FileChunkRepository) GetByFile(ctx context.Context, path string) ([]*FileChunk, error) { + return r.GetByPath(ctx, path) +} diff --git a/internal/database/files.go b/internal/database/files.go index b8a5179..3705a73 100644 --- a/internal/database/files.go +++ b/internal/database/files.go @@ -143,3 +143,49 @@ func (r *FileRepository) Delete(ctx context.Context, tx *sql.Tx, path string) er return nil } + +func (r *FileRepository) ListByPrefix(ctx context.Context, prefix string) ([]*File, error) { + query := ` + SELECT path, mtime, ctime, size, mode, uid, gid, link_target + FROM files + WHERE path LIKE ? || '%' + ORDER BY path + ` + + rows, err := r.db.conn.QueryContext(ctx, query, prefix) + if err != nil { + return nil, fmt.Errorf("querying files: %w", err) + } + defer CloseRows(rows) + + var files []*File + for rows.Next() { + var file File + var mtimeUnix, ctimeUnix int64 + var linkTarget sql.NullString + + err := rows.Scan( + &file.Path, + &mtimeUnix, + &ctimeUnix, + &file.Size, + &file.Mode, + &file.UID, + &file.GID, + &linkTarget, + ) + if err != nil { + return nil, fmt.Errorf("scanning file: %w", err) + } + + file.MTime = time.Unix(mtimeUnix, 0) + file.CTime = time.Unix(ctimeUnix, 0) + if linkTarget.Valid { + file.LinkTarget = linkTarget.String + } + + files = append(files, &file) + } + + return files, rows.Err() +} diff --git a/internal/globals/globals.go b/internal/globals/globals.go index 53ad9aa..2c25e04 100644 --- a/internal/globals/globals.go +++ b/internal/globals/globals.go @@ -19,12 +19,9 @@ type Globals struct { } func New() (*Globals, error) { - n := &Globals{ - Appname: Appname, - Version: Version, - Commit: Commit, - StartTime: time.Now(), - } - - return n, nil + return &Globals{ + Appname: Appname, + Version: Version, + Commit: Commit, + }, nil } diff --git a/internal/globals/globals_test.go b/internal/globals/globals_test.go index 82fe7c3..164602c 100644 --- a/internal/globals/globals_test.go +++ b/internal/globals/globals_test.go @@ -2,35 +2,29 @@ package globals import ( "testing" - - "go.uber.org/fx" - "go.uber.org/fx/fxtest" ) // TestGlobalsNew ensures the globals package initializes correctly func TestGlobalsNew(t *testing.T) { - app := fxtest.New(t, - fx.Provide(New), - fx.Invoke(func(g *Globals) { - if g == nil { - t.Fatal("Globals instance is nil") - } + g, err := New() + if err != nil { + t.Fatalf("Failed to create Globals: %v", err) + } - if g.Appname != "vaultik" { - t.Errorf("Expected Appname to be 'vaultik', got '%s'", g.Appname) - } + if g == nil { + t.Fatal("Globals instance is nil") + } - // Version and Commit will be "dev" and "unknown" by default - if g.Version == "" { - t.Error("Version should not be empty") - } + if g.Appname != "vaultik" { + t.Errorf("Expected Appname to be 'vaultik', got '%s'", g.Appname) + } - if g.Commit == "" { - t.Error("Commit should not be empty") - } - }), - ) + // Version and Commit will be "dev" and "unknown" by default + if g.Version == "" { + t.Error("Version should not be empty") + } - app.RequireStart() - app.RequireStop() + if g.Commit == "" { + t.Error("Commit should not be empty") + } } diff --git a/internal/s3/s3_test.go b/internal/s3/s3_test.go index 890a81b..bc359f0 100644 --- a/internal/s3/s3_test.go +++ b/internal/s3/s3_test.go @@ -3,6 +3,7 @@ package s3_test import ( "bytes" "context" + "fmt" "io" "net/http" "os" @@ -14,6 +15,7 @@ import ( "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/smithy-go/logging" "github.com/johannesboyne/gofakes3" "github.com/johannesboyne/gofakes3/backend/s3mem" ) @@ -32,6 +34,7 @@ type TestServer struct { backend gofakes3.Backend s3Client *s3.Client tempDir string + logBuf *bytes.Buffer } // NewTestServer creates and starts a new test server @@ -62,7 +65,10 @@ func NewTestServer(t *testing.T) *TestServer { // Wait for server to be ready time.Sleep(100 * time.Millisecond) - // Create S3 client + // Create a buffer to capture logs + logBuf := &bytes.Buffer{} + + // Create S3 client with custom logger cfg, err := config.LoadDefaultConfig(context.Background(), config.WithRegion(testRegion), config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider( @@ -71,6 +77,13 @@ func NewTestServer(t *testing.T) *TestServer { "", )), config.WithClientLogMode(aws.LogRetries|aws.LogRequestWithBody|aws.LogResponseWithBody), + config.WithLogger(logging.LoggerFunc(func(classification logging.Classification, format string, v ...interface{}) { + // Capture logs to buffer instead of stdout + fmt.Fprintf(logBuf, "SDK %s %s %s\n", + time.Now().Format("2006/01/02 15:04:05"), + string(classification), + fmt.Sprintf(format, v...)) + })), ) if err != nil { t.Fatalf("failed to create AWS config: %v", err) @@ -86,8 +99,16 @@ func NewTestServer(t *testing.T) *TestServer { backend: backend, s3Client: s3Client, tempDir: tempDir, + logBuf: logBuf, } + // Register cleanup to show logs on test failure + t.Cleanup(func() { + if t.Failed() && logBuf.Len() > 0 { + t.Logf("S3 SDK Debug Output:\n%s", logBuf.String()) + } + }) + // Create test bucket _, err = s3Client.CreateBucket(context.Background(), &s3.CreateBucketInput{ Bucket: aws.String(testBucket),