From 6cf92114075d4ebdd835353417b10c4eea1bca95 Mon Sep 17 00:00:00 2001 From: sneak Date: Fri, 7 Aug 2026 17:01:52 +0000 Subject: [PATCH 1/2] Apply mechanical lint fixes for golangci-lint v2.12.2 rollout Auto-remediate style-only findings (wsl_v5, nlreturn, noinlineerr, modernize, intrange, perfsprint, usetesting, unconvert, errorlint, gocritic, testifylint) and rename printf-style helpers to f-suffixed names (goprintffuncname): ui.Writer message methods, cli.ReportErrorf, database.Fatalf, vaultik stdoutf. --- .golangci.yml | 26 +- Dockerfile | 4 +- Makefile | 2 +- cmd/vaultik/main.go | 4 + internal/blob/packer.go | 67 +++- internal/blob/packer_test.go | 31 +- internal/blobgen/compress.go | 2 + internal/blobgen/compress_test.go | 13 +- internal/blobgen/reader.go | 2 + internal/blobgen/writer.go | 17 +- internal/chunker/chunker.go | 11 +- internal/chunker/chunker_isolated_test.go | 4 +- internal/chunker/chunker_test.go | 3 + internal/chunker/fastcdc.go | 21 +- internal/cli/app.go | 27 +- internal/cli/config.go | 33 +- internal/cli/config_test.go | 22 +- internal/cli/database.go | 4 + internal/cli/duration.go | 6 +- internal/cli/duration_test.go | 1 + internal/cli/entry.go | 15 +- internal/cli/entry_test.go | 6 + internal/cli/info.go | 15 +- internal/cli/prune.go | 15 +- internal/cli/remote.go | 33 +- internal/cli/root.go | 3 + internal/cli/snapshot.go | 81 ++-- internal/cli/snapshot_restore.go | 16 +- internal/cli/version.go | 1 + internal/config/config.go | 48 ++- internal/config/config_test.go | 1 + internal/config/size.go | 8 +- internal/crypto/encryption.go | 7 +- internal/crypto/encryption_test.go | 5 + internal/database/blob_chunks.go | 15 + internal/database/blob_chunks_test.go | 15 + internal/database/blobs.go | 45 ++- internal/database/blobs_test.go | 11 + internal/database/cascade_debug_test.go | 17 +- internal/database/chunk_files.go | 44 ++- internal/database/chunk_files_test.go | 16 + internal/database/chunks.go | 18 +- internal/database/chunks_ext.go | 1 + internal/database/chunks_test.go | 8 + internal/database/database.go | 38 +- internal/database/database_test.go | 27 +- internal/database/errors.go | 9 +- internal/database/file_chunks.go | 47 ++- internal/database/file_chunks_test.go | 17 +- internal/database/files.go | 70 +++- internal/database/files_test.go | 22 +- internal/database/local_meta.go | 4 + internal/database/local_meta_test.go | 5 +- internal/database/module.go | 7 +- internal/database/repositories.go | 25 +- internal/database/repositories_test.go | 54 ++- .../database/repository_comprehensive_test.go | 65 +++- internal/database/repository_debug_test.go | 27 +- .../database/repository_edge_cases_test.go | 22 +- internal/database/snapshots.go | 80 ++-- internal/database/snapshots_test.go | 15 +- internal/database/uploads.go | 15 +- internal/globals/globals.go | 1 + internal/log/log.go | 3 + internal/log/tty_handler.go | 8 + internal/pidlock/pidlock.go | 2 + internal/pidlock/pidlock_test.go | 4 + internal/s3/client.go | 26 +- internal/s3/client_test.go | 11 +- internal/s3/s3_test.go | 20 +- internal/snapshot/backup_test.go | 41 +- internal/snapshot/exclude_test.go | 34 ++ internal/snapshot/file_change_test.go | 15 +- internal/snapshot/manifest.go | 2 + internal/snapshot/permission_error_test.go | 6 +- internal/snapshot/progress.go | 18 +- internal/snapshot/remotekey.go | 1 + internal/snapshot/scanner.go | 276 ++++++++++---- internal/snapshot/scanner_test.go | 27 +- internal/snapshot/snapshot.go | 160 ++++++-- internal/snapshot/snapshot_test.go | 13 +- internal/storage/file.go | 25 +- internal/storage/module.go | 3 + internal/storage/rclone.go | 12 +- internal/storage/s3.go | 4 + internal/storage/url.go | 26 +- internal/types/types.go | 14 +- internal/ui/ui.go | 57 ++- internal/ui/ui_test.go | 48 ++- internal/vaultik/blob_fetch.go | 10 +- internal/vaultik/blob_fetch_hash_test.go | 12 + internal/vaultik/blobcache.go | 39 +- internal/vaultik/blobcache_test.go | 20 +- internal/vaultik/helpers.go | 10 +- internal/vaultik/helpers_test.go | 3 + internal/vaultik/info.go | 149 +++++--- internal/vaultik/integration_test.go | 77 +++- internal/vaultik/prune.go | 65 +++- internal/vaultik/purge_per_name_test.go | 3 + internal/vaultik/remove_snapshot_test.go | 14 +- internal/vaultik/restore.go | 172 +++++++-- internal/vaultik/restore_locality_test.go | 32 +- internal/vaultik/restore_plan.go | 24 ++ internal/vaultik/restore_sweeper.go | 11 +- .../restore_sweeper_integration_test.go | 24 +- internal/vaultik/snapshot.go | 349 +++++++++++++----- internal/vaultik/storage_bind.go | 5 +- internal/vaultik/vaultik.go | 13 +- internal/vaultik/verify.go | 103 ++++-- internal/vaultik/verify_test.go | 3 + 110 files changed, 2566 insertions(+), 722 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 34a8e31..26b1610 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,5 +1,9 @@ version: "2" +# Config schema uses the golangci-lint v2 layout (settings live under +# linters.settings, not top-level linters-settings) so that the +# thresholds below are actually applied by golangci-lint >= v2. + run: timeout: 5m modules-download-mode: readonly @@ -14,19 +18,17 @@ linters: - wsl # Deprecated, replaced by wsl_v5 - wrapcheck # Too verbose for internal packages - varnamelen # Short names like db, id are idiomatic Go - -linters-settings: - lll: - line-length: 88 - funlen: - lines: 80 - statements: 50 - cyclop: - max-complexity: 15 - dupl: - threshold: 100 + settings: + lll: + line-length: 88 + funlen: + lines: 80 + statements: 50 + cyclop: + max-complexity: 15 + dupl: + threshold: 100 issues: - exclude-use-default: false max-issues-per-linter: 0 max-same-issues: 0 diff --git a/Dockerfile b/Dockerfile index 72ea9b2..cebd9d9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ # Lint stage -# golangci/golangci-lint:v2.11.3-alpine, 2026-03-17 -FROM golangci/golangci-lint:v2.11.3-alpine@sha256:b1c3de5862ad0a95b4e45a993b0f00415835d687e4f12c845c7493b86c13414e AS lint +# golangci/golangci-lint:v2.12.2-alpine, 2026-08-07 +FROM golangci/golangci-lint:v2.12.2-alpine@sha256:91b27804074a0bacea298707f016911e60cf0cdbc6c7bf5ccacb5f0606d18d60 AS lint RUN apk add --no-cache make build-base diff --git a/Makefile b/Makefile index c026ebf..e70ab9e 100644 --- a/Makefile +++ b/Makefile @@ -55,7 +55,7 @@ clean: # Install dependencies. deps: go mod download - go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest + go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.12.2 # Run tests with coverage. test-coverage: diff --git a/cmd/vaultik/main.go b/cmd/vaultik/main.go index 70e7c19..234de24 100644 --- a/cmd/vaultik/main.go +++ b/cmd/vaultik/main.go @@ -16,9 +16,11 @@ func main() { panic("could not create CPU profile: " + err.Error()) } defer func() { _ = f.Close() }() + if err := pprof.StartCPUProfile(f); err != nil { panic("could not start CPU profile: " + err.Error()) } + defer pprof.StopCPUProfile() } @@ -30,7 +32,9 @@ func main() { panic("could not create memory profile: " + err.Error()) } defer func() { _ = f.Close() }() + runtime.GC() // get up-to-date statistics + if err := pprof.WriteHeapProfile(f); err != nil { panic("could not write memory profile: " + err.Error()) } diff --git a/internal/blob/packer.go b/internal/blob/packer.go index 116c365..1e02096 100644 --- a/internal/blob/packer.go +++ b/internal/blob/packer.go @@ -18,6 +18,7 @@ import ( "context" "database/sql" "encoding/hex" + "errors" "fmt" "io" "sync" @@ -124,6 +125,7 @@ type BlobChunkRef struct { // BlobWithReader wraps a FinishedBlob with its data reader type BlobWithReader struct { *FinishedBlob + Reader io.ReadSeeker TempFile afero.File // Optional, only set for disk-based blobs InsertedChunkHashes []string // Chunk hashes that were inserted to DB with this blob @@ -134,14 +136,17 @@ type BlobWithReader struct { // Returns an error if required configuration fields are missing or invalid. func NewPacker(cfg PackerConfig) (*Packer, error) { if len(cfg.Recipients) == 0 { - return nil, fmt.Errorf("recipients are required - blobs must be encrypted") + return nil, errors.New("recipients are required - blobs must be encrypted") } + if cfg.MaxBlobSize <= 0 { - return nil, fmt.Errorf("max blob size must be positive") + return nil, errors.New("max blob size must be positive") } + if cfg.Fs == nil { - return nil, fmt.Errorf("filesystem is required") + return nil, errors.New("filesystem is required") } + return &Packer{ maxBlobSize: cfg.MaxBlobSize, compressionLevel: cfg.CompressionLevel, @@ -160,6 +165,7 @@ func NewPacker(cfg PackerConfig) (*Packer, error) { func (p *Packer) SetBlobHandler(handler BlobHandler) { p.mu.Lock() defer p.mu.Unlock() + p.blobHandler = handler } @@ -169,6 +175,7 @@ func (p *Packer) SetBlobHandler(handler BlobHandler) { func (p *Packer) AddPendingChunk(hash string, size int64) { p.mu.Lock() defer p.mu.Unlock() + p.pendingChunks = append(p.pendingChunks, PendingChunk{Hash: hash, Size: size}) } @@ -183,7 +190,8 @@ func (p *Packer) AddChunk(chunk *ChunkRef) error { // Initialize new blob if needed if p.currentBlob == nil { - if err := p.startNewBlob(); err != nil { + err := p.startNewBlob() + if err != nil { return fmt.Errorf("starting new blob: %w", err) } } @@ -202,7 +210,8 @@ func (p *Packer) AddChunk(chunk *ChunkRef) error { } // Add chunk to current blob - if err := p.addChunkToCurrentBlob(chunk); err != nil { + err := p.addChunkToCurrentBlob(chunk) + if err != nil { return err } @@ -218,7 +227,8 @@ func (p *Packer) Flush() error { defer p.mu.Unlock() if p.currentBlob != nil && len(p.currentBlob.chunks) > 0 { - if err := p.finalizeCurrentBlob(); err != nil { + err := p.finalizeCurrentBlob() + if err != nil { return fmt.Errorf("finalizing blob: %w", err) } } @@ -253,6 +263,7 @@ func (p *Packer) GetFinishedBlobs() []*FinishedBlob { blobs := p.finishedBlobs p.finishedBlobs = make([]*FinishedBlob, 0) + return blobs } @@ -267,6 +278,7 @@ func (p *Packer) startNewBlob() error { if err != nil { return fmt.Errorf("parsing blob ID: %w", err) } + blob := &database.Blob{ ID: blobIDTyped, Hash: types.BlobHash("temp-placeholder-" + blobID), // Temporary placeholder until finalized @@ -276,6 +288,7 @@ func (p *Packer) startNewBlob() error { CompressedSize: 0, UploadedTS: nil, } + if err := p.repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error { return p.repos.Blobs.Create(ctx, tx, blob) }); err != nil { @@ -294,6 +307,7 @@ func (p *Packer) startNewBlob() error { if err != nil { _ = tempFile.Close() _ = p.fs.Remove(tempFile.Name()) + return fmt.Errorf("creating blobgen writer: %w", err) } @@ -308,6 +322,7 @@ func (p *Packer) startNewBlob() error { } log.Debug("Created new blob container", "blob_id", blobID, "temp_file", tempFile.Name()) + return nil } @@ -316,6 +331,7 @@ func (p *Packer) addChunkToCurrentBlob(chunk *ChunkRef) error { // Skip if chunk already in current blob if p.currentBlob.chunkSet[chunk.Hash] { log.Debug("Skipping duplicate chunk already in current blob", "chunk_hash", chunk.Hash) + return nil } @@ -403,24 +419,31 @@ func (p *Packer) finalizeCurrentBlob() error { func (p *Packer) closeBlobWriter() (string, int64, error) { if err := p.currentBlob.writer.Close(); err != nil { p.cleanupTempFile() + return "", 0, fmt.Errorf("closing blobgen writer: %w", err) } + if err := p.currentBlob.tempFile.Sync(); err != nil { p.cleanupTempFile() + return "", 0, fmt.Errorf("syncing temp file: %w", err) } finalSize, err := p.currentBlob.tempFile.Seek(0, io.SeekCurrent) if err != nil { p.cleanupTempFile() + return "", 0, fmt.Errorf("getting file size: %w", err) } + if _, err := p.currentBlob.tempFile.Seek(0, io.SeekStart); err != nil { p.cleanupTempFile() + return "", 0, fmt.Errorf("seeking to start: %w", err) } finalHash := p.currentBlob.writer.Sum256() + return hex.EncodeToString(finalHash), finalSize, nil } @@ -432,6 +455,7 @@ func (p *Packer) buildChunkRefs() []*BlobChunkRef { ChunkHash: chunk.Hash, Offset: chunk.Offset, Length: chunk.Size, }) } + return refs } @@ -444,13 +468,16 @@ func (p *Packer) commitBlobToDatabase(blobHash string, finalSize int64, chunksTo blobIDTyped, parseErr := types.ParseBlobID(p.currentBlob.id) if parseErr != nil { p.cleanupTempFile() + return fmt.Errorf("parsing blob ID: %w", parseErr) } err := p.repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error { for _, chunk := range chunksToInsert { dbChunk := &database.Chunk{ChunkHash: types.ChunkHash(chunk.Hash), Size: chunk.Size} - if err := p.repos.Chunks.Create(ctx, tx, dbChunk); err != nil { + + err := p.repos.Chunks.Create(ctx, tx, dbChunk) + if err != nil { return fmt.Errorf("creating chunk: %w", err) } } @@ -460,7 +487,9 @@ func (p *Packer) commitBlobToDatabase(blobHash string, finalSize int64, chunksTo BlobID: blobIDTyped, ChunkHash: types.ChunkHash(chunk.Hash), Offset: chunk.Offset, Length: chunk.Size, } - if err := p.repos.BlobChunks.Create(ctx, tx, blobChunk); err != nil { + + err := p.repos.BlobChunks.Create(ctx, tx, blobChunk) + if err != nil { return fmt.Errorf("creating blob_chunk: %w", err) } } @@ -469,11 +498,13 @@ func (p *Packer) commitBlobToDatabase(blobHash string, finalSize int64, chunksTo }) if err != nil { p.cleanupTempFile() + return fmt.Errorf("finalizing blob transaction: %w", err) } log.Debug("Committed blob transaction", "chunks_inserted", len(chunksToInsert), "blob_chunks_inserted", len(p.currentBlob.chunks)) + return nil } @@ -482,6 +513,7 @@ func (p *Packer) deliverFinishedBlob(finished *FinishedBlob, insertedChunkHashes if p.blobHandler != nil { if _, err := p.currentBlob.tempFile.Seek(0, io.SeekStart); err != nil { p.cleanupTempFile() + return fmt.Errorf("seeking for handler: %w", err) } @@ -492,30 +524,39 @@ func (p *Packer) deliverFinishedBlob(finished *FinishedBlob, insertedChunkHashes InsertedChunkHashes: insertedChunkHashes, } - if err := p.blobHandler(blobWithReader); err != nil { + err := p.blobHandler(blobWithReader) + if err != nil { p.cleanupTempFile() + return fmt.Errorf("blob handler failed: %w", err) } + p.currentBlob = nil + return nil } // No handler - read data for legacy behavior log.Debug("No blob handler callback configured", "blob_hash", finished.Hash[:8]+"...") + if _, err := p.currentBlob.tempFile.Seek(0, io.SeekStart); err != nil { p.cleanupTempFile() + return fmt.Errorf("seeking to read data: %w", err) } data, err := io.ReadAll(p.currentBlob.tempFile) if err != nil { p.cleanupTempFile() + return fmt.Errorf("reading blob data: %w", err) } + finished.Data = data p.finishedBlobs = append(p.finishedBlobs, finished) p.cleanupTempFile() p.currentBlob = nil + return nil } @@ -532,13 +573,15 @@ func (p *Packer) cleanupTempFile() { func (p *Packer) PackChunks(chunks []*ChunkRef) error { for _, chunk := range chunks { err := p.AddChunk(chunk) - if err == ErrBlobSizeLimitExceeded { + if errors.Is(err, ErrBlobSizeLimitExceeded) { // Finalize current blob and retry - if err := p.FinalizeBlob(); err != nil { + err := p.FinalizeBlob() + if err != nil { return fmt.Errorf("finalizing blob before retry: %w", err) } // Retry the chunk - if err := p.AddChunk(chunk); err != nil { + err = p.AddChunk(chunk) + if err != nil { return fmt.Errorf("adding chunk %s after finalize: %w", chunk.Hash, err) } } else if err != nil { diff --git a/internal/blob/packer_test.go b/internal/blob/packer_test.go index dba194b..d44bc22 100644 --- a/internal/blob/packer_test.go +++ b/internal/blob/packer_test.go @@ -6,6 +6,7 @@ import ( "crypto/sha256" "database/sql" "encoding/hex" + "errors" "io" "testing" @@ -40,6 +41,7 @@ func TestPacker(t *testing.T) { t.Fatalf("failed to create test db: %v", err) } defer func() { _ = db.Close() }() + repos := database.NewRepositories(db) cfg := PackerConfig{ @@ -49,6 +51,7 @@ func TestPacker(t *testing.T) { Repositories: repos, Fs: afero.NewMemMapFs(), } + packer, err := NewPacker(cfg) if err != nil { t.Fatalf("failed to create packer: %v", err) @@ -64,6 +67,7 @@ func TestPacker(t *testing.T) { ChunkHash: types.ChunkHash(hashStr), Size: int64(len(data)), } + err = repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error { return repos.Chunks.Create(ctx, tx, dbChunk) }) @@ -130,6 +134,7 @@ func TestPacker(t *testing.T) { t.Fatalf("failed to create test db: %v", err) } defer func() { _ = db.Close() }() + repos := database.NewRepositories(db) cfg := PackerConfig{ @@ -139,6 +144,7 @@ func TestPacker(t *testing.T) { Repositories: repos, Fs: afero.NewMemMapFs(), } + packer, err := NewPacker(cfg) if err != nil { t.Fatalf("failed to create packer: %v", err) @@ -146,7 +152,8 @@ func TestPacker(t *testing.T) { // Create multiple small chunks chunks := make([]*ChunkRef, 10) - for i := 0; i < 10; i++ { + + for i := range 10 { data := bytes.Repeat([]byte{byte(i)}, 1000) hash := sha256.Sum256(data) hashStr := hex.EncodeToString(hash[:]) @@ -156,6 +163,7 @@ func TestPacker(t *testing.T) { ChunkHash: types.ChunkHash(hashStr), Size: int64(len(data)), } + err = repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error { return repos.Chunks.Create(ctx, tx, dbChunk) }) @@ -198,9 +206,11 @@ func TestPacker(t *testing.T) { if chunkRef.Offset != expectedOffset { t.Errorf("chunk %d: expected offset %d, got %d", i, expectedOffset, chunkRef.Offset) } + if chunkRef.Length != 1000 { t.Errorf("chunk %d: expected length 1000, got %d", i, chunkRef.Length) } + expectedOffset += chunkRef.Length } }) @@ -212,6 +222,7 @@ func TestPacker(t *testing.T) { t.Fatalf("failed to create test db: %v", err) } defer func() { _ = db.Close() }() + repos := database.NewRepositories(db) // Small blob size limit to force multiple blobs @@ -222,6 +233,7 @@ func TestPacker(t *testing.T) { Repositories: repos, Fs: afero.NewMemMapFs(), } + packer, err := NewPacker(cfg) if err != nil { t.Fatalf("failed to create packer: %v", err) @@ -229,7 +241,8 @@ func TestPacker(t *testing.T) { // Create chunks that will exceed the limit chunks := make([]*ChunkRef, 10) - for i := 0; i < 10; i++ { + + for i := range 10 { data := bytes.Repeat([]byte{byte(i)}, 1000) // 1KB each hash := sha256.Sum256(data) hashStr := hex.EncodeToString(hash[:]) @@ -239,6 +252,7 @@ func TestPacker(t *testing.T) { ChunkHash: types.ChunkHash(hashStr), Size: int64(len(data)), } + err = repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error { return repos.Chunks.Create(ctx, tx, dbChunk) }) @@ -257,14 +271,17 @@ func TestPacker(t *testing.T) { // Add chunks and handle size limit errors for _, chunk := range chunks { err := packer.AddChunk(chunk) - if err == ErrBlobSizeLimitExceeded { + if errors.Is(err, ErrBlobSizeLimitExceeded) { // Finalize current blob - if err := packer.FinalizeBlob(); err != nil { + err := packer.FinalizeBlob() + if err != nil { t.Fatalf("failed to finalize blob: %v", err) } + blobCount++ // Retry adding the chunk - if err := packer.AddChunk(chunk); err != nil { + err = packer.AddChunk(chunk) + if err != nil { t.Fatalf("failed to add chunk after finalize: %v", err) } } else if err != nil { @@ -301,6 +318,7 @@ func TestPacker(t *testing.T) { t.Fatalf("failed to create test db: %v", err) } defer func() { _ = db.Close() }() + repos := database.NewRepositories(db) // Generate test identity (using the one from parent test) @@ -311,6 +329,7 @@ func TestPacker(t *testing.T) { Repositories: repos, Fs: afero.NewMemMapFs(), } + packer, err := NewPacker(cfg) if err != nil { t.Fatalf("failed to create packer: %v", err) @@ -326,6 +345,7 @@ func TestPacker(t *testing.T) { ChunkHash: types.ChunkHash(hashStr), Size: int64(len(data)), } + err = repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error { return repos.Chunks.Create(ctx, tx, dbChunk) }) @@ -342,6 +362,7 @@ func TestPacker(t *testing.T) { if err := packer.AddChunk(chunk); err != nil { t.Fatalf("failed to add chunk: %v", err) } + if err := packer.Flush(); err != nil { t.Fatalf("failed to flush: %v", err) } diff --git a/internal/blobgen/compress.go b/internal/blobgen/compress.go index e8a8799..20f8571 100644 --- a/internal/blobgen/compress.go +++ b/internal/blobgen/compress.go @@ -28,6 +28,7 @@ func CompressData(data []byte, compressionLevel int, recipients []string) (*Comp // Write data if _, err := w.Write(data); err != nil { _ = w.Close() + return nil, fmt.Errorf("writing data: %w", err) } @@ -68,6 +69,7 @@ func CompressStream(dst io.Writer, src io.Reader, compressionLevel int, recipien if err := w.Close(); err != nil { return 0, "", fmt.Errorf("closing writer: %w", err) } + closed = true return w.BytesWritten(), hex.EncodeToString(w.Sum256()), nil diff --git a/internal/blobgen/compress_test.go b/internal/blobgen/compress_test.go index 6d1240c..37c41e9 100644 --- a/internal/blobgen/compress_test.go +++ b/internal/blobgen/compress_test.go @@ -20,13 +20,14 @@ const testRecipient = "age1cplgrwj77ta54dnmydvvmzn64ltk83ankxl5sww04mrtmu62kv3s8 // cause a double close. func TestCompressStreamNoDoubleClose(t *testing.T) { input := []byte("regression test data for issue #28 double-close fix") + var buf bytes.Buffer written, hash, err := CompressStream(&buf, bytes.NewReader(input), 3, []string{testRecipient}) require.NoError(t, err, "CompressStream should not return an error") - assert.True(t, written > 0, "expected bytes written > 0") + assert.Positive(t, written, "expected bytes written > 0") assert.NotEmpty(t, hash, "expected non-empty hash") - assert.True(t, buf.Len() > 0, "expected non-empty output") + assert.Positive(t, buf.Len(), "expected non-empty output") } // TestCompressStreamLargeInput exercises CompressStream with a larger payload @@ -37,9 +38,10 @@ func TestCompressStreamLargeInput(t *testing.T) { require.NoError(t, err) var buf bytes.Buffer + written, hash, err := CompressStream(&buf, bytes.NewReader(data), 3, []string{testRecipient}) require.NoError(t, err) - assert.True(t, written > 0) + assert.Positive(t, written) assert.NotEmpty(t, hash) } @@ -47,6 +49,7 @@ func TestCompressStreamLargeInput(t *testing.T) { // without double-close issues. func TestCompressStreamEmptyInput(t *testing.T) { var buf bytes.Buffer + _, hash, err := CompressStream(&buf, strings.NewReader(""), 3, []string{testRecipient}) require.NoError(t, err) assert.NotEmpty(t, hash) @@ -58,7 +61,7 @@ func TestCompressDataNoDoubleClose(t *testing.T) { input := []byte("CompressData regression test for double-close") result, err := CompressData(input, 3, []string{testRecipient}) require.NoError(t, err) - assert.True(t, result.CompressedSize > 0) - assert.True(t, result.UncompressedSize == int64(len(input))) + assert.Positive(t, result.CompressedSize) + assert.Equal(t, result.UncompressedSize, int64(len(input))) assert.NotEmpty(t, result.SHA256) } diff --git a/internal/blobgen/reader.go b/internal/blobgen/reader.go index cc3edd4..6d7f2f8 100644 --- a/internal/blobgen/reader.go +++ b/internal/blobgen/reader.go @@ -53,12 +53,14 @@ func NewReader(r io.Reader, identity age.Identity) (*Reader, error) { func (r *Reader) Read(p []byte) (n int, err error) { n, err = r.teeReader.Read(p) r.bytesRead += int64(n) + return n, err } // Close closes the decompressor func (r *Reader) Close() error { r.decompressor.Close() + return nil } diff --git a/internal/blobgen/writer.go b/internal/blobgen/writer.go index 920a572..a14574c 100644 --- a/internal/blobgen/writer.go +++ b/internal/blobgen/writer.go @@ -36,11 +36,13 @@ func NewWriter(w io.Writer, compressionLevel int, recipients []string) (*Writer, // Parse recipients var ageRecipients []age.Recipient + for _, recipient := range recipients { r, err := age.ParseX25519Recipient(recipient) if err != nil { return nil, fmt.Errorf("parsing recipient %s: %w", recipient, err) } + ageRecipients = append(ageRecipients, r) } @@ -51,10 +53,7 @@ func NewWriter(w io.Writer, compressionLevel int, recipients []string) (*Writer, } // Calculate compression concurrency: CPUs - 2, minimum 1 - concurrency := runtime.NumCPU() - 2 - if concurrency < 1 { - concurrency = 1 - } + concurrency := max(runtime.NumCPU()-2, 1) // Create compression writer with encryption as destination compressor, err := zstd.NewWriter(encWriter, @@ -63,6 +62,7 @@ func NewWriter(w io.Writer, compressionLevel int, recipients []string) (*Writer, ) if err != nil { _ = encWriter.Close() + return nil, fmt.Errorf("creating compression writer: %w", err) } @@ -82,18 +82,21 @@ func NewWriter(w io.Writer, compressionLevel int, recipients []string) (*Writer, func (w *Writer) Write(p []byte) (n int, err error) { n, err = w.teeWriter.Write(p) w.bytesWritten += int64(n) + return n, err } // Close closes all layers and returns any errors func (w *Writer) Close() error { // Close compressor first - if err := w.compressor.Close(); err != nil { + err := w.compressor.Close() + if err != nil { return fmt.Errorf("closing compressor: %w", err) } // Then close encryptor - if err := w.encryptor.Close(); err != nil { + err = w.encryptor.Close() + if err != nil { return fmt.Errorf("closing encryptor: %w", err) } @@ -109,6 +112,7 @@ func (w *Writer) Sum256() []byte { firstHash := w.hasher.Sum(nil) // Second hash: SHA256(firstHash) - this is the blob ID secondHash := sha256.Sum256(firstHash) + return secondHash[:] } @@ -123,5 +127,6 @@ func validateCompressionLevel(level int) error { if level < 1 || level > 19 { return fmt.Errorf("invalid compression level %d: must be between 1 and 19", level) } + return nil } diff --git a/internal/chunker/chunker.go b/internal/chunker/chunker.go index 3995990..1615899 100644 --- a/internal/chunker/chunker.go +++ b/internal/chunker/chunker.go @@ -3,6 +3,7 @@ package chunker import ( "crypto/sha256" "encoding/hex" + "errors" "fmt" "io" "os" @@ -50,13 +51,15 @@ func (c *Chunker) ChunkReader(r io.Reader) ([]Chunk, error) { defer chunker.Release() var chunks []Chunk + offset := int64(0) for { chunk, err := chunker.Next() - if err == io.EOF { + if errors.Is(err, io.EOF) { break } + if err != nil { return nil, fmt.Errorf("reading chunk: %w", err) } @@ -104,9 +107,10 @@ func (c *Chunker) ChunkReaderStreaming(r io.Reader, callback ChunkCallback) (str for { chunk, err := chunker.Next() - if err == io.EOF { + if errors.Is(err, io.EOF) { break } + if err != nil { return "", fmt.Errorf("reading chunk: %w", err) } @@ -143,7 +147,8 @@ func (c *Chunker) ChunkFile(path string) ([]Chunk, error) { return nil, fmt.Errorf("opening file: %w", err) } defer func() { - if err := file.Close(); err != nil && err.Error() != "invalid argument" { + err := file.Close() + if err != nil && err.Error() != "invalid argument" { // Log error or handle as needed _ = err } diff --git a/internal/chunker/chunker_isolated_test.go b/internal/chunker/chunker_isolated_test.go index 8e3fb0f..b5f86bf 100644 --- a/internal/chunker/chunker_isolated_test.go +++ b/internal/chunker/chunker_isolated_test.go @@ -42,7 +42,7 @@ func TestChunkerExpectedChunkCount(t *testing.T) { // Create data with some variation to trigger chunk boundaries data := make([]byte, tt.fileSize) - for i := 0; i < len(data); i++ { + for i := range data { // Use a pattern that should create boundaries data[i] = byte((i * 17) ^ (i >> 5)) } @@ -59,6 +59,7 @@ func TestChunkerExpectedChunkCount(t *testing.T) { t.Errorf("too few chunks: got %d, expected at least %d", len(chunks), tt.minExpected) } + if len(chunks) > tt.maxExpected { t.Errorf("too many chunks: got %d, expected at most %d", len(chunks), tt.maxExpected) @@ -69,6 +70,7 @@ func TestChunkerExpectedChunkCount(t *testing.T) { for _, chunk := range chunks { reconstructed = append(reconstructed, chunk.Data...) } + if !bytes.Equal(data, reconstructed) { t.Error("reconstructed data doesn't match original") } diff --git a/internal/chunker/chunker_test.go b/internal/chunker/chunker_test.go index a13e143..fca59f2 100644 --- a/internal/chunker/chunker_test.go +++ b/internal/chunker/chunker_test.go @@ -60,6 +60,7 @@ func TestChunker(t *testing.T) { if chunk.Offset != expectedOffset { t.Errorf("chunk %d: expected offset %d, got %d", i, expectedOffset, chunk.Offset) } + expectedOffset += chunk.Size } }) @@ -90,6 +91,7 @@ func TestChunker(t *testing.T) { if chunks1[i].Hash != chunks2[i].Hash { t.Errorf("chunk %d: different hashes", i) } + if chunks1[i].Size != chunks2[i].Size { t.Errorf("chunk %d: different sizes", i) } @@ -121,6 +123,7 @@ func TestChunkBoundaries(t *testing.T) { if i < len(chunks)-1 && chunk.Size < minSize { t.Errorf("chunk %d size %d is below minimum %d", i, chunk.Size, minSize) } + if chunk.Size > maxSize { t.Errorf("chunk %d size %d exceeds maximum %d", i, chunk.Size, maxSize) } diff --git a/internal/chunker/fastcdc.go b/internal/chunker/fastcdc.go index 54c87c7..223eb2b 100644 --- a/internal/chunker/fastcdc.go +++ b/internal/chunker/fastcdc.go @@ -28,7 +28,7 @@ type ReusableChunker struct { // reusableChunkerPool pools ReusableChunker instances to avoid allocations. var reusableChunkerPool = sync.Pool{ - New: func() interface{} { + New: func() any { return &ReusableChunker{} }, } @@ -39,17 +39,20 @@ var bufferPools = sync.Map{} func getBuffer(size int) []byte { poolI, _ := bufferPools.LoadOrStore(size, &sync.Pool{ - New: func() interface{} { + New: func() any { buf := make([]byte, size) + return &buf }, }) pool := poolI.(*sync.Pool) + return *pool.Get().(*[]byte) } func putBuffer(buf []byte) { size := cap(buf) + poolI, ok := bufferPools.Load(size) if ok { pool := poolI.(*sync.Pool) @@ -77,6 +80,7 @@ func AcquireReusableChunker(rd io.Reader, minSize, avgSize, maxSize int) *Reusab if c.buf != nil { putBuffer(c.buf) } + c.buf = getBuffer(bufSize) } else { // Restore buffer to full capacity (may have been truncated by previous EOF) @@ -120,6 +124,7 @@ func (c *ReusableChunker) fillBuffer() error { if c.eof { c.buf = c.buf[:n] + return nil } @@ -134,15 +139,18 @@ func (c *ReusableChunker) fillBuffer() error { } else if err != nil { return err } + return nil } // Next returns the next chunk or io.EOF when done. // The returned Data slice is only valid until the next call to Next. func (c *ReusableChunker) Next() (FastCDCChunk, error) { - if err := c.fillBuffer(); err != nil { + err := c.fillBuffer() + if err != nil { return FastCDCChunk{}, err } + if len(c.buf) == 0 { return FastCDCChunk{}, io.EOF } @@ -189,13 +197,6 @@ func (c *ReusableChunker) nextChunk(data []byte) (int, uint64) { return i, fp } -func min(a, b int) int { - if a < b { - return a - } - return b -} - // 256 random uint64s for the rolling hash function (from FastCDC paper) var table = [256]uint64{ 0xe80e8d55032474b3, 0x11b25b61f5924e15, 0x03aa5bd82a9eb669, 0xc45a153ef107a38c, diff --git a/internal/cli/app.go b/internal/cli/app.go index c1da377..36ae64f 100644 --- a/internal/cli/app.go +++ b/internal/cli/app.go @@ -44,9 +44,11 @@ func setupGlobals(lc fx.Lifecycle, g *globals.Globals, v *vaultik.Vaultik, opts lc.Append(fx.Hook{ OnStart: func(ctx context.Context) error { g.StartTime = time.Now().UTC() + if opts.Cron || opts.Quiet { v.UI.SetQuiet(true) } + return nil }, }) @@ -56,12 +58,12 @@ func setupGlobals(lc fx.Lifecycle, g *globals.Globals, v *vaultik.Vaultik, opts // blank line. Used both from the fx hook (for subcommand invocations) and // from the root cobra Run handler (for `vaultik` with no subcommand). func writeStartupBanner(w *ui.Writer, startTime time.Time, shortCommit string) { - w.Banner("%s %s by %s (commit %s, built on %s) starting up at %s.", + w.Bannerf("%s %s by %s (commit %s, built on %s) starting up at %s.", globals.Appname, globals.Version, globals.Author, shortCommit, globals.CommitDate, startTime.Format(time.RFC3339)) - w.Banner("%s", globals.Homepage) - w.Banner("") + w.Bannerf("%s", globals.Homepage) + w.Bannerf("") } // NewApp creates a new fx application with common modules. @@ -105,6 +107,7 @@ func cleanStartupError(err error) error { if idx := strings.LastIndex(msg, "): "); idx >= 0 { msg = msg[idx+3:] } + return errors.New(msg) } @@ -122,7 +125,8 @@ func RunApp(ctx context.Context, app *fx.App) error { defer cancel() // Start the app - if err := app.Start(ctx); err != nil { + err := app.Start(ctx) + if err != nil { return cleanStartupError(err) } @@ -130,6 +134,7 @@ func RunApp(ctx context.Context, app *fx.App) error { shutdownComplete := make(chan struct{}) go func() { defer close(shutdownComplete) + <-sigChan log.Notice("Received interrupt signal, shutting down gracefully...") @@ -137,7 +142,8 @@ func RunApp(ctx context.Context, app *fx.App) error { shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 30*time.Second) defer shutdownCancel() - if err := app.Stop(shutdownCtx); err != nil { + err := app.Stop(shutdownCtx) + if err != nil { log.Error("Error during shutdown", "error", err) } }() @@ -149,9 +155,11 @@ func RunApp(ctx context.Context, app *fx.App) error { return nil case <-ctx.Done(): // Context cancelled (shouldn't happen in normal operation) - if err := app.Stop(context.Background()); err != nil { + err := app.Stop(context.Background()) + if err != nil { log.Error("Error stopping app", "error", err) } + return ctx.Err() case <-app.Done(): // App finished running (e.g., backup completed) @@ -166,19 +174,24 @@ func RunApp(ctx context.Context, app *fx.App) error { func RunWithApp(ctx context.Context, opts AppOptions) error { // Acquire PID lock to prevent concurrent instances lockDir := filepath.Join(xdg.DataHome, "vaultik") + lock, err := pidlock.Acquire(lockDir) if err != nil { if errors.Is(err, pidlock.ErrAlreadyRunning) { return fmt.Errorf("cannot start: %w", err) } + return fmt.Errorf("failed to acquire lock: %w", err) } + defer func() { - if err := lock.Release(); err != nil { + err := lock.Release() + if err != nil { log.Warn("Failed to release PID lock", "error", err) } }() app := NewApp(opts) + return RunApp(ctx, app) } diff --git a/internal/cli/config.go b/internal/cli/config.go index 1dc2c4f..c1bc9c4 100644 --- a/internal/cli/config.go +++ b/internal/cli/config.go @@ -1,6 +1,7 @@ package cli import ( + "errors" "fmt" "os" "os/exec" @@ -240,16 +241,20 @@ on macOS, ~/.config/ on Linux, /etc/vaultik/ as root).`, } dir := filepath.Dir(path) - if err := os.MkdirAll(dir, 0o755); err != nil { + + err := os.MkdirAll(dir, 0o755) + if err != nil { return fmt.Errorf("creating config directory %s: %w", dir, err) } - if err := os.WriteFile(path, []byte(defaultConfigTemplate), 0o600); err != nil { + err = os.WriteFile(path, []byte(defaultConfigTemplate), 0o600) + if err != nil { return fmt.Errorf("writing config file: %w", err) } fmt.Printf("Config written to %s\n", path) fmt.Println("Edit it to set your age_recipients, snapshots, and storage_url.") + return nil }, } @@ -276,6 +281,7 @@ func newConfigEditCommand() *cobra.Command { ed.Stdin = os.Stdin ed.Stdout = os.Stdout ed.Stderr = os.Stderr + return ed.Run() }, } @@ -305,6 +311,7 @@ func newConfigGetCommand() *cobra.Command { if node.Kind == yaml.ScalarNode { fmt.Println(node.Value) + return nil } @@ -312,7 +319,9 @@ func newConfigGetCommand() *cobra.Command { if err != nil { return fmt.Errorf("marshaling value: %w", err) } + fmt.Print(string(out)) + return nil }, } @@ -363,6 +372,7 @@ Examples: } fmt.Printf("%s = %s\n", args[0], args[1]) + return nil }, } @@ -399,8 +409,9 @@ func yamlPathGet(root *yaml.Node, keys []string) (*yaml.Node, error) { node := root if node.Kind == yaml.DocumentNode { if len(node.Content) == 0 { - return nil, fmt.Errorf("empty config file") + return nil, errors.New("empty config file") } + node = node.Content[0] } @@ -408,13 +419,16 @@ func yamlPathGet(root *yaml.Node, keys []string) (*yaml.Node, error) { switch node.Kind { case yaml.MappingNode: found := false + for j := 0; j+1 < len(node.Content); j += 2 { if node.Content[j].Value == key { node = node.Content[j+1] found = true + break } } + if !found { return nil, fmt.Errorf("key not found: %s", strings.Join(keys[:i+1], ".")) } @@ -423,9 +437,11 @@ func yamlPathGet(root *yaml.Node, keys []string) (*yaml.Node, error) { if err != nil { return nil, fmt.Errorf("key %q is a list; use a numeric index", strings.Join(keys[:i], ".")) } + if idx < 0 || idx >= len(node.Content) { return nil, fmt.Errorf("index %d out of range for %s (len %d)", idx, strings.Join(keys[:i], "."), len(node.Content)) } + node = node.Content[idx] default: return nil, fmt.Errorf("key %q is not a map or list", strings.Join(keys[:i], ".")) @@ -445,6 +461,7 @@ func yamlPathSet(root *yaml.Node, keys []string, value string) error { if len(node.Content) == 0 { node.Content = []*yaml.Node{{Kind: yaml.MappingNode}} } + node = node.Content[0] } @@ -454,19 +471,23 @@ func yamlPathSet(root *yaml.Node, keys []string, value string) error { switch node.Kind { case yaml.MappingNode: var valueNode *yaml.Node + for j := 0; j+1 < len(node.Content); j += 2 { if node.Content[j].Value == key { valueNode = node.Content[j+1] + break } } if valueNode == nil { keyNode := &yaml.Node{Kind: yaml.ScalarNode, Value: key} + valueNode = &yaml.Node{Kind: yaml.MappingNode} if last { valueNode = &yaml.Node{Kind: yaml.ScalarNode, Value: value} } + node.Content = append(node.Content, keyNode, valueNode) } else if last { setScalar(valueNode, value) @@ -479,18 +500,22 @@ func yamlPathSet(root *yaml.Node, keys []string, value string) error { if err != nil { return fmt.Errorf("key %q is a list; use a numeric index", strings.Join(keys[:i], ".")) } + if idx < 0 || idx > len(node.Content) { return fmt.Errorf("index %d out of range for %s (len %d)", idx, strings.Join(keys[:i], "."), len(node.Content)) } + if idx == len(node.Content) { newNode := &yaml.Node{Kind: yaml.MappingNode} if last { newNode = &yaml.Node{Kind: yaml.ScalarNode, Value: value} } + node.Content = append(node.Content, newNode) } else if last { setScalar(node.Content[idx], value) } + node = node.Content[idx] default: @@ -516,8 +541,10 @@ func configPathForInit() string { if rootFlags.ConfigPath != "" { return rootFlags.ConfigPath } + if envPath := os.Getenv("VAULTIK_CONFIG"); envPath != "" { return envPath } + return DefaultConfigPath() } diff --git a/internal/cli/config_test.go b/internal/cli/config_test.go index cf45f04..24ad8ab 100644 --- a/internal/cli/config_test.go +++ b/internal/cli/config_test.go @@ -12,7 +12,9 @@ import ( // that unmarshals into the Config struct with the expected snapshots. func TestDefaultConfigTemplateParses(t *testing.T) { var cfg config.Config - if err := yaml.Unmarshal([]byte(defaultConfigTemplate), &cfg); err != nil { + + err := yaml.Unmarshal([]byte(defaultConfigTemplate), &cfg) + if err != nil { t.Fatalf("default config template is not valid YAML: %v", err) } @@ -24,9 +26,11 @@ func TestDefaultConfigTemplateParses(t *testing.T) { if !ok { t.Fatal("expected 'home' snapshot in default config") } + if len(home.Paths) == 0 { t.Error("home snapshot should have at least one path") } + if len(home.Exclude) == 0 { t.Error("home snapshot should have exclude patterns") } @@ -35,9 +39,11 @@ func TestDefaultConfigTemplateParses(t *testing.T) { if !ok { t.Fatal("expected 'apps' snapshot in default config") } + if len(apps.Paths) != 1 || apps.Paths[0] != "/Applications" { t.Errorf("apps snapshot should back up /Applications, got %v", apps.Paths) } + if len(apps.Exclude) == 0 { t.Error("apps snapshot should have exclude patterns") } @@ -58,10 +64,14 @@ snapshots: func parseTestYAML(t *testing.T) *yaml.Node { t.Helper() + var root yaml.Node - if err := yaml.Unmarshal([]byte(testYAML), &root); err != nil { + + err := yaml.Unmarshal([]byte(testYAML), &root) + if err != nil { t.Fatalf("parsing test yaml: %v", err) } + return &root } @@ -91,11 +101,14 @@ func TestYAMLPathGet(t *testing.T) { if err == nil { t.Fatalf("expected error for %q", tt.path) } + return } + if err != nil { t.Fatalf("unexpected error: %v", err) } + if node.Value != tt.want { t.Errorf("get %q = %q, want %q", tt.path, node.Value, tt.want) } @@ -115,6 +128,7 @@ func TestYAMLPathSet(t *testing.T) { if err := yamlPathSet(root, splitPath("s3.endpoint"), "s3.example.com"); err != nil { t.Fatalf("set s3.endpoint: %v", err) } + if err := yamlPathSet(root, splitPath("newmap.newkey"), "val"); err != nil { t.Fatalf("set newmap.newkey: %v", err) } @@ -123,9 +137,11 @@ func TestYAMLPathSet(t *testing.T) { if err := yamlPathSet(root, splitPath("age_recipients.0"), "age1bbb"); err != nil { t.Fatalf("set age_recipients.0: %v", err) } + if err := yamlPathSet(root, splitPath("age_recipients.1"), "age1ccc"); err != nil { t.Fatalf("append age_recipients.1: %v", err) } + if err := yamlPathSet(root, splitPath("age_recipients.5"), "age1ddd"); err == nil { t.Error("expected out-of-range append to fail") } @@ -135,6 +151,7 @@ func TestYAMLPathSet(t *testing.T) { if err != nil { t.Fatalf("marshal: %v", err) } + text := string(out) for _, want := range []string{"newbucket", "s3.example.com", "newkey: val", "# top comment", "# inline comment", "age1bbb", "age1ccc"} { @@ -147,6 +164,7 @@ func TestYAMLPathSet(t *testing.T) { if err != nil { t.Fatalf("get after set: %v", err) } + if got.Value != "newbucket" { t.Errorf("s3.bucket = %q after set, want newbucket", got.Value) } diff --git a/internal/cli/database.go b/internal/cli/database.go index cf83da0..a6532fb 100644 --- a/internal/cli/database.go +++ b/internal/cli/database.go @@ -66,6 +66,7 @@ Use --force to skip the confirmation prompt.`, // Check if database exists if _, err := os.Stat(dbPath); os.IsNotExist(err) { fmt.Printf("Database does not exist: %s\n", dbPath) + return nil } @@ -73,9 +74,11 @@ Use --force to skip the confirmation prompt.`, if !force { fmt.Printf("This will delete the local state database at:\n %s\n\n", dbPath) fmt.Print("Are you sure? Type 'yes' to confirm: ") + var confirm string if _, err := fmt.Scanln(&confirm); err != nil || confirm != "yes" { fmt.Println("Aborted.") + return nil } } @@ -97,6 +100,7 @@ Use --force to skip the confirmation prompt.`, } log.Info("Local state database deleted", "path", dbPath) + return nil }, } diff --git a/internal/cli/duration.go b/internal/cli/duration.go index d9a92ab..8fd2350 100644 --- a/internal/cli/duration.go +++ b/internal/cli/duration.go @@ -1,6 +1,7 @@ package cli import ( + "errors" "fmt" "regexp" "strconv" @@ -25,7 +26,7 @@ func parseDuration(s string) (time.Duration, error) { // Extended duration parsing // Check for negative values if strings.HasPrefix(strings.TrimSpace(s), "-") { - return 0, fmt.Errorf("negative durations are not supported") + return 0, errors.New("negative durations are not supported") } // Pattern matches: number + unit, repeated @@ -48,6 +49,7 @@ func parseDuration(s string) (time.Duration, error) { } var d time.Duration + switch unit { // Standard time units case "ns", "nanosecond", "nanoseconds": @@ -75,7 +77,7 @@ func parseDuration(s string) (time.Duration, error) { d = time.Duration(value * float64(365*24*time.Hour)) default: // Try parsing as standard Go duration unit - testStr := fmt.Sprintf("1%s", unit) + testStr := "1" + unit if _, err := time.ParseDuration(testStr); err == nil { // It's a valid Go duration unit, parse the full value fullStr := fmt.Sprintf("%g%s", value, unit) diff --git a/internal/cli/duration_test.go b/internal/cli/duration_test.go index 88cd67e..3edb414 100644 --- a/internal/cli/duration_test.go +++ b/internal/cli/duration_test.go @@ -185,6 +185,7 @@ func TestParseDuration(t *testing.T) { if tt.wantErr { assert.Error(t, err, "expected error for input %q", tt.input) + return } diff --git a/internal/cli/entry.go b/internal/cli/entry.go index 4b9fed8..9709f08 100644 --- a/internal/cli/entry.go +++ b/internal/cli/entry.go @@ -19,24 +19,26 @@ func CLIEntry() { if len(short) > 12 { short = short[:12] } + writeStartupBanner(ui.New(os.Stdout), time.Now().UTC(), short) } rootCmd := NewRootCommand() rootCmd.SilenceErrors = true - if err := rootCmd.Execute(); err != nil { - ReportError("%s", err.Error()) + err := rootCmd.Execute() + if err != nil { + ReportErrorf("%s", err.Error()) os.Exit(1) } } -// ReportError emits a user-facing error to stderr in the standard +// ReportErrorf emits a user-facing error to stderr in the standard // 🛑 ERROR: format. Use it from goroutine error paths (where returning // an error to cobra isn't an option) and anywhere else a CLI command // must surface a failure outside the normal RunE return path. -func ReportError(format string, args ...any) { - ui.New(os.Stderr).Error(format, args...) +func ReportErrorf(format string, args ...any) { + ui.New(os.Stderr).Errorf(format, args...) } // bannerSuppressedInArgs reports whether any of args is a flag that @@ -48,10 +50,12 @@ func bannerSuppressedInArgs(args []string) bool { if a == "--" { return false } + switch a { case "--quiet", "-q", "--cron": return true } + if strings.HasPrefix(a, "--quiet=") || strings.HasPrefix(a, "--cron=") { return true } @@ -64,5 +68,6 @@ func bannerSuppressedInArgs(args []string) bool { } } } + return false } diff --git a/internal/cli/entry_test.go b/internal/cli/entry_test.go index cc7e623..f652399 100644 --- a/internal/cli/entry_test.go +++ b/internal/cli/entry_test.go @@ -21,12 +21,15 @@ func TestCLIEntry(t *testing.T) { expectedCommands := []string{"config", "snapshot", "prune", "info", "version", "remote", "database"} for _, expected := range expectedCommands { found := false + for _, cmd := range cmd.Commands() { if cmd.Use == expected || cmd.Name() == expected { found = true + break } } + if !found { t.Errorf("Expected command '%s' not found", expected) } @@ -41,12 +44,15 @@ func TestCLIEntry(t *testing.T) { expectedSubCommands := []string{"create", "list", "purge", "verify", "remove", "restore"} for _, expected := range expectedSubCommands { found := false + for _, subcmd := range snapshotCmd.Commands() { if subcmd.Use == expected || subcmd.Name() == expected { found = true + break } } + if !found { t.Errorf("Expected snapshot subcommand '%s' not found", expected) } diff --git a/internal/cli/info.go b/internal/cli/info.go index e4aca7e..8cc715d 100644 --- a/internal/cli/info.go +++ b/internal/cli/info.go @@ -2,6 +2,7 @@ package cli import ( "context" + "errors" "os" "github.com/spf13/cobra" @@ -31,6 +32,7 @@ func NewInfoCommand() *cobra.Command { // Use the app framework rootFlags := GetRootFlags() + return RunWithApp(cmd.Context(), AppOptions{ ConfigPath: configPath, LogOptions: log.LogOptions{ @@ -44,21 +46,26 @@ func NewInfoCommand() *cobra.Command { lc.Append(fx.Hook{ OnStart: func(ctx context.Context) error { go func() { - if err := v.ShowInfo(); err != nil { - if err != context.Canceled { + err := v.ShowInfo() + if err != nil { + if !errors.Is(err, context.Canceled) { log.Error("Failed to show info", "error", err) - ReportError("Failed to show info: %v", err) + ReportErrorf("Failed to show info: %v", err) os.Exit(1) } } - if err := v.Shutdowner.Shutdown(); err != nil { + + err = v.Shutdowner.Shutdown() + if err != nil { log.Error("Failed to shutdown", "error", err) } }() + return nil }, OnStop: func(ctx context.Context) error { v.Cancel() + return nil }, }) diff --git a/internal/cli/prune.go b/internal/cli/prune.go index b367e9c..b5cc67e 100644 --- a/internal/cli/prune.go +++ b/internal/cli/prune.go @@ -2,6 +2,7 @@ package cli import ( "context" + "errors" "os" "github.com/spf13/cobra" @@ -39,6 +40,7 @@ work (e.g. after a crashed backup or to reclaim storage).`, // Use the app framework like other commands rootFlags := GetRootFlags() + return RunWithApp(cmd.Context(), AppOptions{ ConfigPath: configPath, LogOptions: log.LogOptions{ @@ -54,26 +56,31 @@ work (e.g. after a crashed backup or to reclaim storage).`, // Start the prune operation in a goroutine go func() { // Run the prune operation - if err := v.Prune(opts); err != nil { - if err != context.Canceled { + err := v.Prune(opts) + if err != nil { + if !errors.Is(err, context.Canceled) { if !opts.JSON { log.Error("Prune operation failed", "error", err) - ReportError("Prune failed: %v", err) + ReportErrorf("Prune failed: %v", err) } + os.Exit(1) } } // Shutdown the app when prune completes - if err := v.Shutdowner.Shutdown(); err != nil { + err = v.Shutdowner.Shutdown() + if err != nil { log.Error("Failed to shutdown", "error", err) } }() + return nil }, OnStop: func(ctx context.Context) error { log.Debug("Stopping prune operation") v.Cancel() + return nil }, }) diff --git a/internal/cli/remote.go b/internal/cli/remote.go index cde873e..7acb001 100644 --- a/internal/cli/remote.go +++ b/internal/cli/remote.go @@ -2,7 +2,7 @@ package cli import ( "context" - "fmt" + "errors" "os" "github.com/spf13/cobra" @@ -41,7 +41,7 @@ This is destructive and irreversible. Requires --force.`, Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { if !force { - return fmt.Errorf("remote nuke requires --force (this deletes ALL remote snapshots and blobs)") + return errors.New("remote nuke requires --force (this deletes ALL remote snapshots and blobs)") } configPath, err := ResolveConfigPath() @@ -50,6 +50,7 @@ This is destructive and irreversible. Requires --force.`, } rootFlags := GetRootFlags() + return RunWithApp(cmd.Context(), AppOptions{ ConfigPath: configPath, LogOptions: log.LogOptions{ @@ -63,21 +64,26 @@ This is destructive and irreversible. Requires --force.`, lc.Append(fx.Hook{ OnStart: func(ctx context.Context) error { go func() { - if err := v.NukeRemote(true); err != nil { - if err != context.Canceled { + err := v.NukeRemote(true) + if err != nil { + if !errors.Is(err, context.Canceled) { log.Error("Remote nuke failed", "error", err) - ReportError("Remote nuke failed: %v", err) + ReportErrorf("Remote nuke failed: %v", err) os.Exit(1) } } - if err := v.Shutdowner.Shutdown(); err != nil { + + err = v.Shutdowner.Shutdown() + if err != nil { log.Error("Failed to shutdown", "error", err) } }() + return nil }, OnStop: func(ctx context.Context) error { v.Cancel() + return nil }, }) @@ -113,6 +119,7 @@ func newRemoteInfoCommand() *cobra.Command { } rootFlags := GetRootFlags() + return RunWithApp(cmd.Context(), AppOptions{ ConfigPath: configPath, LogOptions: log.LogOptions{ @@ -126,23 +133,29 @@ func newRemoteInfoCommand() *cobra.Command { lc.Append(fx.Hook{ OnStart: func(ctx context.Context) error { go func() { - if err := v.RemoteInfo(jsonOutput); err != nil { - if err != context.Canceled { + err := v.RemoteInfo(jsonOutput) + if err != nil { + if !errors.Is(err, context.Canceled) { if !jsonOutput { log.Error("Failed to get remote info", "error", err) - ReportError("Failed to get remote info: %v", err) + ReportErrorf("Failed to get remote info: %v", err) } + os.Exit(1) } } - if err := v.Shutdowner.Shutdown(); err != nil { + + err = v.Shutdowner.Shutdown() + if err != nil { log.Error("Failed to shutdown", "error", err) } }() + return nil }, OnStop: func(ctx context.Context) error { v.Cancel() + return nil }, }) diff --git a/internal/cli/root.go b/internal/cli/root.go index 5c29b14..2e53dad 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -78,6 +78,7 @@ func ResolveConfigPath() (string, error) { if _, err := os.Stat(path); err != nil { return "", fmt.Errorf("config file from --config not found: %s (run 'vaultik config init --config %s' to create it)", path, path) } + return path, nil } @@ -85,6 +86,7 @@ func ResolveConfigPath() (string, error) { if _, err := os.Stat(path); err != nil { return "", fmt.Errorf("config file from $VAULTIK_CONFIG not found: %s (unset VAULTIK_CONFIG, point it at an existing file, or run 'vaultik config init')", path) } + return path, nil } @@ -114,5 +116,6 @@ func DefaultConfigPath() string { if os.Getuid() == 0 { return "/etc/vaultik/config.yml" } + return filepath.Join(xdg.ConfigHome, "vaultik", "config.yml") } diff --git a/internal/cli/snapshot.go b/internal/cli/snapshot.go index ef01eaa..5f5e088 100644 --- a/internal/cli/snapshot.go +++ b/internal/cli/snapshot.go @@ -2,6 +2,7 @@ package cli import ( "context" + "errors" "fmt" "os" @@ -58,6 +59,7 @@ specifying a path using --config or by setting VAULTIK_CONFIG to a path.`, // Use the backup functionality from cli package rootFlags := GetRootFlags() + return RunWithApp(cmd.Context(), AppOptions{ ConfigPath: configPath, LogOptions: log.LogOptions{ @@ -74,25 +76,29 @@ specifying a path using --config or by setting VAULTIK_CONFIG to a path.`, // Start the snapshot creation in a goroutine go func() { // --cron suppression is wired through v.UI by setupGlobals. - if err := v.CreateSnapshot(opts); err != nil { - if err != context.Canceled { + err := v.CreateSnapshot(opts) + if err != nil { + if !errors.Is(err, context.Canceled) { log.Error("Snapshot creation failed", "error", err) - ReportError("Snapshot creation failed: %v", err) + ReportErrorf("Snapshot creation failed: %v", err) os.Exit(1) } } // Shutdown the app when snapshot completes - if err := v.Shutdowner.Shutdown(); err != nil { + err = v.Shutdowner.Shutdown() + if err != nil { log.Error("Failed to shutdown", "error", err) } }() + return nil }, OnStop: func(ctx context.Context) error { log.Debug("Stopping snapshot creation") // Cancel the Vaultik context v.Cancel() + return nil }, }) @@ -127,6 +133,7 @@ func newSnapshotListCommand() *cobra.Command { } rootFlags := GetRootFlags() + return RunWithApp(cmd.Context(), AppOptions{ ConfigPath: configPath, LogOptions: log.LogOptions{ @@ -140,21 +147,26 @@ func newSnapshotListCommand() *cobra.Command { lc.Append(fx.Hook{ OnStart: func(ctx context.Context) error { go func() { - if err := v.ListSnapshots(jsonOutput); err != nil { - if err != context.Canceled { + err := v.ListSnapshots(jsonOutput) + if err != nil { + if !errors.Is(err, context.Canceled) { log.Error("Failed to list snapshots", "error", err) - ReportError("Failed to list snapshots: %v", err) + ReportErrorf("Failed to list snapshots: %v", err) os.Exit(1) } } - if err := v.Shutdowner.Shutdown(); err != nil { + + err = v.Shutdowner.Shutdown() + if err != nil { log.Error("Failed to shutdown", "error", err) } }() + return nil }, OnStop: func(ctx context.Context) error { v.Cancel() + return nil }, }) @@ -185,10 +197,11 @@ restrict the operation to specific snapshot names.`, RunE: func(cmd *cobra.Command, args []string) error { // Validate flags if !opts.KeepLatest && opts.OlderThan == "" { - return fmt.Errorf("must specify either --keep-latest or --older-than") + return errors.New("must specify either --keep-latest or --older-than") } + if opts.KeepLatest && opts.OlderThan != "" { - return fmt.Errorf("cannot specify both --keep-latest and --older-than") + return errors.New("cannot specify both --keep-latest and --older-than") } // Use unified config resolution @@ -198,6 +211,7 @@ restrict the operation to specific snapshot names.`, } rootFlags := GetRootFlags() + return RunWithApp(cmd.Context(), AppOptions{ ConfigPath: configPath, LogOptions: log.LogOptions{ @@ -211,21 +225,26 @@ restrict the operation to specific snapshot names.`, lc.Append(fx.Hook{ OnStart: func(ctx context.Context) error { go func() { - if err := v.PurgeSnapshotsWithOptions(opts); err != nil { - if err != context.Canceled { + err := v.PurgeSnapshotsWithOptions(opts) + if err != nil { + if !errors.Is(err, context.Canceled) { log.Error("Failed to purge snapshots", "error", err) - ReportError("Failed to purge snapshots: %v", err) + ReportErrorf("Failed to purge snapshots: %v", err) os.Exit(1) } } - if err := v.Shutdowner.Shutdown(); err != nil { + + err = v.Shutdowner.Shutdown() + if err != nil { log.Error("Failed to shutdown", "error", err) } }() + return nil }, OnStop: func(ctx context.Context) error { v.Cancel() + return nil }, }) @@ -254,11 +273,14 @@ func newSnapshotVerifyCommand() *cobra.Command { Args: func(cmd *cobra.Command, args []string) error { if len(args) != 1 { _ = cmd.Help() + if len(args) == 0 { - return fmt.Errorf("snapshot ID required") + return errors.New("snapshot ID required") } + return fmt.Errorf("expected 1 argument, got %d", len(args)) } + return nil }, RunE: func(cmd *cobra.Command, args []string) error { @@ -271,6 +293,7 @@ func newSnapshotVerifyCommand() *cobra.Command { } rootFlags := GetRootFlags() + return RunWithApp(cmd.Context(), AppOptions{ ConfigPath: configPath, LogOptions: log.LogOptions{ @@ -284,23 +307,29 @@ func newSnapshotVerifyCommand() *cobra.Command { lc.Append(fx.Hook{ OnStart: func(ctx context.Context) error { go func() { - if err := v.VerifySnapshotWithOptions(snapshotID, opts); err != nil { - if err != context.Canceled { + err := v.VerifySnapshotWithOptions(snapshotID, opts) + if err != nil { + if !errors.Is(err, context.Canceled) { if !opts.JSON { log.Error("Verification failed", "error", err) - ReportError("Verification failed: %v", err) + ReportErrorf("Verification failed: %v", err) } + os.Exit(1) } } - if err := v.Shutdowner.Shutdown(); err != nil { + + err = v.Shutdowner.Shutdown() + if err != nil { log.Error("Failed to shutdown", "error", err) } }() + return nil }, OnStop: func(ctx context.Context) error { v.Cancel() + return nil }, }) @@ -345,11 +374,14 @@ nuke --force' — it is the single supported entry point for that.`, Args: func(cmd *cobra.Command, args []string) error { if len(args) != 1 { _ = cmd.Help() + if len(args) == 0 { - return fmt.Errorf("snapshot ID required") + return errors.New("snapshot ID required") } + return fmt.Errorf("expected 1 argument, got %d", len(args)) } + return nil }, RunE: func(cmd *cobra.Command, args []string) error { @@ -360,6 +392,7 @@ nuke --force' — it is the single supported entry point for that.`, } rootFlags := GetRootFlags() + return RunWithApp(cmd.Context(), AppOptions{ ConfigPath: configPath, LogOptions: log.LogOptions{ @@ -375,22 +408,26 @@ nuke --force' — it is the single supported entry point for that.`, go func() { _, err := v.RemoveSnapshot(args[0], opts) if err != nil { - if err != context.Canceled { + if !errors.Is(err, context.Canceled) { if !opts.JSON { log.Error("Failed to remove snapshot", "error", err) - ReportError("Failed to remove snapshot: %v", err) + ReportErrorf("Failed to remove snapshot: %v", err) } + os.Exit(1) } } + if err := v.Shutdowner.Shutdown(); err != nil { log.Error("Failed to shutdown", "error", err) } }() + return nil }, OnStop: func(ctx context.Context) error { v.Cancel() + return nil }, }) diff --git a/internal/cli/snapshot_restore.go b/internal/cli/snapshot_restore.go index 9f60b4a..4f5d878 100644 --- a/internal/cli/snapshot_restore.go +++ b/internal/cli/snapshot_restore.go @@ -2,6 +2,7 @@ package cli import ( "context" + "errors" "os" "github.com/spf13/cobra" @@ -70,6 +71,7 @@ Examples: // runRestore parses arguments and runs the restore operation through the app framework func runRestore(cmd *cobra.Command, args []string, opts *RestoreOptions) error { snapshotID := args[0] + opts.TargetDir = args[1] if len(args) > 2 { opts.Paths = args[2:] @@ -83,6 +85,7 @@ func runRestore(cmd *cobra.Command, args []string, opts *RestoreOptions) error { // Use the app framework like other commands rootFlags := GetRootFlags() + return RunWithApp(cmd.Context(), AppOptions{ ConfigPath: configPath, LogOptions: log.LogOptions{ @@ -129,24 +132,29 @@ func buildRestoreInvokes(snapshotID string, opts *RestoreOptions) []fx.Option { Verify: opts.Verify, SkipErrors: GetRootFlags().SkipErrors, } - if err := app.Vaultik.Restore(restoreOpts); err != nil { - if err != context.Canceled { + + err := app.Vaultik.Restore(restoreOpts) + if err != nil { + if !errors.Is(err, context.Canceled) { log.Error("Restore operation failed", "error", err) - ReportError("Restore failed: %v", err) + ReportErrorf("Restore failed: %v", err) os.Exit(1) } } // Shutdown the app when restore completes - if err := app.Shutdowner.Shutdown(); err != nil { + err = app.Shutdowner.Shutdown() + if err != nil { log.Error("Failed to shutdown", "error", err) } }() + return nil }, OnStop: func(ctx context.Context) error { log.Debug("Stopping restore operation") app.Vaultik.Cancel() + return nil }, }) diff --git a/internal/cli/version.go b/internal/cli/version.go index cbe3344..7c6d14c 100644 --- a/internal/cli/version.go +++ b/internal/cli/version.go @@ -24,6 +24,7 @@ func NewVersionCommand() *cobra.Command { fmt.Printf(" author: %s\n", globals.Author) fmt.Printf(" homepage: %s\n", globals.Homepage) fmt.Printf(" license: %s\n", globals.License) + if globals.Version == "dev" { fmt.Println() fmt.Println("This is a development build (no version information embedded).") diff --git a/internal/config/config.go b/internal/config/config.go index a47f347..cabf735 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1,6 +1,7 @@ package config import ( + "errors" "fmt" "os" "path/filepath" @@ -21,12 +22,16 @@ const appName = "vaultik" func expandTilde(path string) string { if path == "~" { home, _ := os.UserHomeDir() + return home } + if strings.HasPrefix(path, "~/") { home, _ := os.UserHomeDir() + return filepath.Join(home, path[2:]) } + return path } @@ -34,8 +39,10 @@ func expandTilde(path string) string { func expandTildeInURL(url string) string { if strings.HasPrefix(url, "file://~/") { home, _ := os.UserHomeDir() + return "file://" + filepath.Join(home, url[9:]) } + return url } @@ -63,6 +70,7 @@ func (c *Config) GetExcludes(snapshotName string) []string { combined := make([]string, 0, len(c.Exclude)+len(snap.Exclude)) combined = append(combined, c.Exclude...) combined = append(combined, snap.Exclude...) + return combined } @@ -74,6 +82,7 @@ func (c *Config) SnapshotNames() []string { } // Sort for deterministic order sort.Strings(names) + return names } @@ -126,7 +135,7 @@ type ConfigPath string // Returns an error if the path is empty or if loading fails. func New(path ConfigPath) (*Config, error) { if path == "" { - return nil, fmt.Errorf("config path not provided") + return nil, errors.New("config path not provided") } cfg, err := Load(string(path)) @@ -159,6 +168,7 @@ func Load(path string) (*Config, error) { // Convert smartconfig data to YAML then unmarshal configData := sc.Data() + yamlBytes, err := yaml.Marshal(configData) if err != nil { return nil, fmt.Errorf("failed to marshal config data: %w", err) @@ -177,6 +187,7 @@ func Load(path string) (*Config, error) { for i, path := range snap.Paths { snap.Paths[i] = expandTilde(path) } + cfg.Snapshots[name] = snap } @@ -196,6 +207,7 @@ func Load(path string) (*Config, error) { if err != nil { return nil, fmt.Errorf("failed to get hostname: %w", err) } + cfg.Hostname = hostname } @@ -203,6 +215,7 @@ func Load(path string) (*Config, error) { if cfg.S3.Region == "" { cfg.S3.Region = "us-east-1" } + if cfg.S3.PartSize == 0 { cfg.S3.PartSize = Size(5 * 1024 * 1024) // 5MB } @@ -236,11 +249,11 @@ func Load(path string) (*Config, error) { // Returns an error describing the first validation failure encountered. func (c *Config) Validate() error { if len(c.AgeRecipients) == 0 { - return fmt.Errorf("at least one age_recipient is required (generate with: age-keygen)") + return errors.New("at least one age_recipient is required (generate with: age-keygen)") } if len(c.Snapshots) == 0 { - return fmt.Errorf("at least one snapshot must be configured (see config.example.yml)") + return errors.New("at least one snapshot must be configured (see config.example.yml)") } for name, snap := range c.Snapshots { @@ -250,20 +263,21 @@ func (c *Config) Validate() error { } // Validate storage configuration - if err := c.validateStorage(); err != nil { + err := c.validateStorage() + if err != nil { return err } if c.ChunkSize.Int64() < 1024*1024 { // 1MB minimum - return fmt.Errorf("chunk_size must be at least 1MB") + return errors.New("chunk_size must be at least 1MB") } if c.BlobSizeLimit.Int64() < c.ChunkSize.Int64() { - return fmt.Errorf("blob_size_limit must be at least chunk_size") + return errors.New("blob_size_limit must be at least chunk_size") } if c.CompressionLevel < 1 || c.CompressionLevel > 19 { - return fmt.Errorf("compression_level must be between 1 and 19") + return errors.New("compression_level must be between 1 and 19") } return nil @@ -280,38 +294,43 @@ func (c *Config) validateStorage() error { // File storage doesn't need S3 credentials return nil } + if strings.HasPrefix(c.StorageURL, "s3://") { // S3 storage needs credentials if c.S3.AccessKeyID == "" { - return fmt.Errorf("s3.access_key_id is required for s3:// URLs") + return errors.New("s3.access_key_id is required for s3:// URLs") } + if c.S3.SecretAccessKey == "" { - return fmt.Errorf("s3.secret_access_key is required for s3:// URLs") + return errors.New("s3.secret_access_key is required for s3:// URLs") } + return nil } + if strings.HasPrefix(c.StorageURL, "rclone://") { // Rclone storage uses rclone's own config return nil } - return fmt.Errorf("storage_url must start with s3://, file://, or rclone://") + + return errors.New("storage_url must start with s3://, file://, or rclone://") } // Legacy S3 configuration if c.S3.Endpoint == "" { - return fmt.Errorf("storage not configured; set storage_url or provide s3.endpoint + s3.bucket + credentials") + return errors.New("storage not configured; set storage_url or provide s3.endpoint + s3.bucket + credentials") } if c.S3.Bucket == "" { - return fmt.Errorf("s3.bucket is required (or set storage_url)") + return errors.New("s3.bucket is required (or set storage_url)") } if c.S3.AccessKeyID == "" { - return fmt.Errorf("s3.access_key_id is required") + return errors.New("s3.access_key_id is required") } if c.S3.SecretAccessKey == "" { - return fmt.Errorf("s3.secret_access_key is required") + return errors.New("s3.secret_access_key is required") } return nil @@ -329,6 +348,7 @@ func extractAgeSecretKey(input string) string { if id, ok := identities[0].(*age.X25519Identity); ok { return id.String() } + return strings.TrimSpace(input) } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 4304af6..08d3bce 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -41,6 +41,7 @@ func TestConfigLoad(t *testing.T) { if len(cfg.AgeRecipients) != 2 { t.Errorf("Expected 2 age recipients, got %d", len(cfg.AgeRecipients)) } + if cfg.AgeRecipients[0] != TEST_SNEAK_AGE_PUBLIC_KEY { t.Errorf("Expected first age recipient to be %s, got '%s'", TEST_SNEAK_AGE_PUBLIC_KEY, cfg.AgeRecipients[0]) } diff --git a/internal/config/size.go b/internal/config/size.go index d66049d..58dbf73 100644 --- a/internal/config/size.go +++ b/internal/config/size.go @@ -1,6 +1,7 @@ package config import ( + "errors" "fmt" "github.com/dustin/go-humanize" @@ -14,18 +15,19 @@ type Size int64 // UnmarshalYAML implements yaml.Unmarshaler for Size, allowing it to be // parsed from YAML configuration files. It accepts both numeric values // (interpreted as bytes) and string values with units (e.g., "10MB"). -func (s *Size) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (s *Size) UnmarshalYAML(unmarshal func(any) error) error { // Try to unmarshal as int64 first var intVal int64 if err := unmarshal(&intVal); err == nil { *s = Size(intVal) + return nil } // Try to unmarshal as string var strVal string if err := unmarshal(&strVal); err != nil { - return fmt.Errorf("size must be a number or string") + return errors.New("size must be a number or string") } // Parse the string using go-humanize @@ -35,6 +37,7 @@ func (s *Size) UnmarshalYAML(unmarshal func(interface{}) error) error { } *s = Size(bytes) + return nil } @@ -58,5 +61,6 @@ func ParseSize(s string) (Size, error) { if err != nil { return 0, fmt.Errorf("invalid size format: %w", err) } + return Size(bytes), nil } diff --git a/internal/crypto/encryption.go b/internal/crypto/encryption.go index 1d7b3cb..d3380a7 100644 --- a/internal/crypto/encryption.go +++ b/internal/crypto/encryption.go @@ -2,6 +2,7 @@ package crypto import ( "bytes" + "errors" "fmt" "io" "sync" @@ -25,7 +26,7 @@ type Encryptor struct { // public keys are invalid or if no recipients are specified. func NewEncryptor(publicKeys []string) (*Encryptor, error) { if len(publicKeys) == 0 { - return nil, fmt.Errorf("at least one recipient is required") + return nil, errors.New("at least one recipient is required") } recipients := make([]age.Recipient, 0, len(publicKeys)) @@ -34,6 +35,7 @@ func NewEncryptor(publicKeys []string) (*Encryptor, error) { if err != nil { return nil, fmt.Errorf("parsing age recipient %s: %w", key, err) } + recipients = append(recipients, recipient) } @@ -126,7 +128,7 @@ func (e *Encryptor) EncryptWriter(dst io.Writer) (io.WriteCloser, error) { // of the public keys are invalid or if no recipients are specified. func (e *Encryptor) UpdateRecipients(publicKeys []string) error { if len(publicKeys) == 0 { - return fmt.Errorf("at least one recipient is required") + return errors.New("at least one recipient is required") } recipients := make([]age.Recipient, 0, len(publicKeys)) @@ -135,6 +137,7 @@ func (e *Encryptor) UpdateRecipients(publicKeys []string) error { if err != nil { return fmt.Errorf("parsing age recipient %s: %w", key, err) } + recipients = append(recipients, recipient) } diff --git a/internal/crypto/encryption_test.go b/internal/crypto/encryption_test.go index ddd92ca..584b216 100644 --- a/internal/crypto/encryption_test.go +++ b/internal/crypto/encryption_test.go @@ -58,10 +58,12 @@ func TestEncryptorMultipleRecipients(t *testing.T) { if err != nil { t.Fatalf("failed to generate identity1: %v", err) } + identity2, err := age.GenerateX25519Identity() if err != nil { t.Fatalf("failed to generate identity2: %v", err) } + identity3, err := age.GenerateX25519Identity() if err != nil { t.Fatalf("failed to generate identity3: %v", err) @@ -123,6 +125,7 @@ func TestEncryptorUpdateRecipients(t *testing.T) { // Encrypt with first key plaintext := []byte("test data") + ciphertext1, err := enc.Encrypt(plaintext) if err != nil { t.Fatalf("failed to encrypt: %v", err) @@ -143,6 +146,7 @@ func TestEncryptorUpdateRecipients(t *testing.T) { if _, err := age.Decrypt(bytes.NewReader(ciphertext1), identity1); err != nil { t.Error("failed to decrypt with identity1") } + if _, err := age.Decrypt(bytes.NewReader(ciphertext1), identity2); err == nil { t.Error("should not decrypt with identity2") } @@ -151,6 +155,7 @@ func TestEncryptorUpdateRecipients(t *testing.T) { if _, err := age.Decrypt(bytes.NewReader(ciphertext2), identity2); err != nil { t.Error("failed to decrypt with identity2") } + if _, err := age.Decrypt(bytes.NewReader(ciphertext2), identity1); err == nil { t.Error("should not decrypt with identity1") } diff --git a/internal/database/blob_chunks.go b/internal/database/blob_chunks.go index 13e6b27..6ff9522 100644 --- a/internal/database/blob_chunks.go +++ b/internal/database/blob_chunks.go @@ -49,12 +49,15 @@ func (r *BlobChunkRepository) GetByBlobID(ctx context.Context, blobID string) ([ defer CloseRows(rows) var blobChunks []*BlobChunk + for rows.Next() { var bc BlobChunk + err := rows.Scan(&bc.BlobID, &bc.ChunkHash, &bc.Offset, &bc.Length) if err != nil { return nil, fmt.Errorf("scanning blob chunk: %w", err) } + blobChunks = append(blobChunks, &bc) } @@ -70,7 +73,9 @@ func (r *BlobChunkRepository) GetByChunkHash(ctx context.Context, chunkHash stri ` LogSQL("GetByChunkHash", query, chunkHash) + var bc BlobChunk + err := r.db.conn.QueryRowContext(ctx, query, chunkHash).Scan( &bc.BlobID, &bc.ChunkHash, @@ -80,14 +85,18 @@ func (r *BlobChunkRepository) GetByChunkHash(ctx context.Context, chunkHash stri if err == sql.ErrNoRows { LogSQL("GetByChunkHash", "No rows found", chunkHash) + return nil, nil } + if err != nil { LogSQL("GetByChunkHash", "Error", chunkHash, err) + return nil, fmt.Errorf("querying blob chunk: %w", err) } LogSQL("GetByChunkHash", "Found blob", chunkHash, "blob", bc.BlobID) + return &bc, nil } @@ -101,7 +110,9 @@ func (r *BlobChunkRepository) GetByChunkHashTx(ctx context.Context, tx *sql.Tx, ` LogSQL("GetByChunkHashTx", query, chunkHash) + var bc BlobChunk + err := tx.QueryRowContext(ctx, query, chunkHash).Scan( &bc.BlobID, &bc.ChunkHash, @@ -111,14 +122,18 @@ func (r *BlobChunkRepository) GetByChunkHashTx(ctx context.Context, tx *sql.Tx, if err == sql.ErrNoRows { LogSQL("GetByChunkHashTx", "No rows found", chunkHash) + return nil, nil } + if err != nil { LogSQL("GetByChunkHashTx", "Error", chunkHash, err) + return nil, fmt.Errorf("querying blob chunk: %w", err) } LogSQL("GetByChunkHashTx", "Found blob", chunkHash, "blob", bc.BlobID) + return &bc, nil } diff --git a/internal/database/blob_chunks_test.go b/internal/database/blob_chunks_test.go index 9dae601..97d3e68 100644 --- a/internal/database/blob_chunks_test.go +++ b/internal/database/blob_chunks_test.go @@ -22,6 +22,7 @@ func TestBlobChunkRepository(t *testing.T) { Hash: types.BlobHash("blob1-hash"), CreatedTS: time.Now(), } + err := repos.Blobs.Create(ctx, nil, blob) if err != nil { t.Fatalf("failed to create blob: %v", err) @@ -34,6 +35,7 @@ func TestBlobChunkRepository(t *testing.T) { ChunkHash: chunkHash, Size: 1024, } + err = repos.Chunks.Create(ctx, nil, chunk) if err != nil { t.Fatalf("failed to create chunk %s: %v", chunkHash, err) @@ -60,6 +62,7 @@ func TestBlobChunkRepository(t *testing.T) { Offset: 1024, Length: 2048, } + err = repos.BlobChunks.Create(ctx, nil, bc2) if err != nil { t.Fatalf("failed to create second blob chunk: %v", err) @@ -71,6 +74,7 @@ func TestBlobChunkRepository(t *testing.T) { Offset: 3072, Length: 512, } + err = repos.BlobChunks.Create(ctx, nil, bc3) if err != nil { t.Fatalf("failed to create third blob chunk: %v", err) @@ -81,6 +85,7 @@ func TestBlobChunkRepository(t *testing.T) { if err != nil { t.Fatalf("failed to get blob chunks: %v", err) } + if len(blobChunks) != 3 { t.Errorf("expected 3 chunks, got %d", len(blobChunks)) } @@ -98,12 +103,15 @@ func TestBlobChunkRepository(t *testing.T) { if err != nil { t.Fatalf("failed to get blob chunk by chunk hash: %v", err) } + if bc == nil { t.Fatal("expected blob chunk, got nil") } + if bc.BlobID != blob.ID { t.Errorf("wrong blob ID: expected %s, got %s", blob.ID, bc.BlobID) } + if bc.Offset != 1024 { t.Errorf("wrong offset: expected 1024, got %d", bc.Offset) } @@ -113,6 +121,7 @@ func TestBlobChunkRepository(t *testing.T) { if err == nil { t.Fatal("duplicate blob_chunk insert should fail due to primary key constraint") } + if !strings.Contains(err.Error(), "UNIQUE") && !strings.Contains(err.Error(), "constraint") { t.Fatalf("expected constraint error, got: %v", err) } @@ -122,6 +131,7 @@ func TestBlobChunkRepository(t *testing.T) { if err != nil { t.Fatalf("unexpected error: %v", err) } + if bc != nil { t.Error("expected nil for non-existent chunk") } @@ -150,6 +160,7 @@ func TestBlobChunkRepositoryMultipleBlobs(t *testing.T) { if err != nil { t.Fatalf("failed to create blob1: %v", err) } + err = repos.Blobs.Create(ctx, nil, blob2) if err != nil { t.Fatalf("failed to create blob2: %v", err) @@ -162,6 +173,7 @@ func TestBlobChunkRepositoryMultipleBlobs(t *testing.T) { ChunkHash: chunkHash, Size: 1024, } + err = repos.Chunks.Create(ctx, nil, chunk) if err != nil { t.Fatalf("failed to create chunk %s: %v", chunkHash, err) @@ -189,6 +201,7 @@ func TestBlobChunkRepositoryMultipleBlobs(t *testing.T) { if err != nil { t.Fatalf("failed to get blob1 chunks: %v", err) } + if len(chunks) != 2 { t.Errorf("expected 2 chunks for blob1, got %d", len(chunks)) } @@ -198,6 +211,7 @@ func TestBlobChunkRepositoryMultipleBlobs(t *testing.T) { if err != nil { t.Fatalf("failed to get blob2 chunks: %v", err) } + if len(chunks) != 2 { t.Errorf("expected 2 chunks for blob2, got %d", len(chunks)) } @@ -207,6 +221,7 @@ func TestBlobChunkRepositoryMultipleBlobs(t *testing.T) { if err != nil { t.Fatalf("failed to get shared chunk: %v", err) } + if bc == nil { t.Fatal("expected shared chunk, got nil") } diff --git a/internal/database/blobs.go b/internal/database/blobs.go index 58aed0f..43a48e7 100644 --- a/internal/database/blobs.go +++ b/internal/database/blobs.go @@ -24,10 +24,12 @@ func (r *BlobRepository) Create(ctx context.Context, tx *sql.Tx, blob *Blob) err ` var finishedTS, uploadedTS *int64 + if blob.FinishedTS != nil { ts := blob.FinishedTS.Unix() finishedTS = &ts } + if blob.UploadedTS != nil { ts := blob.UploadedTS.Unix() uploadedTS = &ts @@ -56,9 +58,11 @@ func (r *BlobRepository) GetByHash(ctx context.Context, hash string) (*Blob, err WHERE blob_hash = ? ` - var blob Blob - var createdTSUnix int64 - var finishedTSUnix, uploadedTSUnix sql.NullInt64 + var ( + blob Blob + createdTSUnix int64 + finishedTSUnix, uploadedTSUnix sql.NullInt64 + ) err := r.db.conn.QueryRowContext(ctx, query, hash).Scan( &blob.ID, @@ -73,6 +77,7 @@ func (r *BlobRepository) GetByHash(ctx context.Context, hash string) (*Blob, err if err == sql.ErrNoRows { return nil, nil } + if err != nil { return nil, fmt.Errorf("querying blob: %w", err) } @@ -82,10 +87,12 @@ func (r *BlobRepository) GetByHash(ctx context.Context, hash string) (*Blob, err ts := time.Unix(finishedTSUnix.Int64, 0).UTC() blob.FinishedTS = &ts } + if uploadedTSUnix.Valid { ts := time.Unix(uploadedTSUnix.Int64, 0).UTC() blob.UploadedTS = &ts } + return &blob, nil } @@ -97,9 +104,11 @@ func (r *BlobRepository) GetByID(ctx context.Context, id string) (*Blob, error) WHERE id = ? ` - var blob Blob - var createdTSUnix int64 - var finishedTSUnix, uploadedTSUnix sql.NullInt64 + var ( + blob Blob + createdTSUnix int64 + finishedTSUnix, uploadedTSUnix sql.NullInt64 + ) err := r.db.conn.QueryRowContext(ctx, query, id).Scan( &blob.ID, @@ -114,6 +123,7 @@ func (r *BlobRepository) GetByID(ctx context.Context, id string) (*Blob, error) if err == sql.ErrNoRows { return nil, nil } + if err != nil { return nil, fmt.Errorf("querying blob: %w", err) } @@ -123,10 +133,12 @@ func (r *BlobRepository) GetByID(ctx context.Context, id string) (*Blob, error) ts := time.Unix(finishedTSUnix.Int64, 0).UTC() blob.FinishedTS = &ts } + if uploadedTSUnix.Valid { ts := time.Unix(uploadedTSUnix.Int64, 0).UTC() blob.UploadedTS = &ts } + return &blob, nil } @@ -146,11 +158,15 @@ func (r *BlobRepository) GetAll(ctx context.Context) (map[string]*Blob, error) { defer CloseRows(rows) out := make(map[string]*Blob) + for rows.Next() { - var blob Blob - var createdTSUnix int64 - var finishedTSUnix, uploadedTSUnix sql.NullInt64 - if err := rows.Scan( + var ( + blob Blob + createdTSUnix int64 + finishedTSUnix, uploadedTSUnix sql.NullInt64 + ) + + err := rows.Scan( &blob.ID, &blob.Hash, &createdTSUnix, @@ -158,20 +174,25 @@ func (r *BlobRepository) GetAll(ctx context.Context) (map[string]*Blob, error) { &blob.UncompressedSize, &blob.CompressedSize, &uploadedTSUnix, - ); err != nil { + ) + if err != nil { return nil, fmt.Errorf("scanning blob: %w", err) } + blob.CreatedTS = time.Unix(createdTSUnix, 0).UTC() if finishedTSUnix.Valid { ts := time.Unix(finishedTSUnix.Int64, 0).UTC() blob.FinishedTS = &ts } + if uploadedTSUnix.Valid { ts := time.Unix(uploadedTSUnix.Int64, 0).UTC() blob.UploadedTS = &ts } + out[blob.ID.String()] = &blob } + return out, rows.Err() } @@ -184,6 +205,7 @@ func (r *BlobRepository) UpdateFinished(ctx context.Context, tx *sql.Tx, id stri ` now := time.Now().UTC().Unix() + var err error if tx != nil { _, err = tx.ExecContext(ctx, query, hash, now, uncompressedSize, compressedSize, id) @@ -207,6 +229,7 @@ func (r *BlobRepository) UpdateUploaded(ctx context.Context, tx *sql.Tx, id stri ` now := time.Now().UTC().Unix() + var err error if tx != nil { _, err = tx.ExecContext(ctx, query, now, id) diff --git a/internal/database/blobs_test.go b/internal/database/blobs_test.go index 3d06c5d..2b8e88b 100644 --- a/internal/database/blobs_test.go +++ b/internal/database/blobs_test.go @@ -32,12 +32,15 @@ func TestBlobRepository(t *testing.T) { if err != nil { t.Fatalf("failed to get blob: %v", err) } + if retrieved == nil { t.Fatal("expected blob, got nil") } + if retrieved.Hash != blob.Hash { t.Errorf("blob hash mismatch: got %s, want %s", retrieved.Hash, blob.Hash) } + if !retrieved.CreatedTS.Equal(blob.CreatedTS) { t.Errorf("created timestamp mismatch: got %v, want %v", retrieved.CreatedTS, blob.CreatedTS) } @@ -47,9 +50,11 @@ func TestBlobRepository(t *testing.T) { if err != nil { t.Fatalf("failed to get blob by ID: %v", err) } + if retrievedByID == nil { t.Fatal("expected blob, got nil") } + if retrievedByID.ID != blob.ID { t.Errorf("blob ID mismatch: got %s, want %s", retrievedByID.ID, blob.ID) } @@ -60,6 +65,7 @@ func TestBlobRepository(t *testing.T) { Hash: types.BlobHash("blobhash456"), CreatedTS: time.Now().Truncate(time.Second), } + err = repo.Create(ctx, nil, blob2) if err != nil { t.Fatalf("failed to create second blob: %v", err) @@ -67,6 +73,7 @@ func TestBlobRepository(t *testing.T) { // Test UpdateFinished now := time.Now() + err = repo.UpdateFinished(ctx, nil, blob.ID.String(), blob.Hash.String(), 1000, 500) if err != nil { t.Fatalf("failed to update blob as finished: %v", err) @@ -77,12 +84,15 @@ func TestBlobRepository(t *testing.T) { if err != nil { t.Fatalf("failed to get updated blob: %v", err) } + if updated.FinishedTS == nil { t.Fatal("expected finished timestamp to be set") } + if updated.UncompressedSize != 1000 { t.Errorf("expected uncompressed size 1000, got %d", updated.UncompressedSize) } + if updated.CompressedSize != 500 { t.Errorf("expected compressed size 500, got %d", updated.CompressedSize) } @@ -98,6 +108,7 @@ func TestBlobRepository(t *testing.T) { if err != nil { t.Fatalf("failed to get uploaded blob: %v", err) } + if uploaded.UploadedTS == nil { t.Fatal("expected uploaded timestamp to be set") } diff --git a/internal/database/cascade_debug_test.go b/internal/database/cascade_debug_test.go index 6ef933f..fc71f9f 100644 --- a/internal/database/cascade_debug_test.go +++ b/internal/database/cascade_debug_test.go @@ -19,10 +19,12 @@ func TestCascadeDeleteDebug(t *testing.T) { // Check if foreign keys are enabled var fkEnabled int + err := db.conn.QueryRow("PRAGMA foreign_keys").Scan(&fkEnabled) if err != nil { t.Fatal(err) } + t.Logf("Foreign keys enabled: %d", fkEnabled) // Create a file @@ -34,18 +36,21 @@ func TestCascadeDeleteDebug(t *testing.T) { UID: 1000, GID: 1000, } + err = repos.Files.Create(ctx, nil, file) if err != nil { t.Fatalf("failed to create file: %v", err) } + t.Logf("Created file with ID: %s", file.ID) // Create chunks and file-chunk mappings - for i := 0; i < 3; i++ { + for i := range 3 { chunk := &Chunk{ ChunkHash: types.ChunkHash(fmt.Sprintf("cascade-chunk-%d", i)), Size: 1024, } + err = repos.Chunks.Create(ctx, nil, chunk) if err != nil { t.Fatalf("failed to create chunk: %v", err) @@ -56,10 +61,12 @@ func TestCascadeDeleteDebug(t *testing.T) { Idx: i, ChunkHash: chunk.ChunkHash, } + err = repos.FileChunks.Create(ctx, nil, fc) if err != nil { t.Fatalf("failed to create file chunk: %v", err) } + t.Logf("Created file chunk mapping: file_id=%s, idx=%d, chunk=%s", fc.FileID, fc.Idx, fc.ChunkHash) } @@ -68,10 +75,12 @@ func TestCascadeDeleteDebug(t *testing.T) { if err != nil { t.Fatal(err) } + t.Logf("File chunks before delete: %d", len(fileChunks)) // Check the foreign key constraint var fkInfo string + err = db.conn.QueryRow(` SELECT sql FROM sqlite_master WHERE type='table' AND name='file_chunks' @@ -79,10 +88,12 @@ func TestCascadeDeleteDebug(t *testing.T) { if err != nil { t.Fatal(err) } + t.Logf("file_chunks table definition:\n%s", fkInfo) // Delete the file t.Log("Deleting file...") + err = repos.Files.DeleteByID(ctx, nil, file.ID) if err != nil { t.Fatalf("failed to delete file: %v", err) @@ -93,6 +104,7 @@ func TestCascadeDeleteDebug(t *testing.T) { if err != nil { t.Fatal(err) } + if deletedFile != nil { t.Error("file should have been deleted") } else { @@ -104,14 +116,17 @@ func TestCascadeDeleteDebug(t *testing.T) { if err != nil { t.Fatal(err) } + t.Logf("File chunks after delete: %d", len(fileChunks)) // Manually check the database var count int + err = db.conn.QueryRow("SELECT COUNT(*) FROM file_chunks WHERE file_id = ?", file.ID).Scan(&count) if err != nil { t.Fatal(err) } + t.Logf("Manual count of file_chunks for deleted file: %d", count) if len(fileChunks) != 0 { diff --git a/internal/database/chunk_files.go b/internal/database/chunk_files.go index 0e5f0df..01e9175 100644 --- a/internal/database/chunk_files.go +++ b/internal/database/chunk_files.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "fmt" + "strings" "sneak.berlin/go/vaultik/internal/types" ) @@ -90,18 +91,25 @@ func (r *ChunkFileRepository) GetByFileID(ctx context.Context, fileID types.File // scanChunkFiles is a helper that scans chunk file rows func (r *ChunkFileRepository) scanChunkFiles(rows *sql.Rows) ([]*ChunkFile, error) { var chunkFiles []*ChunkFile + for rows.Next() { - var cf ChunkFile - var chunkHashStr, fileIDStr string + var ( + cf ChunkFile + chunkHashStr, fileIDStr string + ) + err := rows.Scan(&chunkHashStr, &fileIDStr, &cf.FileOffset, &cf.Length) if err != nil { return nil, fmt.Errorf("scanning chunk file: %w", err) } + cf.ChunkHash = types.ChunkHash(chunkHashStr) + cf.FileID, err = types.ParseFileID(fileIDStr) if err != nil { return nil, fmt.Errorf("parsing file ID: %w", err) } + chunkFiles = append(chunkFiles, &cf) } @@ -136,14 +144,13 @@ func (r *ChunkFileRepository) DeleteByFileIDs(ctx context.Context, tx *sql.Tx, f const batchSize = 500 for i := 0; i < len(fileIDs); i += batchSize { - end := i + batchSize - if end > len(fileIDs) { - end = len(fileIDs) - } + end := min(i+batchSize, len(fileIDs)) + batch := fileIDs[i:end] query := "DELETE FROM chunk_files WHERE file_id IN (?" + repeatPlaceholder(len(batch)-1) + ")" - args := make([]interface{}, len(batch)) + + args := make([]any, len(batch)) for j, id := range batch { args[j] = id.String() } @@ -154,6 +161,7 @@ func (r *ChunkFileRepository) DeleteByFileIDs(ctx context.Context, tx *sql.Tx, f } else { _, err = r.db.ExecWithLog(ctx, query, args...) } + if err != nil { return fmt.Errorf("batch deleting chunk_files: %w", err) } @@ -172,21 +180,28 @@ func (r *ChunkFileRepository) CreateBatch(ctx context.Context, tx *sql.Tx, cfs [ const batchSize = 200 for i := 0; i < len(cfs); i += batchSize { - end := i + batchSize - if end > len(cfs) { - end = len(cfs) - } + end := min(i+batchSize, len(cfs)) + batch := cfs[i:end] query := "INSERT INTO chunk_files (chunk_hash, file_id, file_offset, length) VALUES " - args := make([]interface{}, 0, len(batch)*4) + + args := make([]any, 0, len(batch)*4) + + var querySb183 strings.Builder + for j, cf := range batch { if j > 0 { - query += ", " + querySb183.WriteString(", ") } - query += "(?, ?, ?, ?)" + + querySb183.WriteString("(?, ?, ?, ?)") + args = append(args, cf.ChunkHash.String(), cf.FileID.String(), cf.FileOffset, cf.Length) } + + query += querySb183.String() + query += " ON CONFLICT(chunk_hash, file_id) DO NOTHING" var err error @@ -195,6 +210,7 @@ func (r *ChunkFileRepository) CreateBatch(ctx context.Context, tx *sql.Tx, cfs [ } else { _, err = r.db.ExecWithLog(ctx, query, args...) } + if err != nil { return fmt.Errorf("batch inserting chunk_files: %w", err) } diff --git a/internal/database/chunk_files_test.go b/internal/database/chunk_files_test.go index a4e8bbc..ca3fd98 100644 --- a/internal/database/chunk_files_test.go +++ b/internal/database/chunk_files_test.go @@ -28,6 +28,7 @@ func TestChunkFileRepository(t *testing.T) { GID: 1000, LinkTarget: "", } + err := fileRepo.Create(ctx, nil, file1) if err != nil { t.Fatalf("failed to create file1: %v", err) @@ -42,6 +43,7 @@ func TestChunkFileRepository(t *testing.T) { GID: 1000, LinkTarget: "", } + err = fileRepo.Create(ctx, nil, file2) if err != nil { t.Fatalf("failed to create file2: %v", err) @@ -52,6 +54,7 @@ func TestChunkFileRepository(t *testing.T) { ChunkHash: types.ChunkHash("chunk1"), Size: 1024, } + err = chunksRepo.Create(ctx, nil, chunk) if err != nil { t.Fatalf("failed to create chunk: %v", err) @@ -77,6 +80,7 @@ func TestChunkFileRepository(t *testing.T) { FileOffset: 2048, Length: 1024, } + err = repo.Create(ctx, nil, cf2) if err != nil { t.Fatalf("failed to create second chunk file: %v", err) @@ -87,6 +91,7 @@ func TestChunkFileRepository(t *testing.T) { if err != nil { t.Fatalf("failed to get chunk files: %v", err) } + if len(chunkFiles) != 2 { t.Errorf("expected 2 files for chunk, got %d", len(chunkFiles)) } @@ -94,14 +99,17 @@ func TestChunkFileRepository(t *testing.T) { // Verify both files are returned foundFile1 := false foundFile2 := false + for _, cf := range chunkFiles { if cf.FileID == file1.ID && cf.FileOffset == 0 { foundFile1 = true } + if cf.FileID == file2.ID && cf.FileOffset == 2048 { foundFile2 = true } } + if !foundFile1 || !foundFile2 { t.Error("not all expected files found") } @@ -111,9 +119,11 @@ func TestChunkFileRepository(t *testing.T) { if err != nil { t.Fatalf("failed to get chunks by file ID: %v", err) } + if len(chunkFiles) != 1 { t.Errorf("expected 1 chunk for file, got %d", len(chunkFiles)) } + if chunkFiles[0].ChunkHash != types.ChunkHash("chunk1") { t.Errorf("wrong chunk hash: expected chunk1, got %s", chunkFiles[0].ChunkHash) } @@ -143,9 +153,11 @@ func TestChunkFileRepositoryComplexDeduplication(t *testing.T) { if err := fileRepo.Create(ctx, nil, file1); err != nil { t.Fatalf("failed to create file1: %v", err) } + if err := fileRepo.Create(ctx, nil, file2); err != nil { t.Fatalf("failed to create file2: %v", err) } + if err := fileRepo.Create(ctx, nil, file3); err != nil { t.Fatalf("failed to create file3: %v", err) } @@ -157,6 +169,7 @@ func TestChunkFileRepositoryComplexDeduplication(t *testing.T) { ChunkHash: chunkHash, Size: 1024, } + err := chunksRepo.Create(ctx, nil, chunk) if err != nil { t.Fatalf("failed to create chunk %s: %v", chunkHash, err) @@ -194,6 +207,7 @@ func TestChunkFileRepositoryComplexDeduplication(t *testing.T) { if err != nil { t.Fatalf("failed to get files for chunk1: %v", err) } + if len(files) != 2 { t.Errorf("expected 2 files for chunk1, got %d", len(files)) } @@ -203,6 +217,7 @@ func TestChunkFileRepositoryComplexDeduplication(t *testing.T) { if err != nil { t.Fatalf("failed to get files for chunk2: %v", err) } + if len(files) != 2 { t.Errorf("expected 2 files for chunk2, got %d", len(files)) } @@ -212,6 +227,7 @@ func TestChunkFileRepositoryComplexDeduplication(t *testing.T) { if err != nil { t.Fatalf("failed to get chunks for file2: %v", err) } + if len(file2Chunks) != 3 { t.Errorf("expected 3 chunks for file2, got %d", len(file2Chunks)) } diff --git a/internal/database/chunks.go b/internal/database/chunks.go index 04fc4ef..5c54a14 100644 --- a/internal/database/chunks.go +++ b/internal/database/chunks.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "fmt" + "strings" "sneak.berlin/go/vaultik/internal/log" ) @@ -54,6 +55,7 @@ func (r *ChunkRepository) GetByHash(ctx context.Context, hash string) (*Chunk, e if err == sql.ErrNoRows { return nil, nil } + if err != nil { return nil, fmt.Errorf("querying chunk: %w", err) } @@ -71,14 +73,22 @@ func (r *ChunkRepository) GetByHashes(ctx context.Context, hashes []string) ([]* FROM chunks WHERE chunk_hash IN (` - args := make([]interface{}, len(hashes)) + args := make([]any, len(hashes)) + + var querySb75 strings.Builder + for i, hash := range hashes { if i > 0 { - query += ", " + querySb75.WriteString(", ") } - query += "?" + + querySb75.WriteString("?") + args[i] = hash } + + query += querySb75.String() + query += ") ORDER BY chunk_hash" rows, err := r.db.conn.QueryContext(ctx, query, args...) @@ -88,6 +98,7 @@ func (r *ChunkRepository) GetByHashes(ctx context.Context, hashes []string) ([]* defer CloseRows(rows) var chunks []*Chunk + for rows.Next() { var chunk Chunk @@ -122,6 +133,7 @@ func (r *ChunkRepository) ListUnpacked(ctx context.Context, limit int) ([]*Chunk defer CloseRows(rows) var chunks []*Chunk + for rows.Next() { var chunk Chunk diff --git a/internal/database/chunks_ext.go b/internal/database/chunks_ext.go index b38c170..a322a9a 100644 --- a/internal/database/chunks_ext.go +++ b/internal/database/chunks_ext.go @@ -19,6 +19,7 @@ func (r *ChunkRepository) List(ctx context.Context) ([]*Chunk, error) { defer CloseRows(rows) var chunks []*Chunk + for rows.Next() { var chunk Chunk diff --git a/internal/database/chunks_test.go b/internal/database/chunks_test.go index 5122bb1..8f96bd1 100644 --- a/internal/database/chunks_test.go +++ b/internal/database/chunks_test.go @@ -30,12 +30,15 @@ func TestChunkRepository(t *testing.T) { if err != nil { t.Fatalf("failed to get chunk: %v", err) } + if retrieved == nil { t.Fatal("expected chunk, got nil") } + if retrieved.ChunkHash != chunk.ChunkHash { t.Errorf("chunk hash mismatch: got %s, want %s", retrieved.ChunkHash, chunk.ChunkHash) } + if retrieved.Size != chunk.Size { t.Errorf("size mismatch: got %d, want %d", retrieved.Size, chunk.Size) } @@ -51,6 +54,7 @@ func TestChunkRepository(t *testing.T) { ChunkHash: types.ChunkHash("chunkhash456"), Size: 8192, } + err = repo.Create(ctx, nil, chunk2) if err != nil { t.Fatalf("failed to create second chunk: %v", err) @@ -60,6 +64,7 @@ func TestChunkRepository(t *testing.T) { if err != nil { t.Fatalf("failed to get chunks by hashes: %v", err) } + if len(chunks) != 2 { t.Errorf("expected 2 chunks, got %d", len(chunks)) } @@ -69,6 +74,7 @@ func TestChunkRepository(t *testing.T) { if err != nil { t.Fatalf("failed to list unpacked chunks: %v", err) } + if len(unpacked) != 2 { t.Errorf("expected 2 unpacked chunks, got %d", len(unpacked)) } @@ -86,6 +92,7 @@ func TestChunkRepositoryNotFound(t *testing.T) { if err != nil { t.Fatalf("unexpected error: %v", err) } + if chunk != nil { t.Error("expected nil for non-existent chunk") } @@ -95,6 +102,7 @@ func TestChunkRepositoryNotFound(t *testing.T) { if err != nil { t.Fatalf("unexpected error: %v", err) } + if chunks != nil { t.Error("expected nil for empty hash list") } diff --git a/internal/database/database.go b/internal/database/database.go index 5f0f14c..7fc717f 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -57,8 +57,8 @@ func ParseMigrationVersion(filename string) (int, error) { // Split on underscore to separate version from description. // If there's no underscore, the entire stem is the version. versionStr := name - if idx := strings.IndexByte(name, '_'); idx >= 0 { - versionStr = name[:idx] + if before, _, ok := strings.Cut(name, "_"); ok { + versionStr = before } if versionStr == "" { @@ -98,6 +98,7 @@ func New(ctx context.Context, path string) (*DB, error) { // First attempt with standard WAL mode log.Debug("Attempting to open database with WAL mode", "path", path) + conn, err := sql.Open( "sqlite", path+"?_journal_mode=WAL&_synchronous=NORMAL&_busy_timeout=10000&_locking_mode=NORMAL&_foreign_keys=ON", @@ -110,7 +111,8 @@ func New(ctx context.Context, path string) (*DB, error) { conn.SetMaxOpenConns(1) conn.SetMaxIdleConns(1) - if err := conn.PingContext(ctx); err == nil { + err := conn.PingContext(ctx) + if err == nil { // Success on first try log.Debug("Database opened successfully with WAL mode", "path", path) @@ -120,13 +122,19 @@ func New(ctx context.Context, path string) (*DB, error) { } db := &DB{conn: conn, path: path} - if err := applyMigrations(ctx, conn); err != nil { + + err := applyMigrations(ctx, conn) + if err != nil { _ = conn.Close() + return nil, fmt.Errorf("applying migrations: %w", err) } + return db, nil } + log.Debug("Failed to ping database, closing connection", "path", path, "error", err) + _ = conn.Close() } @@ -135,6 +143,7 @@ func New(ctx context.Context, path string) (*DB, error) { "Database appears locked, attempting recovery with TRUNCATE mode", "path", path, ) + conn, err = sql.Open( "sqlite", path+"?_journal_mode=TRUNCATE&_synchronous=NORMAL&_busy_timeout=10000&_foreign_keys=ON", @@ -152,7 +161,9 @@ func New(ctx context.Context, path string) (*DB, error) { if err := conn.PingContext(ctx); err != nil { log.Debug("Failed to ping database in recovery mode, closing", "path", path, "error", err) + _ = conn.Close() + return nil, fmt.Errorf( "database still locked after recovery attempt: %w", err, @@ -163,6 +174,7 @@ func New(ctx context.Context, path string) (*DB, error) { // Switch back to WAL mode log.Debug("Switching database back to WAL mode", "path", path) + if _, err := conn.ExecContext(ctx, "PRAGMA journal_mode=WAL"); err != nil { log.Warn("Failed to switch back to WAL mode", "path", path, "error", err) } @@ -175,10 +187,12 @@ func New(ctx context.Context, path string) (*DB, error) { db := &DB{conn: conn, path: path} if err := applyMigrations(ctx, conn); err != nil { _ = conn.Close() + return nil, fmt.Errorf("applying migrations: %w", err) } log.Debug("Database connection established successfully", "path", path) + return db, nil } @@ -187,11 +201,16 @@ func New(ctx context.Context, path string) (*DB, error) { // Returns an error if the database connection cannot be closed properly. func (db *DB) Close() error { log.Debug("Closing database connection", "path", db.path) - if err := db.conn.Close(); err != nil { + + err := db.conn.Close() + if err != nil { log.Error("Failed to close database", "path", db.path, "error", err) + return fmt.Errorf("failed to close database: %w", err) } + log.Debug("Database connection closed successfully", "path", db.path) + return nil } @@ -227,9 +246,10 @@ func (db *DB) BeginTx( func (db *DB) ExecWithLog( ctx context.Context, query string, - args ...interface{}, + args ...any, ) (sql.Result, error) { LogSQL("Execute", query, args...) + return db.conn.ExecContext(ctx, query, args...) } @@ -240,9 +260,10 @@ func (db *DB) ExecWithLog( func (db *DB) QueryRowWithLog( ctx context.Context, query string, - args ...interface{}, + args ...any, ) *sql.Row { LogSQL("QueryRow", query, args...) + return db.conn.QueryRowContext(ctx, query, args...) } @@ -375,6 +396,7 @@ func repeatPlaceholder(n int) string { if n <= 0 { return "" } + return strings.Repeat(", ?", n) } @@ -385,7 +407,7 @@ func repeatPlaceholder(n int) string { // The operation parameter describes the type of SQL operation (e.g., "Execute", "Query"). // The query parameter is the SQL statement being executed. // The args parameter contains the query arguments that will be interpolated. -func LogSQL(operation, query string, args ...interface{}) { +func LogSQL(operation, query string, args ...any) { if strings.Contains(os.Getenv("GODEBUG"), "vaultik") { log.Debug( "SQL "+operation, diff --git a/internal/database/database_test.go b/internal/database/database_test.go index 6d763a3..7700ac1 100644 --- a/internal/database/database_test.go +++ b/internal/database/database_test.go @@ -17,7 +17,8 @@ func TestDatabase(t *testing.T) { t.Fatalf("failed to create database: %v", err) } defer func() { - if err := db.Close(); err != nil { + err := db.Close() + if err != nil { t.Errorf("failed to close database: %v", err) } }() @@ -37,6 +38,7 @@ func TestDatabase(t *testing.T) { for _, table := range tables { var name string + err := db.conn.QueryRow("SELECT name FROM sqlite_master WHERE type='table' AND name=?", table).Scan(&name) if err != nil { t.Errorf("table %s does not exist: %v", table, err) @@ -63,7 +65,8 @@ func TestDatabaseConcurrentAccess(t *testing.T) { t.Fatalf("failed to create database: %v", err) } defer func() { - if err := db.Close(); err != nil { + err := db.Close() + if err != nil { t.Errorf("failed to close database: %v", err) } }() @@ -73,9 +76,10 @@ func TestDatabaseConcurrentAccess(t *testing.T) { index int err error } + results := make(chan result, 10) - for i := 0; i < 10; i++ { + for i := range 10 { go func(i int) { _, err := db.ExecWithLog(ctx, "INSERT INTO chunks (chunk_hash, size) VALUES (?, ?)", fmt.Sprintf("hash%d", i), i*1024) @@ -84,7 +88,7 @@ func TestDatabaseConcurrentAccess(t *testing.T) { } // Wait for all goroutines and check results - for i := 0; i < 10; i++ { + for range 10 { r := <-results if r.err != nil { t.Fatalf("concurrent insert %d failed: %v", r.index, r.err) @@ -93,10 +97,12 @@ func TestDatabaseConcurrentAccess(t *testing.T) { // Verify all inserts succeeded var count int + err = db.conn.QueryRowContext(ctx, "SELECT COUNT(*) FROM chunks").Scan(&count) if err != nil { t.Fatalf("failed to count chunks: %v", err) } + if count != 10 { t.Errorf("expected 10 chunks, got %d", count) } @@ -127,12 +133,16 @@ func TestParseMigrationVersion(t *testing.T) { if err == nil { t.Errorf("ParseMigrationVersion(%q) = %d, nil; want error", tc.filename, got) } + return } + if err != nil { t.Errorf("ParseMigrationVersion(%q) unexpected error: %v", tc.filename, err) + return } + if got != tc.wantVer { t.Errorf("ParseMigrationVersion(%q) = %d; want %d", tc.filename, got, tc.wantVer) } @@ -148,7 +158,8 @@ func TestApplyMigrations_Idempotent(t *testing.T) { t.Fatalf("failed to open database: %v", err) } defer func() { - if err := conn.Close(); err != nil { + err := conn.Close() + if err != nil { t.Errorf("failed to close database: %v", err) } }() @@ -191,7 +202,8 @@ func TestBootstrapMigrationsTable_FreshDatabase(t *testing.T) { t.Fatalf("failed to open database: %v", err) } defer func() { - if err := conn.Close(); err != nil { + err := conn.Close() + if err != nil { t.Errorf("failed to close database: %v", err) } }() @@ -206,6 +218,7 @@ func TestBootstrapMigrationsTable_FreshDatabase(t *testing.T) { ).Scan(&tableBefore); err != nil { t.Fatalf("failed to check for table before bootstrap: %v", err) } + if tableBefore != 0 { t.Fatal("schema_migrations table should not exist before bootstrap") } @@ -222,6 +235,7 @@ func TestBootstrapMigrationsTable_FreshDatabase(t *testing.T) { ).Scan(&tableAfter); err != nil { t.Fatalf("failed to check for table after bootstrap: %v", err) } + if tableAfter != 1 { t.Fatalf("schema_migrations table should exist after bootstrap, got count=%d", tableAfter) } @@ -233,6 +247,7 @@ func TestBootstrapMigrationsTable_FreshDatabase(t *testing.T) { ).Scan(&version); err != nil { t.Fatalf("version 0 row not found in schema_migrations: %v", err) } + if version != 0 { t.Errorf("expected version 0, got %d", version) } diff --git a/internal/database/errors.go b/internal/database/errors.go index b2b4497..860c23f 100644 --- a/internal/database/errors.go +++ b/internal/database/errors.go @@ -6,15 +6,16 @@ import ( "os" ) -// Fatal prints an error message to stderr and exits with status 1 -func Fatal(format string, args ...interface{}) { +// Fatalf prints an error message to stderr and exits with status 1 +func Fatalf(format string, args ...any) { fmt.Fprintf(os.Stderr, "FATAL: "+format+"\n", args...) os.Exit(1) } // CloseRows closes rows and exits on error func CloseRows(rows *sql.Rows) { - if err := rows.Close(); err != nil { - Fatal("failed to close rows: %v", err) + err := rows.Close() + if err != nil { + Fatalf("failed to close rows: %v", err) } } diff --git a/internal/database/file_chunks.go b/internal/database/file_chunks.go index 31fa142..c6b32f4 100644 --- a/internal/database/file_chunks.go +++ b/internal/database/file_chunks.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "fmt" + "strings" "sneak.berlin/go/vaultik/internal/types" ) @@ -84,6 +85,7 @@ func (r *FileChunkRepository) GetByPathTx(ctx context.Context, tx *sql.Tx, path ` LogSQL("GetByPathTx", query, path) + rows, err := tx.QueryContext(ctx, query, path) if err != nil { return nil, fmt.Errorf("querying file chunks: %w", err) @@ -92,23 +94,30 @@ func (r *FileChunkRepository) GetByPathTx(ctx context.Context, tx *sql.Tx, path fileChunks, err := r.scanFileChunks(rows) LogSQL("GetByPathTx", "Complete", path, "count", len(fileChunks)) + return fileChunks, err } // scanFileChunks is a helper that scans file chunk rows func (r *FileChunkRepository) scanFileChunks(rows *sql.Rows) ([]*FileChunk, error) { var fileChunks []*FileChunk + for rows.Next() { - var fc FileChunk - var fileIDStr, chunkHashStr string + var ( + fc FileChunk + fileIDStr, chunkHashStr string + ) + err := rows.Scan(&fileIDStr, &fc.Idx, &chunkHashStr) if err != nil { return nil, fmt.Errorf("scanning file chunk: %w", err) } + fc.FileID, err = types.ParseFileID(fileIDStr) if err != nil { return nil, fmt.Errorf("parsing file ID: %w", err) } + fc.ChunkHash = types.ChunkHash(chunkHashStr) fileChunks = append(fileChunks, &fc) } @@ -161,14 +170,13 @@ func (r *FileChunkRepository) DeleteByFileIDs(ctx context.Context, tx *sql.Tx, f const batchSize = 500 for i := 0; i < len(fileIDs); i += batchSize { - end := i + batchSize - if end > len(fileIDs) { - end = len(fileIDs) - } + end := min(i+batchSize, len(fileIDs)) + batch := fileIDs[i:end] query := "DELETE FROM file_chunks WHERE file_id IN (?" + repeatPlaceholder(len(batch)-1) + ")" - args := make([]interface{}, len(batch)) + + args := make([]any, len(batch)) for j, id := range batch { args[j] = id.String() } @@ -179,6 +187,7 @@ func (r *FileChunkRepository) DeleteByFileIDs(ctx context.Context, tx *sql.Tx, f } else { _, err = r.db.ExecWithLog(ctx, query, args...) } + if err != nil { return fmt.Errorf("batch deleting file_chunks: %w", err) } @@ -199,22 +208,29 @@ func (r *FileChunkRepository) CreateBatch(ctx context.Context, tx *sql.Tx, fcs [ const batchSize = 300 for i := 0; i < len(fcs); i += batchSize { - end := i + batchSize - if end > len(fcs) { - end = len(fcs) - } + end := min(i+batchSize, len(fcs)) + batch := fcs[i:end] // Build the query with multiple value sets query := "INSERT INTO file_chunks (file_id, idx, chunk_hash) VALUES " - args := make([]interface{}, 0, len(batch)*3) + + args := make([]any, 0, len(batch)*3) + + var querySb211 strings.Builder + for j, fc := range batch { if j > 0 { - query += ", " + querySb211.WriteString(", ") } - query += "(?, ?, ?)" + + querySb211.WriteString("(?, ?, ?)") + args = append(args, fc.FileID.String(), fc.Idx, fc.ChunkHash.String()) } + + query += querySb211.String() + query += " ON CONFLICT(file_id, idx) DO NOTHING" var err error @@ -223,6 +239,7 @@ func (r *FileChunkRepository) CreateBatch(ctx context.Context, tx *sql.Tx, fcs [ } else { _, err = r.db.ExecWithLog(ctx, query, args...) } + if err != nil { return fmt.Errorf("batch inserting file_chunks: %w", err) } @@ -236,6 +253,7 @@ func (r *FileChunkRepository) GetByFile(ctx context.Context, path string) ([]*Fi LogSQL("GetByFile", "Starting", path) result, err := r.GetByPath(ctx, path) LogSQL("GetByFile", "Complete", path, "count", len(result)) + return result, err } @@ -244,5 +262,6 @@ func (r *FileChunkRepository) GetByFileTx(ctx context.Context, tx *sql.Tx, path LogSQL("GetByFileTx", "Starting", path) result, err := r.GetByPathTx(ctx, tx, path) LogSQL("GetByFileTx", "Complete", path, "count", len(result)) + return result, err } diff --git a/internal/database/file_chunks_test.go b/internal/database/file_chunks_test.go index ebd2e01..e1eb2b9 100644 --- a/internal/database/file_chunks_test.go +++ b/internal/database/file_chunks_test.go @@ -28,6 +28,7 @@ func TestFileChunkRepository(t *testing.T) { GID: 1000, LinkTarget: "", } + err := fileRepo.Create(ctx, nil, file) if err != nil { t.Fatalf("failed to create file: %v", err) @@ -36,11 +37,13 @@ func TestFileChunkRepository(t *testing.T) { // Create chunks first chunks := []types.ChunkHash{"chunk1", "chunk2", "chunk3"} chunkRepo := NewChunkRepository(db) + for _, chunkHash := range chunks { chunk := &Chunk{ ChunkHash: chunkHash, Size: 1024, } + err = chunkRepo.Create(ctx, nil, chunk) if err != nil { t.Fatalf("failed to create chunk %s: %v", chunkHash, err) @@ -65,6 +68,7 @@ func TestFileChunkRepository(t *testing.T) { Idx: 1, ChunkHash: types.ChunkHash("chunk2"), } + err = repo.Create(ctx, nil, fc2) if err != nil { t.Fatalf("failed to create second file chunk: %v", err) @@ -75,6 +79,7 @@ func TestFileChunkRepository(t *testing.T) { Idx: 2, ChunkHash: types.ChunkHash("chunk3"), } + err = repo.Create(ctx, nil, fc3) if err != nil { t.Fatalf("failed to create third file chunk: %v", err) @@ -85,6 +90,7 @@ func TestFileChunkRepository(t *testing.T) { if err != nil { t.Fatalf("failed to get file chunks: %v", err) } + if len(fileChunks) != 3 { t.Errorf("expected 3 chunks, got %d", len(fileChunks)) } @@ -112,6 +118,7 @@ func TestFileChunkRepository(t *testing.T) { if err != nil { t.Fatalf("failed to get deleted file chunks: %v", err) } + if len(fileChunks) != 0 { t.Errorf("expected 0 chunks after delete, got %d", len(fileChunks)) } @@ -140,22 +147,26 @@ func TestFileChunkRepositoryMultipleFiles(t *testing.T) { GID: 1000, LinkTarget: "", } + err := fileRepo.Create(ctx, nil, file) if err != nil { t.Fatalf("failed to create file %s: %v", path, err) } + files[i] = file } // Create all chunks first chunkRepo := NewChunkRepository(db) + for i := range files { - for j := 0; j < 2; j++ { + for j := range 2 { chunkHash := types.ChunkHash(fmt.Sprintf("file%d_chunk%d", i, j)) chunk := &Chunk{ ChunkHash: chunkHash, Size: 1024, } + err := chunkRepo.Create(ctx, nil, chunk) if err != nil { t.Fatalf("failed to create chunk %s: %v", chunkHash, err) @@ -165,12 +176,13 @@ func TestFileChunkRepositoryMultipleFiles(t *testing.T) { // Create chunks for multiple files for i, file := range files { - for j := 0; j < 2; j++ { + for j := range 2 { fc := &FileChunk{ FileID: file.ID, Idx: j, ChunkHash: types.ChunkHash(fmt.Sprintf("file%d_chunk%d", i, j)), } + err := repo.Create(ctx, nil, fc) if err != nil { t.Fatalf("failed to create file chunk: %v", err) @@ -184,6 +196,7 @@ func TestFileChunkRepositoryMultipleFiles(t *testing.T) { if err != nil { t.Fatalf("failed to get chunks for file %d: %v", i, err) } + if len(chunks) != 2 { t.Errorf("expected 2 chunks for file %d, got %d", i, len(chunks)) } diff --git a/internal/database/files.go b/internal/database/files.go index bf81e18..77ec1ec 100644 --- a/internal/database/files.go +++ b/internal/database/files.go @@ -3,7 +3,9 @@ package database import ( "context" "database/sql" + "errors" "fmt" + "strings" "time" "sneak.berlin/go/vaultik/internal/log" @@ -38,8 +40,11 @@ func (r *FileRepository) Create(ctx context.Context, tx *sql.Tx, file *File) err RETURNING id ` - var idStr string - var err error + var ( + idStr string + err error + ) + if tx != nil { LogSQL("Execute", query, file.ID.String(), file.Path.String(), file.SourcePath.String(), file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID, file.LinkTarget.String()) err = tx.QueryRowContext(ctx, query, file.ID.String(), file.Path.String(), file.SourcePath.String(), file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID, file.LinkTarget.String()).Scan(&idStr) @@ -68,9 +73,10 @@ func (r *FileRepository) GetByPath(ctx context.Context, path string) (*File, err ` file, err := r.scanFile(r.db.conn.QueryRowContext(ctx, query, path)) - if err == sql.ErrNoRows { + if errors.Is(err, sql.ErrNoRows) { return nil, nil } + if err != nil { return nil, fmt.Errorf("querying file: %w", err) } @@ -87,9 +93,10 @@ func (r *FileRepository) GetByID(ctx context.Context, id types.FileID) (*File, e ` file, err := r.scanFile(r.db.conn.QueryRowContext(ctx, query, id.String())) - if err == sql.ErrNoRows { + if errors.Is(err, sql.ErrNoRows) { return nil, nil } + if err != nil { return nil, fmt.Errorf("querying file: %w", err) } @@ -108,9 +115,10 @@ func (r *FileRepository) GetByPathTx(ctx context.Context, tx *sql.Tx, path strin file, err := r.scanFile(tx.QueryRowContext(ctx, query, path)) LogSQL("GetByPathTx Scan complete", query, path) - if err == sql.ErrNoRows { + if errors.Is(err, sql.ErrNoRows) { return nil, nil } + if err != nil { return nil, fmt.Errorf("querying file: %w", err) } @@ -120,10 +128,12 @@ func (r *FileRepository) GetByPathTx(ctx context.Context, tx *sql.Tx, path strin // scanFile is a helper that scans a single file row func (r *FileRepository) scanFile(row *sql.Row) (*File, error) { - var file File - var idStr, pathStr, sourcePathStr string - var mtimeUnix int64 - var linkTarget sql.NullString + var ( + file File + idStr, pathStr, sourcePathStr string + mtimeUnix int64 + linkTarget sql.NullString + ) err := row.Scan( &idStr, @@ -144,8 +154,10 @@ func (r *FileRepository) scanFile(row *sql.Row) (*File, error) { if err != nil { return nil, fmt.Errorf("parsing file ID: %w", err) } + file.Path = types.FilePath(pathStr) file.SourcePath = types.SourcePath(sourcePathStr) + file.MTime = time.Unix(mtimeUnix, 0).UTC() if linkTarget.Valid { file.LinkTarget = types.FilePath(linkTarget.String) @@ -156,10 +168,12 @@ func (r *FileRepository) scanFile(row *sql.Row) (*File, error) { // scanFileRows is a helper that scans a file row from rows iterator func (r *FileRepository) scanFileRows(rows *sql.Rows) (*File, error) { - var file File - var idStr, pathStr, sourcePathStr string - var mtimeUnix int64 - var linkTarget sql.NullString + var ( + file File + idStr, pathStr, sourcePathStr string + mtimeUnix int64 + linkTarget sql.NullString + ) err := rows.Scan( &idStr, @@ -180,8 +194,10 @@ func (r *FileRepository) scanFileRows(rows *sql.Rows) (*File, error) { if err != nil { return nil, fmt.Errorf("parsing file ID: %w", err) } + file.Path = types.FilePath(pathStr) file.SourcePath = types.SourcePath(sourcePathStr) + file.MTime = time.Unix(mtimeUnix, 0).UTC() if linkTarget.Valid { file.LinkTarget = types.FilePath(linkTarget.String) @@ -205,11 +221,13 @@ func (r *FileRepository) ListModifiedSince(ctx context.Context, since time.Time) defer CloseRows(rows) var files []*File + for rows.Next() { file, err := r.scanFileRows(rows) if err != nil { return nil, fmt.Errorf("scanning file: %w", err) } + files = append(files, file) } @@ -266,11 +284,13 @@ func (r *FileRepository) ListByPrefix(ctx context.Context, prefix string) ([]*Fi defer CloseRows(rows) var files []*File + for rows.Next() { file, err := r.scanFileRows(rows) if err != nil { return nil, fmt.Errorf("scanning file: %w", err) } + files = append(files, file) } @@ -292,11 +312,13 @@ func (r *FileRepository) ListAll(ctx context.Context) ([]*File, error) { defer CloseRows(rows) var files []*File + for rows.Next() { file, err := r.scanFileRows(rows) if err != nil { return nil, fmt.Errorf("scanning file: %w", err) } + files = append(files, file) } @@ -314,21 +336,28 @@ func (r *FileRepository) CreateBatch(ctx context.Context, tx *sql.Tx, files []*F const batchSize = 100 for i := 0; i < len(files); i += batchSize { - end := i + batchSize - if end > len(files) { - end = len(files) - } + end := min(i+batchSize, len(files)) + batch := files[i:end] query := `INSERT INTO files (id, path, source_path, mtime, size, mode, uid, gid, link_target) VALUES ` - args := make([]interface{}, 0, len(batch)*9) + + args := make([]any, 0, len(batch)*9) + + var querySb325 strings.Builder + for j, f := range batch { if j > 0 { - query += ", " + querySb325.WriteString(", ") } - query += "(?, ?, ?, ?, ?, ?, ?, ?, ?)" + + querySb325.WriteString("(?, ?, ?, ?, ?, ?, ?, ?, ?)") + args = append(args, f.ID.String(), f.Path.String(), f.SourcePath.String(), f.MTime.Unix(), f.Size, f.Mode, f.UID, f.GID, f.LinkTarget.String()) } + + query += querySb325.String() + query += ` ON CONFLICT(path) DO UPDATE SET source_path = excluded.source_path, mtime = excluded.mtime, @@ -344,6 +373,7 @@ func (r *FileRepository) CreateBatch(ctx context.Context, tx *sql.Tx, files []*F } else { _, err = r.db.ExecWithLog(ctx, query, args...) } + if err != nil { return fmt.Errorf("batch inserting files: %w", err) } diff --git a/internal/database/files_test.go b/internal/database/files_test.go index 8f16421..7f4b472 100644 --- a/internal/database/files_test.go +++ b/internal/database/files_test.go @@ -3,7 +3,7 @@ package database import ( "context" "database/sql" - "fmt" + "errors" "os" "path/filepath" "testing" @@ -20,7 +20,8 @@ func setupTestDB(t *testing.T) (*DB, func()) { } cleanup := func() { - if err := db.Close(); err != nil { + err := db.Close() + if err != nil { t.Errorf("failed to close database: %v", err) } } @@ -56,18 +57,23 @@ func TestFileRepository(t *testing.T) { if err != nil { t.Fatalf("failed to get file: %v", err) } + if retrieved == nil { t.Fatal("expected file, got nil") } + if retrieved.Path != file.Path { t.Errorf("path mismatch: got %s, want %s", retrieved.Path, file.Path) } + if !retrieved.MTime.Equal(file.MTime) { t.Errorf("mtime mismatch: got %v, want %v", retrieved.MTime, file.MTime) } + if retrieved.Size != file.Size { t.Errorf("size mismatch: got %d, want %d", retrieved.Size, file.Size) } + if retrieved.Mode != file.Mode { t.Errorf("mode mismatch: got %o, want %o", retrieved.Mode, file.Mode) } @@ -75,6 +81,7 @@ func TestFileRepository(t *testing.T) { // Test Update (upsert) file.Size = 2048 file.MTime = time.Now().Truncate(time.Second) + err = repo.Create(ctx, nil, file) if err != nil { t.Fatalf("failed to update file: %v", err) @@ -84,6 +91,7 @@ func TestFileRepository(t *testing.T) { if err != nil { t.Fatalf("failed to get updated file: %v", err) } + if retrieved.Size != 2048 { t.Errorf("size not updated: got %d, want %d", retrieved.Size, 2048) } @@ -93,6 +101,7 @@ func TestFileRepository(t *testing.T) { if err != nil { t.Fatalf("failed to list files: %v", err) } + if len(files) != 1 { t.Errorf("expected 1 file, got %d", len(files)) } @@ -107,6 +116,7 @@ func TestFileRepository(t *testing.T) { if err != nil { t.Fatalf("error getting deleted file: %v", err) } + if retrieved != nil { t.Error("expected nil for deleted file") } @@ -139,9 +149,11 @@ func TestFileRepositorySymlink(t *testing.T) { if err != nil { t.Fatalf("failed to get symlink: %v", err) } + if !retrieved.IsSymlink() { t.Error("expected IsSymlink() to be true") } + if retrieved.LinkTarget != symlink.LinkTarget { t.Errorf("link target mismatch: got %s, want %s", retrieved.LinkTarget, symlink.LinkTarget) } @@ -165,12 +177,13 @@ func TestFileRepositoryTransaction(t *testing.T) { GID: 1000, } - if err := repos.Files.Create(ctx, tx, file); err != nil { + err := repos.Files.Create(ctx, tx, file) + if err != nil { return err } // Return error to trigger rollback - return fmt.Errorf("test rollback") + return errors.New("test rollback") }) if err == nil || err.Error() != "test rollback" { @@ -182,6 +195,7 @@ func TestFileRepositoryTransaction(t *testing.T) { if err != nil { t.Fatalf("error checking for file: %v", err) } + if retrieved != nil { t.Error("file should not exist after rollback") } diff --git a/internal/database/local_meta.go b/internal/database/local_meta.go index cf92ff5..83c2048 100644 --- a/internal/database/local_meta.go +++ b/internal/database/local_meta.go @@ -27,15 +27,18 @@ func NewLocalMetaRepository(db *DB) *LocalMetaRepository { // "unset" (bind on first use) from "set to something" (compare). func (r *LocalMetaRepository) Get(ctx context.Context, key string) (string, error) { var value string + err := r.db.conn.QueryRowContext(ctx, "SELECT value FROM local_meta WHERE key = ?", key, ).Scan(&value) if errors.Is(err, sql.ErrNoRows) { return "", nil } + if err != nil { return "", fmt.Errorf("reading local_meta %q: %w", key, err) } + return value, nil } @@ -49,5 +52,6 @@ func (r *LocalMetaRepository) Set(ctx context.Context, key, value string) error if err != nil { return fmt.Errorf("writing local_meta %q: %w", key, err) } + return nil } diff --git a/internal/database/local_meta_test.go b/internal/database/local_meta_test.go index b2be08d..d9b1d64 100644 --- a/internal/database/local_meta_test.go +++ b/internal/database/local_meta_test.go @@ -11,18 +11,20 @@ import ( func TestLocalMetaEmptyOnFresh(t *testing.T) { db, err := database.NewTestDB() require.NoError(t, err) + defer func() { _ = db.Close() }() repos := database.NewRepositories(db) got, err := repos.LocalMeta.Get(context.Background(), database.LocalMetaKeyStorageURL) require.NoError(t, err) - require.Equal(t, "", got, "fresh DB must return empty for unset keys, not error") + require.Empty(t, got, "fresh DB must return empty for unset keys, not error") } func TestLocalMetaSetGetRoundTrip(t *testing.T) { db, err := database.NewTestDB() require.NoError(t, err) + defer func() { _ = db.Close() }() repos := database.NewRepositories(db) @@ -38,6 +40,7 @@ func TestLocalMetaSetGetRoundTrip(t *testing.T) { func TestLocalMetaSetOverwrites(t *testing.T) { db, err := database.NewTestDB() require.NoError(t, err) + defer func() { _ = db.Close() }() repos := database.NewRepositories(db) diff --git a/internal/database/module.go b/internal/database/module.go index 98a5b74..e4af208 100644 --- a/internal/database/module.go +++ b/internal/database/module.go @@ -34,11 +34,16 @@ func provideDatabase(lc fx.Lifecycle, cfg *config.Config) (*DB, error) { lc.Append(fx.Hook{ OnStop: func(ctx context.Context) error { log.Debug("Database module OnStop hook called") - if err := db.Close(); err != nil { + + err := db.Close() + if err != nil { log.Error("Failed to close database in OnStop hook", "error", err) + return err } + log.Debug("Database closed successfully in OnStop hook") + return nil }, }) diff --git a/internal/database/repositories.go b/internal/database/repositories.go index 8bcf2e7..64a3f66 100644 --- a/internal/database/repositories.go +++ b/internal/database/repositories.go @@ -50,21 +50,26 @@ type TxFunc func(ctx context.Context, tx *sql.Tx) error // This method should be used for all write operations to ensure atomicity. func (r *Repositories) WithTx(ctx context.Context, fn TxFunc) error { LogSQL("WithTx", "Beginning transaction", "") + tx, err := r.db.BeginTx(ctx, nil) if err != nil { return fmt.Errorf("beginning transaction: %w", err) } + LogSQL("WithTx", "Transaction started", "") defer func() { if p := recover(); p != nil { - if rollbackErr := tx.Rollback(); rollbackErr != nil { - Fatal("failed to rollback transaction: %v", rollbackErr) + rollbackErr := tx.Rollback() + if rollbackErr != nil { + Fatalf("failed to rollback transaction: %v", rollbackErr) } + panic(p) } else if err != nil { - if rollbackErr := tx.Rollback(); rollbackErr != nil { - Fatal("failed to rollback transaction: %v", rollbackErr) + rollbackErr := tx.Rollback() + if rollbackErr != nil { + Fatalf("failed to rollback transaction: %v", rollbackErr) } } }() @@ -90,6 +95,7 @@ func (r *Repositories) WithReadTx(ctx context.Context, fn TxFunc) error { opts := &sql.TxOptions{ ReadOnly: true, } + tx, err := r.db.BeginTx(ctx, opts) if err != nil { return fmt.Errorf("beginning read transaction: %w", err) @@ -97,13 +103,16 @@ func (r *Repositories) WithReadTx(ctx context.Context, fn TxFunc) error { defer func() { if p := recover(); p != nil { - if rollbackErr := tx.Rollback(); rollbackErr != nil { - Fatal("failed to rollback transaction: %v", rollbackErr) + rollbackErr := tx.Rollback() + if rollbackErr != nil { + Fatalf("failed to rollback transaction: %v", rollbackErr) } + panic(p) } else if err != nil { - if rollbackErr := tx.Rollback(); rollbackErr != nil { - Fatal("failed to rollback transaction: %v", rollbackErr) + rollbackErr := tx.Rollback() + if rollbackErr != nil { + Fatalf("failed to rollback transaction: %v", rollbackErr) } } }() diff --git a/internal/database/repositories_test.go b/internal/database/repositories_test.go index 6324824..485ba61 100644 --- a/internal/database/repositories_test.go +++ b/internal/database/repositories_test.go @@ -3,7 +3,7 @@ package database import ( "context" "database/sql" - "fmt" + "errors" "testing" "time" @@ -28,7 +28,9 @@ func TestRepositoriesTransaction(t *testing.T) { UID: 1000, GID: 1000, } - if err := repos.Files.Create(ctx, tx, file); err != nil { + + err := repos.Files.Create(ctx, tx, file) + if err != nil { return err } @@ -37,7 +39,9 @@ func TestRepositoriesTransaction(t *testing.T) { ChunkHash: types.ChunkHash("tx_chunk1"), Size: 512, } - if err := repos.Chunks.Create(ctx, tx, chunk1); err != nil { + + err = repos.Chunks.Create(ctx, tx, chunk1) + if err != nil { return err } @@ -45,7 +49,9 @@ func TestRepositoriesTransaction(t *testing.T) { ChunkHash: types.ChunkHash("tx_chunk2"), Size: 512, } - if err := repos.Chunks.Create(ctx, tx, chunk2); err != nil { + + err = repos.Chunks.Create(ctx, tx, chunk2) + if err != nil { return err } @@ -55,7 +61,9 @@ func TestRepositoriesTransaction(t *testing.T) { Idx: 0, ChunkHash: chunk1.ChunkHash, } - if err := repos.FileChunks.Create(ctx, tx, fc1); err != nil { + + err = repos.FileChunks.Create(ctx, tx, fc1) + if err != nil { return err } @@ -64,7 +72,9 @@ func TestRepositoriesTransaction(t *testing.T) { Idx: 1, ChunkHash: chunk2.ChunkHash, } - if err := repos.FileChunks.Create(ctx, tx, fc2); err != nil { + + err = repos.FileChunks.Create(ctx, tx, fc2) + if err != nil { return err } @@ -74,7 +84,9 @@ func TestRepositoriesTransaction(t *testing.T) { Hash: types.BlobHash("tx_blob1"), CreatedTS: time.Now().Truncate(time.Second), } - if err := repos.Blobs.Create(ctx, tx, blob); err != nil { + + err = repos.Blobs.Create(ctx, tx, blob) + if err != nil { return err } @@ -85,7 +97,9 @@ func TestRepositoriesTransaction(t *testing.T) { Offset: 0, Length: 512, } - if err := repos.BlobChunks.Create(ctx, tx, bc1); err != nil { + + err = repos.BlobChunks.Create(ctx, tx, bc1) + if err != nil { return err } @@ -95,13 +109,14 @@ func TestRepositoriesTransaction(t *testing.T) { Offset: 512, Length: 512, } - if err := repos.BlobChunks.Create(ctx, tx, bc2); err != nil { + + err = repos.BlobChunks.Create(ctx, tx, bc2) + if err != nil { return err } return nil }) - if err != nil { t.Fatalf("transaction failed: %v", err) } @@ -111,6 +126,7 @@ func TestRepositoriesTransaction(t *testing.T) { if err != nil { t.Fatalf("failed to get file: %v", err) } + if file == nil { t.Error("expected file after transaction") } @@ -119,6 +135,7 @@ func TestRepositoriesTransaction(t *testing.T) { if err != nil { t.Fatalf("failed to get file chunks: %v", err) } + if len(chunks) != 2 { t.Errorf("expected 2 file chunks, got %d", len(chunks)) } @@ -127,6 +144,7 @@ func TestRepositoriesTransaction(t *testing.T) { if err != nil { t.Fatalf("failed to get blob: %v", err) } + if blob == nil { t.Error("expected blob after transaction") } @@ -150,7 +168,9 @@ func TestRepositoriesTransactionRollback(t *testing.T) { UID: 1000, GID: 1000, } - if err := repos.Files.Create(ctx, tx, file); err != nil { + + err := repos.Files.Create(ctx, tx, file) + if err != nil { return err } @@ -159,12 +179,14 @@ func TestRepositoriesTransactionRollback(t *testing.T) { ChunkHash: types.ChunkHash("rollback_chunk"), Size: 1024, } - if err := repos.Chunks.Create(ctx, tx, chunk); err != nil { + + err = repos.Chunks.Create(ctx, tx, chunk) + if err != nil { return err } // Return error to trigger rollback - return fmt.Errorf("intentional rollback") + return errors.New("intentional rollback") }) if err == nil || err.Error() != "intentional rollback" { @@ -176,6 +198,7 @@ func TestRepositoriesTransactionRollback(t *testing.T) { if err != nil { t.Fatalf("error checking for file: %v", err) } + if file != nil { t.Error("file should not exist after rollback") } @@ -184,6 +207,7 @@ func TestRepositoriesTransactionRollback(t *testing.T) { if err != nil { t.Fatalf("error checking for chunk: %v", err) } + if chunk != nil { t.Error("chunk should not exist after rollback") } @@ -205,6 +229,7 @@ func TestRepositoriesReadTransaction(t *testing.T) { UID: 1000, GID: 1000, } + err := repos.Files.Create(ctx, nil, file) if err != nil { t.Fatalf("failed to create file: %v", err) @@ -212,8 +237,10 @@ func TestRepositoriesReadTransaction(t *testing.T) { // Test read-only transaction var retrievedFile *File + err = repos.WithReadTx(ctx, func(ctx context.Context, tx *sql.Tx) error { var err error + retrievedFile, err = repos.Files.GetByPathTx(ctx, tx, "/test/read_file.txt") if err != nil { return err @@ -232,7 +259,6 @@ func TestRepositoriesReadTransaction(t *testing.T) { return nil }) - if err != nil { t.Fatalf("read transaction failed: %v", err) } diff --git a/internal/database/repository_comprehensive_test.go b/internal/database/repository_comprehensive_test.go index 384f80a..f3fce80 100644 --- a/internal/database/repository_comprehensive_test.go +++ b/internal/database/repository_comprehensive_test.go @@ -3,6 +3,7 @@ package database import ( "context" "database/sql" + "errors" "fmt" "testing" "time" @@ -39,6 +40,7 @@ func TestFileRepositoryUUIDGeneration(t *testing.T) { } uuids := make(map[string]bool) + for _, file := range files { err := repo.Create(ctx, nil, file) if err != nil { @@ -54,6 +56,7 @@ func TestFileRepositoryUUIDGeneration(t *testing.T) { if uuids[file.ID.String()] { t.Errorf("duplicate UUID generated: %s", file.ID) } + uuids[file.ID.String()] = true } } @@ -90,16 +93,19 @@ func TestFileRepositoryGetByID(t *testing.T) { if retrieved.ID != file.ID { t.Errorf("ID mismatch: expected %s, got %s", file.ID, retrieved.ID) } + if retrieved.Path != file.Path { t.Errorf("Path mismatch: expected %s, got %s", file.Path, retrieved.Path) } // Test non-existent ID nonExistentID := types.NewFileID() // Generate a new UUID that won't exist in the database + nonExistent, err := repo.GetByID(ctx, nonExistentID) if err != nil { t.Fatalf("GetByID should not return error for non-existent ID: %v", err) } + if nonExistent != nil { t.Error("expected nil for non-existent ID") } @@ -135,6 +141,7 @@ func TestOrphanedFileCleanup(t *testing.T) { if err != nil { t.Fatalf("failed to create file1: %v", err) } + err = repos.Files.Create(ctx, nil, file2) if err != nil { t.Fatalf("failed to create file2: %v", err) @@ -146,6 +153,7 @@ func TestOrphanedFileCleanup(t *testing.T) { Hostname: "test-host", StartedAt: time.Now(), } + err = repos.Snapshots.Create(ctx, nil, snapshot) if err != nil { t.Fatalf("failed to create snapshot: %v", err) @@ -168,6 +176,7 @@ func TestOrphanedFileCleanup(t *testing.T) { if err != nil { t.Fatalf("error getting file: %v", err) } + if orphanedFile != nil { t.Error("orphaned file should have been deleted") } @@ -177,6 +186,7 @@ func TestOrphanedFileCleanup(t *testing.T) { if err != nil { t.Fatalf("error getting file: %v", err) } + if referencedFile == nil { t.Error("referenced file should not have been deleted") } @@ -204,6 +214,7 @@ func TestOrphanedChunkCleanup(t *testing.T) { if err != nil { t.Fatalf("failed to create chunk1: %v", err) } + err = repos.Chunks.Create(ctx, nil, chunk2) if err != nil { t.Fatalf("failed to create chunk2: %v", err) @@ -218,6 +229,7 @@ func TestOrphanedChunkCleanup(t *testing.T) { UID: 1000, GID: 1000, } + err = repos.Files.Create(ctx, nil, file) if err != nil { t.Fatalf("failed to create file: %v", err) @@ -229,6 +241,7 @@ func TestOrphanedChunkCleanup(t *testing.T) { Idx: 0, ChunkHash: chunk2.ChunkHash, } + err = repos.FileChunks.Create(ctx, nil, fc) if err != nil { t.Fatalf("failed to create file chunk: %v", err) @@ -245,6 +258,7 @@ func TestOrphanedChunkCleanup(t *testing.T) { if err != nil { t.Fatalf("error getting chunk: %v", err) } + if orphanedChunk != nil { t.Error("orphaned chunk should have been deleted") } @@ -254,6 +268,7 @@ func TestOrphanedChunkCleanup(t *testing.T) { if err != nil { t.Fatalf("error getting chunk: %v", err) } + if referencedChunk == nil { t.Error("referenced chunk should not have been deleted") } @@ -283,6 +298,7 @@ func TestOrphanedBlobCleanup(t *testing.T) { if err != nil { t.Fatalf("failed to create blob1: %v", err) } + err = repos.Blobs.Create(ctx, nil, blob2) if err != nil { t.Fatalf("failed to create blob2: %v", err) @@ -294,6 +310,7 @@ func TestOrphanedBlobCleanup(t *testing.T) { Hostname: "test-host", StartedAt: time.Now(), } + err = repos.Snapshots.Create(ctx, nil, snapshot) if err != nil { t.Fatalf("failed to create snapshot: %v", err) @@ -316,6 +333,7 @@ func TestOrphanedBlobCleanup(t *testing.T) { if err != nil { t.Fatalf("error getting blob: %v", err) } + if orphanedBlob != nil { t.Error("orphaned blob should have been deleted") } @@ -325,6 +343,7 @@ func TestOrphanedBlobCleanup(t *testing.T) { if err != nil { t.Fatalf("error getting blob: %v", err) } + if referencedBlob == nil { t.Error("referenced blob should not have been deleted") } @@ -347,6 +366,7 @@ func TestFileChunkRepositoryWithUUIDs(t *testing.T) { UID: 1000, GID: 1000, } + err := repos.Files.Create(ctx, nil, file) if err != nil { t.Fatalf("failed to create file: %v", err) @@ -359,6 +379,7 @@ func TestFileChunkRepositoryWithUUIDs(t *testing.T) { ChunkHash: chunkHash, Size: 1024, } + err = repos.Chunks.Create(ctx, nil, chunk) if err != nil { t.Fatalf("failed to create chunk: %v", err) @@ -370,6 +391,7 @@ func TestFileChunkRepositoryWithUUIDs(t *testing.T) { Idx: i, ChunkHash: chunkHash, } + err = repos.FileChunks.Create(ctx, nil, fc) if err != nil { t.Fatalf("failed to create file chunk: %v", err) @@ -381,6 +403,7 @@ func TestFileChunkRepositoryWithUUIDs(t *testing.T) { if err != nil { t.Fatalf("failed to get file chunks: %v", err) } + if len(fileChunks) != 3 { t.Errorf("expected 3 chunks, got %d", len(fileChunks)) } @@ -395,6 +418,7 @@ func TestFileChunkRepositoryWithUUIDs(t *testing.T) { if err != nil { t.Fatalf("failed to get file chunks after delete: %v", err) } + if len(fileChunks) != 0 { t.Errorf("expected 0 chunks after delete, got %d", len(fileChunks)) } @@ -430,6 +454,7 @@ func TestChunkFileRepositoryWithUUIDs(t *testing.T) { if err != nil { t.Fatalf("failed to create file1: %v", err) } + err = repos.Files.Create(ctx, nil, file2) if err != nil { t.Fatalf("failed to create file2: %v", err) @@ -440,6 +465,7 @@ func TestChunkFileRepositoryWithUUIDs(t *testing.T) { ChunkHash: types.ChunkHash("shared-chunk"), Size: 1024, } + err = repos.Chunks.Create(ctx, nil, chunk) if err != nil { t.Fatalf("failed to create chunk: %v", err) @@ -463,6 +489,7 @@ func TestChunkFileRepositoryWithUUIDs(t *testing.T) { if err != nil { t.Fatalf("failed to create chunk file 1: %v", err) } + err = repos.ChunkFiles.Create(ctx, nil, cf2) if err != nil { t.Fatalf("failed to create chunk file 2: %v", err) @@ -473,6 +500,7 @@ func TestChunkFileRepositoryWithUUIDs(t *testing.T) { if err != nil { t.Fatalf("failed to get chunk files: %v", err) } + if len(chunkFiles) != 2 { t.Errorf("expected 2 files for chunk, got %d", len(chunkFiles)) } @@ -482,6 +510,7 @@ func TestChunkFileRepositoryWithUUIDs(t *testing.T) { if err != nil { t.Fatalf("failed to get chunks by file ID: %v", err) } + if len(chunkFiles) != 1 { t.Errorf("expected 1 chunk for file, got %d", len(chunkFiles)) } @@ -528,15 +557,19 @@ func TestSnapshotRepositoryExtendedFields(t *testing.T) { if retrieved.VaultikVersion != snapshot.VaultikVersion { t.Errorf("version mismatch: expected %s, got %s", snapshot.VaultikVersion, retrieved.VaultikVersion) } + if retrieved.VaultikGitRevision != snapshot.VaultikGitRevision { t.Errorf("git revision mismatch: expected %s, got %s", snapshot.VaultikGitRevision, retrieved.VaultikGitRevision) } + if retrieved.CompressionLevel != snapshot.CompressionLevel { t.Errorf("compression level mismatch: expected %d, got %d", snapshot.CompressionLevel, retrieved.CompressionLevel) } + if retrieved.BlobUncompressedSize != snapshot.BlobUncompressedSize { t.Errorf("uncompressed size mismatch: expected %d, got %d", snapshot.BlobUncompressedSize, retrieved.BlobUncompressedSize) } + if retrieved.UploadDurationMs != snapshot.UploadDurationMs { t.Errorf("upload duration mismatch: expected %d, got %d", snapshot.UploadDurationMs, retrieved.UploadDurationMs) } @@ -566,6 +599,7 @@ func TestComplexOrphanedDataScenario(t *testing.T) { if err != nil { t.Fatalf("failed to create snapshot1: %v", err) } + err = repos.Snapshots.Create(ctx, nil, snapshot2) if err != nil { t.Fatalf("failed to create snapshot2: %v", err) @@ -582,6 +616,7 @@ func TestComplexOrphanedDataScenario(t *testing.T) { UID: 1000, GID: 1000, } + err = repos.Files.Create(ctx, nil, files[i]) if err != nil { t.Fatalf("failed to create file%d: %v", i, err) @@ -598,14 +633,17 @@ func TestComplexOrphanedDataScenario(t *testing.T) { if err != nil { t.Fatal(err) } + err = repos.Snapshots.AddFileByID(ctx, nil, snapshot1.ID.String(), files[1].ID) if err != nil { t.Fatal(err) } + err = repos.Snapshots.AddFileByID(ctx, nil, snapshot2.ID.String(), files[1].ID) if err != nil { t.Fatal(err) } + err = repos.Snapshots.AddFileByID(ctx, nil, snapshot2.ID.String(), files[2].ID) if err != nil { t.Fatal(err) @@ -616,6 +654,7 @@ func TestComplexOrphanedDataScenario(t *testing.T) { if err != nil { t.Fatal(err) } + err = repos.Snapshots.Delete(ctx, snapshot1.ID.String()) if err != nil { t.Fatal(err) @@ -633,6 +672,7 @@ func TestComplexOrphanedDataScenario(t *testing.T) { if err != nil { t.Fatalf("error getting file0: %v", err) } + if file0 != nil { t.Error("file0 should have been deleted") } @@ -642,6 +682,7 @@ func TestComplexOrphanedDataScenario(t *testing.T) { if err != nil { t.Fatalf("error getting file1: %v", err) } + if file1 == nil { t.Error("file1 should still exist") } @@ -651,6 +692,7 @@ func TestComplexOrphanedDataScenario(t *testing.T) { if err != nil { t.Fatalf("error getting file2: %v", err) } + if file2 == nil { t.Error("file2 should still exist") } @@ -673,17 +715,19 @@ func TestCascadeDelete(t *testing.T) { UID: 1000, GID: 1000, } + err := repos.Files.Create(ctx, nil, file) if err != nil { t.Fatalf("failed to create file: %v", err) } // Create chunks and file-chunk mappings - for i := 0; i < 3; i++ { + for i := range 3 { chunk := &Chunk{ ChunkHash: types.ChunkHash(fmt.Sprintf("cascade-chunk-%d", i)), Size: 1024, } + err = repos.Chunks.Create(ctx, nil, chunk) if err != nil { t.Fatalf("failed to create chunk: %v", err) @@ -694,6 +738,7 @@ func TestCascadeDelete(t *testing.T) { Idx: i, ChunkHash: chunk.ChunkHash, } + err = repos.FileChunks.Create(ctx, nil, fc) if err != nil { t.Fatalf("failed to create file chunk: %v", err) @@ -705,6 +750,7 @@ func TestCascadeDelete(t *testing.T) { if err != nil { t.Fatal(err) } + if len(fileChunks) != 3 { t.Errorf("expected 3 file chunks, got %d", len(fileChunks)) } @@ -720,6 +766,7 @@ func TestCascadeDelete(t *testing.T) { if err != nil { t.Fatal(err) } + if len(fileChunks) != 0 { t.Errorf("expected 0 file chunks after cascade delete, got %d", len(fileChunks)) } @@ -744,6 +791,7 @@ func TestTransactionIsolation(t *testing.T) { UID: 1000, GID: 1000, } + err := repos.Files.Create(ctx, tx, file) if err != nil { return err @@ -754,9 +802,8 @@ func TestTransactionIsolation(t *testing.T) { // For now, we'll just test that rollback works // Return an error to trigger rollback - return fmt.Errorf("intentional rollback") + return errors.New("intentional rollback") }) - if err == nil { t.Fatal("expected error from transaction") } @@ -766,6 +813,7 @@ func TestTransactionIsolation(t *testing.T) { if err != nil { t.Fatal(err) } + if len(files) != 0 { t.Error("file should not exist after rollback") } @@ -790,13 +838,14 @@ func TestConcurrentOrphanedCleanup(t *testing.T) { Hostname: "test-host", StartedAt: time.Now(), } + err := repos.Snapshots.Create(ctx, nil, snapshot) if err != nil { t.Fatal(err) } // Create many files, some orphaned - for i := 0; i < 20; i++ { + for i := range 20 { file := &File{ Path: types.FilePath(fmt.Sprintf("/concurrent-%d.txt", i)), MTime: time.Now().Truncate(time.Second), @@ -805,6 +854,7 @@ func TestConcurrentOrphanedCleanup(t *testing.T) { UID: 1000, GID: 1000, } + err = repos.Files.Create(ctx, nil, file) if err != nil { t.Fatal(err) @@ -822,14 +872,15 @@ func TestConcurrentOrphanedCleanup(t *testing.T) { // Run multiple cleanup operations concurrently // Note: SQLite has limited support for concurrent writes, so we expect some to fail done := make(chan error, 3) - for i := 0; i < 3; i++ { + + for range 3 { go func() { done <- repos.Files.DeleteOrphaned(ctx) }() } // Wait for all to complete - for i := 0; i < 3; i++ { + for i := range 3 { err := <-done if err != nil { t.Errorf("cleanup %d failed: %v", i, err) @@ -850,10 +901,12 @@ func TestConcurrentOrphanedCleanup(t *testing.T) { // Verify all remaining files are even-numbered for _, file := range files { var num int + _, err := fmt.Sscanf(file.Path.String(), "/concurrent-%d.txt", &num) if err != nil { t.Logf("failed to parse file number from %s: %v", file.Path, err) } + if num%2 != 0 { t.Errorf("odd-numbered file %s should have been deleted", file.Path) } diff --git a/internal/database/repository_debug_test.go b/internal/database/repository_debug_test.go index 2bd9493..9deedaa 100644 --- a/internal/database/repository_debug_test.go +++ b/internal/database/repository_debug_test.go @@ -36,12 +36,14 @@ func TestOrphanedFileCleanupDebug(t *testing.T) { if err != nil { t.Fatalf("failed to create file1: %v", err) } + t.Logf("Created file1 with ID: %s", file1.ID) err = repos.Files.Create(ctx, nil, file2) if err != nil { t.Fatalf("failed to create file2: %v", err) } + t.Logf("Created file2 with ID: %s", file2.ID) // Create a snapshot and reference only file2 @@ -50,18 +52,22 @@ func TestOrphanedFileCleanupDebug(t *testing.T) { Hostname: "test-host", StartedAt: time.Now(), } + err = repos.Snapshots.Create(ctx, nil, snapshot) if err != nil { t.Fatalf("failed to create snapshot: %v", err) } + t.Logf("Created snapshot: %s", snapshot.ID) // Check snapshot_files before adding var count int + err = db.conn.QueryRow("SELECT COUNT(*) FROM snapshot_files").Scan(&count) if err != nil { t.Fatal(err) } + t.Logf("snapshot_files count before add: %d", count) // Add file2 to snapshot @@ -69,6 +75,7 @@ func TestOrphanedFileCleanupDebug(t *testing.T) { if err != nil { t.Fatalf("failed to add file to snapshot: %v", err) } + t.Logf("Added file2 to snapshot") // Check snapshot_files after adding @@ -76,6 +83,7 @@ func TestOrphanedFileCleanupDebug(t *testing.T) { if err != nil { t.Fatal(err) } + t.Logf("snapshot_files count after add: %d", count) // Check which files are referenced @@ -84,16 +92,22 @@ func TestOrphanedFileCleanupDebug(t *testing.T) { t.Fatal(err) } defer func() { - if err := rows.Close(); err != nil { + err := rows.Close() + if err != nil { t.Logf("failed to close rows: %v", err) } }() + t.Log("Files in snapshot_files:") + for rows.Next() { var fileID string - if err := rows.Scan(&fileID); err != nil { + + err := rows.Scan(&fileID) + if err != nil { t.Fatal(err) } + t.Logf(" - %s", fileID) } @@ -102,6 +116,7 @@ func TestOrphanedFileCleanupDebug(t *testing.T) { if err != nil { t.Fatal(err) } + t.Logf("Files count before cleanup: %d", count) // Run orphaned cleanup @@ -109,6 +124,7 @@ func TestOrphanedFileCleanupDebug(t *testing.T) { if err != nil { t.Fatalf("failed to delete orphaned files: %v", err) } + t.Log("Ran orphaned cleanup") // Check files after cleanup @@ -116,6 +132,7 @@ func TestOrphanedFileCleanupDebug(t *testing.T) { if err != nil { t.Fatal(err) } + t.Logf("Files count after cleanup: %d", count) // List remaining files @@ -123,7 +140,9 @@ func TestOrphanedFileCleanupDebug(t *testing.T) { if err != nil { t.Fatal(err) } + t.Log("Remaining files:") + for _, f := range files { t.Logf(" - ID: %s, Path: %s", f.ID, f.Path) } @@ -133,10 +152,12 @@ func TestOrphanedFileCleanupDebug(t *testing.T) { if err != nil { t.Fatalf("error getting file: %v", err) } + if orphanedFile != nil { t.Error("orphaned file should have been deleted") // Let's check why it wasn't deleted var exists bool + err = db.conn.QueryRow(` SELECT EXISTS( SELECT 1 FROM snapshot_files @@ -145,6 +166,7 @@ func TestOrphanedFileCleanupDebug(t *testing.T) { if err != nil { t.Fatal(err) } + t.Logf("File1 exists in snapshot_files: %v", exists) } else { t.Log("Orphaned file was correctly deleted") @@ -155,6 +177,7 @@ func TestOrphanedFileCleanupDebug(t *testing.T) { if err != nil { t.Fatalf("error getting file: %v", err) } + if referencedFile == nil { t.Error("referenced file should not have been deleted") } else { diff --git a/internal/database/repository_edge_cases_test.go b/internal/database/repository_edge_cases_test.go index b4ba6ba..666f33f 100644 --- a/internal/database/repository_edge_cases_test.go +++ b/internal/database/repository_edge_cases_test.go @@ -98,6 +98,7 @@ func TestFileRepositoryEdgeCases(t *testing.T) { if (err != nil) != tt.wantErr { t.Errorf("Create() error = %v, wantErr %v", err, tt.wantErr) } + if err != nil && tt.errMsg != "" && !strings.Contains(err.Error(), tt.errMsg) { t.Errorf("Create() error = %v, want error containing %q", err, tt.errMsg) } @@ -136,6 +137,7 @@ func TestDuplicateHandling(t *testing.T) { if err != nil { t.Fatalf("failed to create file1: %v", err) } + originalID := file1.ID // Create with same path should update the existing record (UPSERT behavior) @@ -190,6 +192,7 @@ func TestDuplicateHandling(t *testing.T) { UID: 1000, GID: 1000, } + err := repos.Files.Create(ctx, nil, file) if err != nil { t.Fatal(err) @@ -199,6 +202,7 @@ func TestDuplicateHandling(t *testing.T) { ChunkHash: types.ChunkHash("test-chunk-dup"), Size: 1024, } + err = repos.Chunks.Create(ctx, nil, chunk) if err != nil { t.Fatal(err) @@ -325,6 +329,7 @@ func TestLargeDatasets(t *testing.T) { Hostname: "test-host", StartedAt: time.Now(), } + err := repos.Snapshots.Create(ctx, nil, snapshot) if err != nil { t.Fatal(err) @@ -332,11 +337,12 @@ func TestLargeDatasets(t *testing.T) { // Create many files const fileCount = 1000 + fileIDs := make([]types.FileID, fileCount) t.Run("create many files", func(t *testing.T) { start := time.Now() - for i := 0; i < fileCount; i++ { + for i := range fileCount { file := &File{ Path: types.FilePath(fmt.Sprintf("/large/file%05d.txt", i)), MTime: time.Now(), @@ -345,10 +351,12 @@ func TestLargeDatasets(t *testing.T) { UID: uint32(1000 + (i % 10)), GID: uint32(1000 + (i % 10)), } + err := repos.Files.Create(ctx, nil, file) if err != nil { t.Fatalf("failed to create file %d: %v", i, err) } + fileIDs[i] = file.ID // Add half to snapshot @@ -359,29 +367,35 @@ func TestLargeDatasets(t *testing.T) { } } } + t.Logf("Created %d files in %v", fileCount, time.Since(start)) }) // Test ListByPrefix performance t.Run("list by prefix performance", func(t *testing.T) { start := time.Now() + files, err := repos.Files.ListByPrefix(ctx, "/large/") if err != nil { t.Fatal(err) } + if len(files) != fileCount { t.Errorf("expected %d files, got %d", fileCount, len(files)) } + t.Logf("Listed %d files in %v", len(files), time.Since(start)) }) // Test orphaned cleanup performance t.Run("orphaned cleanup performance", func(t *testing.T) { start := time.Now() + err := repos.Files.DeleteOrphaned(ctx) if err != nil { t.Fatal(err) } + t.Logf("Cleaned up orphaned files in %v", time.Since(start)) // Verify correct number remain @@ -389,6 +403,7 @@ func TestLargeDatasets(t *testing.T) { if err != nil { t.Fatal(err) } + if len(files) != fileCount/2 { t.Errorf("expected %d files after cleanup, got %d", fileCount/2, len(files)) } @@ -409,6 +424,7 @@ func TestErrorPropagation(t *testing.T) { if err != nil { t.Errorf("GetByID should not return error for non-existent ID, got: %v", err) } + if file != nil { t.Error("expected nil file for non-existent ID") } @@ -420,6 +436,7 @@ func TestErrorPropagation(t *testing.T) { if err != nil { t.Errorf("GetByPath should not return error for non-existent path, got: %v", err) } + if file != nil { t.Error("expected nil file for non-existent path") } @@ -432,10 +449,12 @@ func TestErrorPropagation(t *testing.T) { Idx: 0, ChunkHash: types.ChunkHash("some-chunk"), } + err := repos.FileChunks.Create(ctx, nil, fc) if err == nil { t.Error("expected error for invalid foreign key") } + if !strings.Contains(err.Error(), "FOREIGN KEY") { t.Errorf("expected foreign key error, got: %v", err) } @@ -475,6 +494,7 @@ func TestQueryInjection(t *testing.T) { // Verify tables still exist var count int + err := db.conn.QueryRow("SELECT COUNT(*) FROM files").Scan(&count) if err != nil { t.Fatal("files table was damaged by injection") diff --git a/internal/database/snapshots.go b/internal/database/snapshots.go index 93cc517..7b620a2 100644 --- a/internal/database/snapshots.go +++ b/internal/database/snapshots.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "fmt" + "strings" "time" "sneak.berlin/go/vaultik/internal/types" @@ -26,6 +27,7 @@ func (r *SnapshotRepository) Create(ctx context.Context, tx *sql.Tx, snapshot *S ` var completedAt *int64 + if snapshot.CompletedAt != nil { ts := snapshot.CompletedAt.Unix() completedAt = &ts @@ -84,9 +86,11 @@ func (r *SnapshotRepository) UpdateCounts(ctx context.Context, tx *sql.Tx, snaps func (r *SnapshotRepository) UpdateExtendedStats(ctx context.Context, tx *sql.Tx, snapshotID string, blobUncompressedSize int64, compressionLevel int, uploadDurationMs int64) error { // Calculate compression ratio based on uncompressed vs compressed sizes var compressionRatio float64 + if blobUncompressedSize > 0 { // Get current blob_size from DB to calculate ratio var blobSize int64 + queryGet := `SELECT blob_size FROM snapshots WHERE id = ?` if tx != nil { err := tx.QueryRowContext(ctx, queryGet, snapshotID).Scan(&blobSize) @@ -99,6 +103,7 @@ func (r *SnapshotRepository) UpdateExtendedStats(ctx context.Context, tx *sql.Tx return fmt.Errorf("getting blob size: %w", err) } } + compressionRatio = float64(blobSize) / float64(blobUncompressedSize) } else { compressionRatio = 1.0 @@ -124,6 +129,7 @@ func (r *SnapshotRepository) UpdateExtendedStats(ctx context.Context, tx *sql.Tx if err != nil { return fmt.Errorf("updating extended stats: %w", err) } + return nil } @@ -136,9 +142,11 @@ func (r *SnapshotRepository) GetByID(ctx context.Context, snapshotID string) (*S WHERE id = ? ` - var snapshot Snapshot - var startedAtUnix int64 - var completedAtUnix *int64 + var ( + snapshot Snapshot + startedAtUnix int64 + completedAtUnix *int64 + ) err := r.db.conn.QueryRowContext(ctx, query, snapshotID).Scan( &snapshot.ID, @@ -162,6 +170,7 @@ func (r *SnapshotRepository) GetByID(ctx context.Context, snapshotID string) (*S if err == sql.ErrNoRows { return nil, nil } + if err != nil { return nil, fmt.Errorf("querying snapshot: %w", err) } @@ -190,10 +199,13 @@ func (r *SnapshotRepository) ListRecent(ctx context.Context, limit int) ([]*Snap defer CloseRows(rows) var snapshots []*Snapshot + for rows.Next() { - var snapshot Snapshot - var startedAtUnix int64 - var completedAtUnix *int64 + var ( + snapshot Snapshot + startedAtUnix int64 + completedAtUnix *int64 + ) err := rows.Scan( &snapshot.ID, @@ -301,28 +313,35 @@ func (r *SnapshotRepository) AddFilesByIDBatch(ctx context.Context, tx *sql.Tx, const batchSize = 400 for i := 0; i < len(fileIDs); i += batchSize { - end := i + batchSize - if end > len(fileIDs) { - end = len(fileIDs) - } + end := min(i+batchSize, len(fileIDs)) + batch := fileIDs[i:end] query := "INSERT OR IGNORE INTO snapshot_files (snapshot_id, file_id) VALUES " - args := make([]interface{}, 0, len(batch)*2) + + args := make([]any, 0, len(batch)*2) + + var querySb312 strings.Builder + for j, fileID := range batch { if j > 0 { - query += ", " + querySb312.WriteString(", ") } - query += "(?, ?)" + + querySb312.WriteString("(?, ?)") + args = append(args, snapshotID, fileID.String()) } + query += querySb312.String() + var err error if tx != nil { _, err = tx.ExecContext(ctx, query, args...) } else { _, err = r.db.ExecWithLog(ctx, query, args...) } + if err != nil { return fmt.Errorf("batch adding files to snapshot: %w", err) } @@ -353,18 +372,22 @@ func (r *SnapshotRepository) PopulateReferencedBlobs(ctx context.Context, tx *sq AND blobs.blob_hash IS NOT NULL ` - var result sql.Result - var err error + var ( + result sql.Result + err error + ) if tx != nil { result, err = tx.ExecContext(ctx, query, snapshotID, snapshotID) } else { result, err = r.db.ExecWithLog(ctx, query, snapshotID, snapshotID) } + if err != nil { return 0, fmt.Errorf("populating referenced blobs: %w", err) } n, _ := result.RowsAffected() + return n, nil } @@ -405,11 +428,15 @@ func (r *SnapshotRepository) GetBlobHashes(ctx context.Context, snapshotID strin defer CloseRows(rows) var blobs []string + for rows.Next() { var blobHash string - if err := rows.Scan(&blobHash); err != nil { + + err := rows.Scan(&blobHash) + if err != nil { return nil, fmt.Errorf("scanning blob hash: %w", err) } + blobs = append(blobs, blobHash) } @@ -426,6 +453,7 @@ func (r *SnapshotRepository) GetSnapshotTotalCompressedSize(ctx context.Context, ` var totalSize int64 + err := r.db.conn.QueryRowContext(ctx, query, snapshotID).Scan(&totalSize) if err != nil { return 0, fmt.Errorf("querying total compressed size: %w", err) @@ -449,6 +477,7 @@ func (r *SnapshotRepository) GetSnapshotUncompressedChunkSize(ctx context.Contex ` var totalSize int64 + err := r.db.conn.QueryRowContext(ctx, query, snapshotID).Scan(&totalSize) if err != nil { return 0, fmt.Errorf("querying uncompressed chunk size: %w", err) @@ -485,6 +514,7 @@ func (r *SnapshotRepository) GetSnapshotNewChunkSize(ctx context.Context, snapsh ` var totalSize int64 + err := r.db.conn.QueryRowContext(ctx, query, snapshotID, snapshotID, snapshotID).Scan(&totalSize) if err != nil { return 0, fmt.Errorf("querying new chunk size: %w", err) @@ -509,10 +539,13 @@ func (r *SnapshotRepository) GetIncompleteSnapshots(ctx context.Context) ([]*Sna defer CloseRows(rows) var snapshots []*Snapshot + for rows.Next() { - var snapshot Snapshot - var startedAtUnix int64 - var completedAtUnix *int64 + var ( + snapshot Snapshot + startedAtUnix int64 + completedAtUnix *int64 + ) err := rows.Scan( &snapshot.ID, @@ -560,10 +593,13 @@ func (r *SnapshotRepository) GetIncompleteByHostname(ctx context.Context, hostna defer CloseRows(rows) var snapshots []*Snapshot + for rows.Next() { - var snapshot Snapshot - var startedAtUnix int64 - var completedAtUnix *int64 + var ( + snapshot Snapshot + startedAtUnix int64 + completedAtUnix *int64 + ) err := rows.Scan( &snapshot.ID, diff --git a/internal/database/snapshots_test.go b/internal/database/snapshots_test.go index c856163..a1efaa1 100644 --- a/internal/database/snapshots_test.go +++ b/internal/database/snapshots_test.go @@ -52,15 +52,19 @@ func TestSnapshotRepository(t *testing.T) { if err != nil { t.Fatalf("failed to get snapshot: %v", err) } + if retrieved == nil { t.Fatal("expected snapshot, got nil") } + if retrieved.ID != snapshot.ID { t.Errorf("ID mismatch: got %s, want %s", retrieved.ID, snapshot.ID) } + if retrieved.Hostname != snapshot.Hostname { t.Errorf("hostname mismatch: got %s, want %s", retrieved.Hostname, snapshot.Hostname) } + if retrieved.FileCount != snapshot.FileCount { t.Errorf("file count mismatch: got %d, want %d", retrieved.FileCount, snapshot.FileCount) } @@ -75,21 +79,27 @@ func TestSnapshotRepository(t *testing.T) { if err != nil { t.Fatalf("failed to get updated snapshot: %v", err) } + if retrieved.FileCount != 200 { t.Errorf("file count not updated: got %d, want %d", retrieved.FileCount, 200) } + if retrieved.ChunkCount != 1000 { t.Errorf("chunk count not updated: got %d, want %d", retrieved.ChunkCount, 1000) } + if retrieved.BlobCount != 20 { t.Errorf("blob count not updated: got %d, want %d", retrieved.BlobCount, 20) } + if retrieved.TotalSize != twoHundredMebibytes { t.Errorf("total size not updated: got %d, want %d", retrieved.TotalSize, twoHundredMebibytes) } + if retrieved.BlobSize != sixtyMebibytes { t.Errorf("blob size not updated: got %d, want %d", retrieved.BlobSize, sixtyMebibytes) } + expectedRatio := compressionRatioPoint3 // 0.3 if math.Abs(retrieved.CompressionRatio-expectedRatio) > 0.001 { t.Errorf("compression ratio not updated: got %f, want %f", retrieved.CompressionRatio, expectedRatio) @@ -108,6 +118,7 @@ func TestSnapshotRepository(t *testing.T) { ChunkCount: int64(500 * i), BlobCount: int64(10 * i), } + err := repo.Create(ctx, nil, s) if err != nil { t.Fatalf("failed to create snapshot %d: %v", i, err) @@ -119,12 +130,13 @@ func TestSnapshotRepository(t *testing.T) { if err != nil { t.Fatalf("failed to list recent snapshots: %v", err) } + if len(recent) != 3 { t.Errorf("expected 3 recent snapshots, got %d", len(recent)) } // Verify order (most recent first) - for i := 0; i < len(recent)-1; i++ { + for i := range len(recent) - 1 { if recent[i].StartedAt.Before(recent[i+1].StartedAt) { t.Error("snapshots not in descending order") } @@ -143,6 +155,7 @@ func TestSnapshotRepositoryNotFound(t *testing.T) { if err != nil { t.Fatalf("unexpected error: %v", err) } + if snapshot != nil { t.Error("expected nil for non-existent snapshot") } diff --git a/internal/database/uploads.go b/internal/database/uploads.go index c1a3c09..154cd09 100644 --- a/internal/database/uploads.go +++ b/internal/database/uploads.go @@ -53,6 +53,7 @@ func (r *UploadRepository) GetByBlobHash(ctx context.Context, blobHash string) ( ` var upload Upload + err := r.conn.QueryRowContext(ctx, query, blobHash).Scan( &upload.BlobHash, &upload.UploadedAt, @@ -63,6 +64,7 @@ func (r *UploadRepository) GetByBlobHash(ctx context.Context, blobHash string) ( if err == sql.ErrNoRows { return nil, nil } + if err != nil { return nil, err } @@ -84,17 +86,22 @@ func (r *UploadRepository) GetRecentUploads(ctx context.Context, limit int) ([]* return nil, err } defer func() { - if err := rows.Close(); err != nil { + err := rows.Close() + if err != nil { log.Error("failed to close rows", "error", err) } }() var uploads []*Upload + for rows.Next() { var upload Upload - if err := rows.Scan(&upload.BlobHash, &upload.UploadedAt, &upload.Size, &upload.DurationMs); err != nil { + + err := rows.Scan(&upload.BlobHash, &upload.UploadedAt, &upload.Size, &upload.DurationMs) + if err != nil { return nil, err } + uploads = append(uploads, &upload) } @@ -115,6 +122,7 @@ func (r *UploadRepository) GetUploadStats(ctx context.Context, since time.Time) ` var stats UploadStats + err := r.conn.QueryRowContext(ctx, query, since).Scan( &stats.Count, &stats.TotalSize, @@ -138,10 +146,13 @@ type UploadStats struct { // GetCountBySnapshot returns the count of uploads for a specific snapshot func (r *UploadRepository) GetCountBySnapshot(ctx context.Context, snapshotID string) (int64, error) { query := `SELECT COUNT(*) FROM uploads WHERE snapshot_id = ?` + var count int64 + err := r.conn.QueryRowContext(ctx, query, snapshotID).Scan(&count) if err != nil { return 0, err } + return count, nil } diff --git a/internal/globals/globals.go b/internal/globals/globals.go index 80c538b..21edab6 100644 --- a/internal/globals/globals.go +++ b/internal/globals/globals.go @@ -50,5 +50,6 @@ func (g *Globals) ShortCommit() string { if len(g.Commit) > 12 { return g.Commit[:12] } + return g.Commit } diff --git a/internal/log/log.go b/internal/log/log.go index 2843d25..c9867ee 100644 --- a/internal/log/log.go +++ b/internal/log/log.go @@ -84,6 +84,7 @@ func getCaller(skip int) string { if !ok { return "unknown" } + return fmt.Sprintf("%s:%d", filepath.Base(file), line) } @@ -94,6 +95,7 @@ func Fatal(msg string, args ...any) { args = append(args, "caller", getCaller(2)) logger.Error(msg, args...) } + os.Exit(1) } @@ -172,6 +174,7 @@ func With(args ...any) *slog.Logger { if logger != nil { return logger.With(args...) } + return slog.Default() } diff --git a/internal/log/tty_handler.go b/internal/log/tty_handler.go index 91ab803..81c1fe6 100644 --- a/internal/log/tty_handler.go +++ b/internal/log/tty_handler.go @@ -33,6 +33,7 @@ func NewTTYHandler(out io.Writer, opts *slog.HandlerOptions) *TTYHandler { if opts == nil { opts = &slog.HandlerOptions{} } + return &TTYHandler{ out: out, opts: *opts, @@ -54,7 +55,9 @@ func (h *TTYHandler) Handle(_ context.Context, r slog.Record) error { // Level and color level := r.Level.String() + var levelColor string + switch r.Level { case slog.LevelDebug: levelColor = colorGray @@ -96,10 +99,12 @@ func (h *TTYHandler) Handle(_ context.Context, r slog.Record) error { _, _ = fmt.Fprintf(h.out, " %s%s%s=%s%s%s", colorCyan, a.Key, colorReset, colorBlue, value, colorReset) + return true }) _, _ = fmt.Fprintln(h.out) + return nil } @@ -122,6 +127,7 @@ func formatDuration(d time.Duration) string { } else if d < time.Minute { return fmt.Sprintf("%.1fs", d.Seconds()) } + return d.String() } @@ -131,10 +137,12 @@ func formatBytes(b int64) string { if b < unit { return fmt.Sprintf("%d B", b) } + div, exp := int64(unit), 0 for n := b / unit; n >= unit; n /= unit { div *= unit exp++ } + return fmt.Sprintf("%.1f %cB", float64(b)/float64(div), "KMGTPE"[exp]) } diff --git a/internal/pidlock/pidlock.go b/internal/pidlock/pidlock.go index dfe0306..e576970 100644 --- a/internal/pidlock/pidlock.go +++ b/internal/pidlock/pidlock.go @@ -77,6 +77,7 @@ func (l *Lock) Release() error { } l.path = "" // Prevent double-release + return nil } @@ -104,5 +105,6 @@ func isProcessRunning(pid int) bool { // On Unix, FindProcess always succeeds. We need to send signal 0 to check. err = process.Signal(syscall.Signal(0)) + return err == nil } diff --git a/internal/pidlock/pidlock_test.go b/internal/pidlock/pidlock_test.go index d256ee1..dce224d 100644 --- a/internal/pidlock/pidlock_test.go +++ b/internal/pidlock/pidlock_test.go @@ -40,6 +40,7 @@ func TestAcquireBlocksSecondInstance(t *testing.T) { // Acquire first lock lock1, err := Acquire(tmpDir) require.NoError(t, err) + require.NotNil(t, lock1) defer func() { _ = lock1.Release() }() @@ -61,6 +62,7 @@ func TestAcquireWithStaleLock(t *testing.T) { // Should be able to acquire lock (stale lock is cleaned up) lock, err := Acquire(tmpDir) require.NoError(t, err) + require.NotNil(t, lock) defer func() { _ = lock.Release() }() @@ -88,6 +90,7 @@ func TestReleaseIsIdempotent(t *testing.T) { func TestReleaseNilLock(t *testing.T) { var lock *Lock + err := lock.Release() assert.NoError(t, err) } @@ -98,6 +101,7 @@ func TestAcquireCreatesDirectory(t *testing.T) { lock, err := Acquire(nestedDir) require.NoError(t, err) + require.NotNil(t, lock) defer func() { _ = lock.Release() }() diff --git a/internal/s3/client.go b/internal/s3/client.go index 2861be7..1079b8f 100644 --- a/internal/s3/client.go +++ b/internal/s3/client.go @@ -42,7 +42,7 @@ type Config struct { // Used to suppress SDK warnings about checksums. type nopLogger struct{} -func (nopLogger) Logf(classification logging.Classification, format string, v ...interface{}) {} +func (nopLogger) Logf(classification logging.Classification, format string, v ...any) {} // NewClient creates a new S3 client with the provided configuration. // It establishes a connection to the S3-compatible storage service and @@ -92,6 +92,7 @@ func (c *Client) PutObject(ctx context.Context, key string, data io.Reader) erro Key: aws.String(fullKey), Body: data, }) + return err } @@ -137,6 +138,7 @@ func (c *Client) PutObjectWithProgress(ctx context.Context, key string, data io. // close the returned reader when done to avoid resource leaks. func (c *Client) GetObject(ctx context.Context, key string) (io.ReadCloser, error) { fullKey := c.prefix + key + result, err := c.s3Client.GetObject(ctx, &s3.GetObjectInput{ Bucket: aws.String(c.bucket), Key: aws.String(fullKey), @@ -144,6 +146,7 @@ func (c *Client) GetObject(ctx context.Context, key string) (io.ReadCloser, erro if err != nil { return nil, err } + return result.Body, nil } @@ -156,6 +159,7 @@ func (c *Client) DeleteObject(ctx context.Context, key string) error { Bucket: aws.String(c.bucket), Key: aws.String(fullKey), }) + return err } @@ -168,6 +172,7 @@ func (c *Client) ListObjects(ctx context.Context, prefix string) ([]string, erro fullPrefix := c.prefix + prefix var keys []string + paginator := s3.NewListObjectsV2Paginator(c.s3Client, &s3.ListObjectsV2Input{ Bucket: aws.String(c.bucket), Prefix: aws.String(fullPrefix), @@ -186,6 +191,7 @@ func (c *Client) ListObjects(ctx context.Context, prefix string) ([]string, erro if len(key) > len(c.prefix) { key = key[len(c.prefix):] } + keys = append(keys, key) } } @@ -200,18 +206,23 @@ func (c *Client) ListObjects(ctx context.Context, prefix string) ([]string, erro // Note: This method returns false for any error, not just "not found". func (c *Client) HeadObject(ctx context.Context, key string) (bool, error) { fullKey := c.prefix + key + _, err := c.s3Client.HeadObject(ctx, &s3.HeadObjectInput{ Bucket: aws.String(c.bucket), Key: aws.String(fullKey), }) if err != nil { - var notFound *s3types.NotFound - var noSuchKey *s3types.NoSuchKey + var ( + notFound *s3types.NotFound + noSuchKey *s3types.NoSuchKey + ) if errors.As(err, ¬Found) || errors.As(err, &noSuchKey) { return false, nil } + return false, err } + return true, nil } @@ -247,6 +258,7 @@ func (c *Client) ListObjectsStream(ctx context.Context, prefix string, recursive page, err := paginator.NextPage(ctx) if err != nil { ch <- ObjectInfo{Err: err} + return } @@ -257,6 +269,7 @@ func (c *Client) ListObjectsStream(ctx context.Context, prefix string, recursive if len(key) > len(c.prefix) { key = key[len(c.prefix):] } + ch <- ObjectInfo{ Key: key, Size: *obj.Size, @@ -275,6 +288,7 @@ func (c *Client) ListObjectsStream(ctx context.Context, prefix string, recursive // Returns an error if the object doesn't exist or if the operation fails. func (c *Client) StatObject(ctx context.Context, key string) (*ObjectInfo, error) { fullKey := c.prefix + key + result, err := c.s3Client.HeadObject(ctx, &s3.HeadObjectInput{ Bucket: aws.String(c.bucket), Key: aws.String(fullKey), @@ -313,6 +327,7 @@ func (c *Client) Endpoint() string { if c.endpoint == "" { return "s3.amazonaws.com" } + return c.endpoint } @@ -329,11 +344,14 @@ func (pr *progressReader) Read(p []byte) (int, error) { n, err := pr.reader.Read(p) if n > 0 { atomic.AddInt64(&pr.read, int64(n)) + if pr.callback != nil { - if callbackErr := pr.callback(atomic.LoadInt64(&pr.read)); callbackErr != nil { + callbackErr := pr.callback(atomic.LoadInt64(&pr.read)) + if callbackErr != nil { return n, callbackErr } } } + return n, err } diff --git a/internal/s3/client_test.go b/internal/s3/client_test.go index 1bb267e..7ebab27 100644 --- a/internal/s3/client_test.go +++ b/internal/s3/client_test.go @@ -12,7 +12,8 @@ import ( func TestClient(t *testing.T) { ts := NewTestServer(t) defer func() { - if err := ts.Cleanup(); err != nil { + err := ts.Cleanup() + if err != nil { t.Errorf("cleanup failed: %v", err) } }() @@ -35,6 +36,7 @@ func TestClient(t *testing.T) { // Test PutObject testKey := "foo/bar.txt" testData := []byte("test data") + err = client.PutObject(ctx, testKey, bytes.NewReader(testData)) if err != nil { t.Fatalf("failed to put object: %v", err) @@ -46,7 +48,8 @@ func TestClient(t *testing.T) { t.Fatalf("failed to get object: %v", err) } defer func() { - if err := reader.Close(); err != nil { + err := reader.Close() + if err != nil { t.Errorf("failed to close reader: %v", err) } }() @@ -65,6 +68,7 @@ func TestClient(t *testing.T) { if err != nil { t.Fatalf("failed to head object: %v", err) } + if !exists { t.Error("expected object to exist") } @@ -74,9 +78,11 @@ func TestClient(t *testing.T) { if err != nil { t.Fatalf("failed to list objects: %v", err) } + if len(keys) != 1 { t.Errorf("expected 1 key, got %d", len(keys)) } + if keys[0] != testKey { t.Errorf("unexpected key: got %s, want %s", keys[0], testKey) } @@ -92,6 +98,7 @@ func TestClient(t *testing.T) { if err != nil { t.Fatalf("failed to head object after deletion: %v", err) } + if exists { t.Error("expected object to not exist after deletion") } diff --git a/internal/s3/s3_test.go b/internal/s3/s3_test.go index bc359f0..57fca9a 100644 --- a/internal/s3/s3_test.go +++ b/internal/s3/s3_test.go @@ -57,7 +57,8 @@ func NewTestServer(t *testing.T) *TestServer { // Start server in background go func() { - if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed { + err := server.ListenAndServe() + if err != nil && err != http.ErrServerClosed { t.Logf("test server error: %v", err) } }() @@ -77,7 +78,7 @@ func NewTestServer(t *testing.T) *TestServer { "", )), config.WithClientLogMode(aws.LogRetries|aws.LogRequestWithBody|aws.LogResponseWithBody), - config.WithLogger(logging.LoggerFunc(func(classification logging.Classification, format string, v ...interface{}) { + config.WithLogger(logging.LoggerFunc(func(classification logging.Classification, format string, v ...any) { // Capture logs to buffer instead of stdout fmt.Fprintf(logBuf, "SDK %s %s %s\n", time.Now().Format("2006/01/02 15:04:05"), @@ -125,7 +126,8 @@ func (ts *TestServer) Cleanup() error { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - if err := ts.server.Shutdown(ctx); err != nil { + err := ts.server.Shutdown(ctx) + if err != nil { return err } @@ -141,7 +143,8 @@ func (ts *TestServer) Client() *s3.Client { func TestBasicS3Operations(t *testing.T) { ts := NewTestServer(t) defer func() { - if err := ts.Cleanup(); err != nil { + err := ts.Cleanup() + if err != nil { t.Errorf("cleanup failed: %v", err) } }() @@ -172,7 +175,8 @@ func TestBasicS3Operations(t *testing.T) { t.Fatalf("failed to get object: %v", err) } defer func() { - if err := result.Body.Close(); err != nil { + err := result.Body.Close() + if err != nil { t.Errorf("failed to close body: %v", err) } }() @@ -192,7 +196,8 @@ func TestBasicS3Operations(t *testing.T) { func TestBlobOperations(t *testing.T) { ts := NewTestServer(t) defer func() { - if err := ts.Cleanup(); err != nil { + err := ts.Cleanup() + if err != nil { t.Errorf("cleanup failed: %v", err) } }() @@ -255,7 +260,8 @@ func TestBlobOperations(t *testing.T) { func TestMetadataOperations(t *testing.T) { ts := NewTestServer(t) defer func() { - if err := ts.Cleanup(); err != nil { + err := ts.Cleanup() + if err != nil { t.Errorf("cleanup failed: %v", err) } }() diff --git a/internal/snapshot/backup_test.go b/internal/snapshot/backup_test.go index fa6b9b4..b310fe7 100644 --- a/internal/snapshot/backup_test.go +++ b/internal/snapshot/backup_test.go @@ -4,6 +4,8 @@ import ( "context" "crypto/sha256" "database/sql" + "encoding/hex" + "errors" "fmt" "io" "io/fs" @@ -30,6 +32,7 @@ func NewMockS3Client() *MockS3Client { func (m *MockS3Client) PutBlob(ctx context.Context, hash string, data []byte) error { m.storage[hash] = data + return nil } @@ -38,11 +41,13 @@ func (m *MockS3Client) GetBlob(ctx context.Context, hash string) ([]byte, error) if !ok { return nil, fmt.Errorf("blob not found: %s", hash) } + return data, nil } func (m *MockS3Client) BlobExists(ctx context.Context, hash string) (bool, error) { _, ok := m.storage[hash] + return ok, nil } @@ -81,12 +86,15 @@ func TestBackupWithInMemoryFS(t *testing.T) { // Initialize the database ctx := context.Background() + db, err := database.New(ctx, dbPath) if err != nil { t.Fatalf("Failed to create database: %v", err) } + defer func() { - if err := db.Close(); err != nil { + err := db.Close() + if err != nil { t.Logf("Failed to close database: %v", err) } }() @@ -142,12 +150,14 @@ func TestBackupWithInMemoryFS(t *testing.T) { if !expectedFiles[file.Path.String()] { t.Errorf("Unexpected file in database: %s", file.Path) } + delete(expectedFiles, file.Path.String()) // Verify file metadata fsFile := testFS[file.Path.String()] if fsFile == nil { t.Errorf("File %s not found in test filesystem", file.Path) + continue } @@ -187,6 +197,7 @@ func TestBackupWithInMemoryFS(t *testing.T) { if err != nil { t.Fatalf("Failed to get blob hashes: %v", err) } + if len(blobHashes) == 0 { t.Error("Expected at least one blob to be created") } @@ -197,6 +208,7 @@ func TestBackupWithInMemoryFS(t *testing.T) { if err != nil { t.Errorf("Failed to check blob %s: %v", blobHash, err) } + if !exists { t.Errorf("Blob %s not found in S3", blobHash) } @@ -229,12 +241,15 @@ func TestBackupDeduplication(t *testing.T) { // Initialize the database ctx := context.Background() + db, err := database.New(ctx, dbPath) if err != nil { t.Fatalf("Failed to create database: %v", err) } + defer func() { - if err := db.Close(); err != nil { + err := db.Close() + if err != nil { t.Logf("Failed to close database: %v", err) } }() @@ -348,6 +363,7 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str UID: 1000, // Default UID for test GID: 1000, // Default GID for test } + err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { return b.repos.Files.Create(ctx, tx, file) }) @@ -364,7 +380,8 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str return err } defer func() { - if err := f.Close(); err != nil { + err := f.Close() + if err != nil { // Log but don't fail since we're already in an error path potentially fmt.Fprintf(os.Stderr, "Failed to close file: %v\n", err) } @@ -376,9 +393,10 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str for { n, err := f.Read(buffer) - if err != nil && err != io.EOF { + if err != nil && !errors.Is(err, io.EOF) { return err } + if n == 0 { break } @@ -395,11 +413,13 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str ChunkHash: types.ChunkHash(chunkHash), Size: int64(n), } + return b.repos.Chunks.Create(ctx, tx, chunk) }) if err != nil { return err } + processedChunks[chunkHash] = true } @@ -410,6 +430,7 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str Idx: chunkIndex, ChunkHash: types.ChunkHash(chunkHash), } + return b.repos.FileChunks.Create(ctx, tx, fileChunk) }) if err != nil { @@ -424,6 +445,7 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str FileOffset: int64(chunkIndex * defaultChunkSize), Length: int64(n), } + return b.repos.ChunkFiles.Create(ctx, tx, chunkFile) }) if err != nil { @@ -435,7 +457,6 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str return nil }) - if err != nil { return "", err } @@ -464,12 +485,14 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str // Create blob entry in a short transaction blobID := types.NewBlobID() + err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { blob := &database.Blob{ ID: blobID, Hash: types.BlobHash(blobHash), CreatedTS: time.Now(), } + return b.repos.Blobs.Create(ctx, tx, blob) }) if err != nil { @@ -487,6 +510,7 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str Offset: 0, Length: chunk.Size, } + return b.repos.BlobChunks.Create(ctx, tx, blobChunk) }) if err != nil { @@ -506,7 +530,6 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { return b.repos.Snapshots.UpdateCounts(ctx, tx, snapshotID, fileCount, chunkCount, blobCount, totalSize, blobSize) }) - if err != nil { return "", err } @@ -517,16 +540,18 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str func calculateHash(data []byte) string { h := sha256.New() h.Write(data) - return fmt.Sprintf("%x", h.Sum(nil)) + + return hex.EncodeToString(h.Sum(nil)) } func generateLargeFileContent(size int) []byte { data := make([]byte, size) // Fill with pattern that changes every chunk to avoid deduplication - for i := 0; i < size; i++ { + for i := range size { chunkNum := i / defaultChunkSize data[i] = byte((i + chunkNum) % 256) } + return data } diff --git a/internal/snapshot/exclude_test.go b/internal/snapshot/exclude_test.go index ba2efef..f908250 100644 --- a/internal/snapshot/exclude_test.go +++ b/internal/snapshot/exclude_test.go @@ -63,6 +63,7 @@ func setupExcludeTestFS(t *testing.T) afero.Fs { } testTime := time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC) + for path, content := range files { dir := filepath.Dir(path) err := fs.MkdirAll(dir, 0755) @@ -107,6 +108,7 @@ func createTestScanner(t *testing.T, fs afero.Fs, excludePatterns []string) (*sn func createSnapshotRecord(t *testing.T, ctx context.Context, repos *database.Repositories, snapshotID string) { t.Helper() + err := repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { snap := &database.Snapshot{ ID: types.SnapshotID(snapshotID), @@ -121,6 +123,7 @@ func createSnapshotRecord(t *testing.T, ctx context.Context, repos *database.Rep BlobSize: 0, CompressionRatio: 1.0, } + return repos.Snapshots.Create(ctx, tx, snap) }) require.NoError(t, err) @@ -128,8 +131,10 @@ func createSnapshotRecord(t *testing.T, ctx context.Context, repos *database.Rep func TestExcludePatterns_ExcludeGitDirectory(t *testing.T) { fs := setupExcludeTestFS(t) + scanner, repos, cleanup := createTestScanner(t, fs, []string{".git"}) defer cleanup() + require.NotNil(t, scanner) ctx := context.Background() @@ -148,8 +153,10 @@ func TestExcludePatterns_ExcludeGitDirectory(t *testing.T) { func TestExcludePatterns_ExcludeByExtension(t *testing.T) { fs := setupExcludeTestFS(t) + scanner, repos, cleanup := createTestScanner(t, fs, []string{"*.log"}) defer cleanup() + require.NotNil(t, scanner) ctx := context.Background() @@ -165,8 +172,10 @@ func TestExcludePatterns_ExcludeByExtension(t *testing.T) { func TestExcludePatterns_ExcludeNodeModules(t *testing.T) { fs := setupExcludeTestFS(t) + scanner, repos, cleanup := createTestScanner(t, fs, []string{"node_modules"}) defer cleanup() + require.NotNil(t, scanner) ctx := context.Background() @@ -182,8 +191,10 @@ func TestExcludePatterns_ExcludeNodeModules(t *testing.T) { func TestExcludePatterns_MultiplePatterns(t *testing.T) { fs := setupExcludeTestFS(t) + scanner, repos, cleanup := createTestScanner(t, fs, []string{".git", "node_modules", "*.log", ".DS_Store", "thumbs.db", "cache", "build"}) defer cleanup() + require.NotNil(t, scanner) ctx := context.Background() @@ -199,8 +210,10 @@ func TestExcludePatterns_MultiplePatterns(t *testing.T) { func TestExcludePatterns_NoExclusions(t *testing.T) { fs := setupExcludeTestFS(t) + scanner, repos, cleanup := createTestScanner(t, fs, []string{}) defer cleanup() + require.NotNil(t, scanner) ctx := context.Background() @@ -215,8 +228,10 @@ func TestExcludePatterns_NoExclusions(t *testing.T) { func TestExcludePatterns_ExcludeHiddenFiles(t *testing.T) { fs := setupExcludeTestFS(t) + scanner, repos, cleanup := createTestScanner(t, fs, []string{".*"}) defer cleanup() + require.NotNil(t, scanner) ctx := context.Background() @@ -232,8 +247,10 @@ func TestExcludePatterns_ExcludeHiddenFiles(t *testing.T) { func TestExcludePatterns_DoubleStarGlob(t *testing.T) { fs := setupExcludeTestFS(t) + scanner, repos, cleanup := createTestScanner(t, fs, []string{"**/*.pack"}) defer cleanup() + require.NotNil(t, scanner) ctx := context.Background() @@ -249,8 +266,10 @@ func TestExcludePatterns_DoubleStarGlob(t *testing.T) { func TestExcludePatterns_ExactFileName(t *testing.T) { fs := setupExcludeTestFS(t) + scanner, repos, cleanup := createTestScanner(t, fs, []string{"thumbs.db", ".DS_Store"}) defer cleanup() + require.NotNil(t, scanner) ctx := context.Background() @@ -267,8 +286,10 @@ func TestExcludePatterns_ExactFileName(t *testing.T) { func TestExcludePatterns_CaseSensitive(t *testing.T) { // Pattern matching should be case-sensitive fs := setupExcludeTestFS(t) + scanner, repos, cleanup := createTestScanner(t, fs, []string{"THUMBS.DB"}) defer cleanup() + require.NotNil(t, scanner) ctx := context.Background() @@ -287,6 +308,7 @@ func TestExcludePatterns_DirectoryWithTrailingSlash(t *testing.T) { // Some users might add trailing slashes to directory patterns scanner, repos, cleanup := createTestScanner(t, fs, []string{"cache/", "build/"}) defer cleanup() + require.NotNil(t, scanner) ctx := context.Background() @@ -305,6 +327,7 @@ func TestExcludePatterns_PatternInSubdirectory(t *testing.T) { // Exclude .hidden file specifically in src directory scanner, repos, cleanup := createTestScanner(t, fs, []string{"src/.hidden"}) defer cleanup() + require.NotNil(t, scanner) ctx := context.Background() @@ -343,6 +366,7 @@ func setupAnchoredTestFS(t *testing.T) afero.Fs { } testTime := time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC) + for path, content := range files { dir := filepath.Dir(path) err := fs.MkdirAll(dir, 0755) @@ -359,8 +383,10 @@ func setupAnchoredTestFS(t *testing.T) afero.Fs { func TestExcludePatterns_AnchoredPattern(t *testing.T) { // Pattern starting with / should only match from root of source dir fs := setupAnchoredTestFS(t) + scanner, repos, cleanup := createTestScanner(t, fs, []string{"/projectname"}) defer cleanup() + require.NotNil(t, scanner) ctx := context.Background() @@ -378,8 +404,10 @@ func TestExcludePatterns_AnchoredPattern(t *testing.T) { func TestExcludePatterns_UnanchoredPattern(t *testing.T) { // Pattern without leading / should match anywhere in path fs := setupAnchoredTestFS(t) + scanner, repos, cleanup := createTestScanner(t, fs, []string{"projectname"}) defer cleanup() + require.NotNil(t, scanner) ctx := context.Background() @@ -398,8 +426,10 @@ func TestExcludePatterns_UnanchoredPattern(t *testing.T) { func TestExcludePatterns_AnchoredPatternWithGlob(t *testing.T) { // Anchored pattern with glob fs := setupAnchoredTestFS(t) + scanner, repos, cleanup := createTestScanner(t, fs, []string{"/src/*.go"}) defer cleanup() + require.NotNil(t, scanner) ctx := context.Background() @@ -416,8 +446,10 @@ func TestExcludePatterns_AnchoredPatternWithGlob(t *testing.T) { func TestExcludePatterns_AnchoredPatternFile(t *testing.T) { // Anchored pattern for exact file at root fs := setupAnchoredTestFS(t) + scanner, repos, cleanup := createTestScanner(t, fs, []string{"/file.txt"}) defer cleanup() + require.NotNil(t, scanner) ctx := context.Background() @@ -435,8 +467,10 @@ func TestExcludePatterns_AnchoredPatternFile(t *testing.T) { func TestExcludePatterns_UnanchoredPatternFile(t *testing.T) { // Unanchored pattern for file should match anywhere fs := setupAnchoredTestFS(t) + scanner, repos, cleanup := createTestScanner(t, fs, []string{"file.txt"}) defer cleanup() + require.NotNil(t, scanner) ctx := context.Background() diff --git a/internal/snapshot/file_change_test.go b/internal/snapshot/file_change_test.go index 033934b..a918010 100644 --- a/internal/snapshot/file_change_test.go +++ b/internal/snapshot/file_change_test.go @@ -30,9 +30,11 @@ func TestFileContentChange(t *testing.T) { // Create test database db, err := database.NewTestDB() + require.NoError(t, err) defer func() { - if err := db.Close(); err != nil { + err := db.Close() + if err != nil { t.Errorf("failed to close database: %v", err) } }() @@ -59,6 +61,7 @@ func TestFileContentChange(t *testing.T) { VaultikVersion: "test", StartedAt: time.Now(), } + return repos.Snapshots.Create(ctx, tx, snapshot) }) require.NoError(t, err) @@ -81,6 +84,7 @@ func TestFileContentChange(t *testing.T) { // Modify the file time.Sleep(10 * time.Millisecond) // Ensure mtime changes + err = afero.WriteFile(fs, "/test.txt", []byte("Modified content with different data"), 0644) require.NoError(t, err) @@ -93,6 +97,7 @@ func TestFileContentChange(t *testing.T) { VaultikVersion: "test", StartedAt: time.Now(), } + return repos.Snapshots.Create(ctx, tx, snapshot) }) require.NoError(t, err) @@ -130,6 +135,7 @@ func TestFileContentChange(t *testing.T) { // Verify that chunk_files for old chunk no longer references this file oldChunkFiles, err := repos.ChunkFiles.GetByChunkHash(ctx, oldChunkHash) require.NoError(t, err) + for _, cf := range oldChunkFiles { file, err := repos.Files.GetByID(ctx, cf.FileID) require.NoError(t, err) @@ -159,9 +165,11 @@ func TestMultipleFileChanges(t *testing.T) { // Create test database db, err := database.NewTestDB() + require.NoError(t, err) defer func() { - if err := db.Close(); err != nil { + err := db.Close() + if err != nil { t.Errorf("failed to close database: %v", err) } }() @@ -188,6 +196,7 @@ func TestMultipleFileChanges(t *testing.T) { VaultikVersion: "test", StartedAt: time.Now(), } + return repos.Snapshots.Create(ctx, tx, snapshot) }) require.NoError(t, err) @@ -200,6 +209,7 @@ func TestMultipleFileChanges(t *testing.T) { // Modify two files time.Sleep(10 * time.Millisecond) // Ensure mtime changes + err = afero.WriteFile(fs, "/file1.txt", []byte("Modified content 1"), 0644) require.NoError(t, err) err = afero.WriteFile(fs, "/file3.txt", []byte("Modified content 3"), 0644) @@ -214,6 +224,7 @@ func TestMultipleFileChanges(t *testing.T) { VaultikVersion: "test", StartedAt: time.Now(), } + return repos.Snapshots.Create(ctx, tx, snapshot) }) require.NoError(t, err) diff --git a/internal/snapshot/manifest.go b/internal/snapshot/manifest.go index 3b152cc..f6dc3ae 100644 --- a/internal/snapshot/manifest.go +++ b/internal/snapshot/manifest.go @@ -52,6 +52,7 @@ func EncodeManifest(manifest *Manifest, compressionLevel int) ([]byte, error) { // Compress using zstd var compressedBuf bytes.Buffer + writer, err := zstd.NewWriter(&compressedBuf, zstd.WithEncoderLevel(zstd.EncoderLevelFromZstd(compressionLevel))) if err != nil { return nil, fmt.Errorf("creating zstd writer: %w", err) @@ -59,6 +60,7 @@ func EncodeManifest(manifest *Manifest, compressionLevel int) ([]byte, error) { if _, err := writer.Write(jsonData); err != nil { _ = writer.Close() + return nil, fmt.Errorf("writing compressed data: %w", err) } diff --git a/internal/snapshot/permission_error_test.go b/internal/snapshot/permission_error_test.go index bbde682..d339d3d 100644 --- a/internal/snapshot/permission_error_test.go +++ b/internal/snapshot/permission_error_test.go @@ -12,7 +12,9 @@ import ( func TestWrapPermissionError(t *testing.T) { // Non-permission errors pass through unchanged. plain := errors.New("disk on fire") - if got := wrapPermissionError("/some/path", plain); got != plain { + + got := wrapPermissionError("/some/path", plain) + if !errors.Is(got, plain) { t.Errorf("non-permission error should pass through, got %v", got) } @@ -23,6 +25,7 @@ func TestWrapPermissionError(t *testing.T) { if !errors.Is(wrapped, os.ErrPermission) { t.Error("wrapped error should still match os.ErrPermission") } + if !strings.Contains(wrapped.Error(), "/Users/u/Library/Calendars") { t.Error("wrapped error should name the offending path") } @@ -31,6 +34,7 @@ func TestWrapPermissionError(t *testing.T) { if !strings.Contains(wrapped.Error(), "Full Disk Access") { t.Errorf("macOS permission error should mention Full Disk Access:\n%s", wrapped.Error()) } + if !strings.Contains(wrapped.Error(), "System Settings") { t.Errorf("macOS permission error should point at System Settings:\n%s", wrapped.Error()) } diff --git a/internal/snapshot/progress.go b/internal/snapshot/progress.go index 568efd2..8795e92 100644 --- a/internal/snapshot/progress.go +++ b/internal/snapshot/progress.go @@ -153,6 +153,7 @@ func (pr *ProgressReporter) printSummaryStatus() { if uploadInfo, ok := pr.stats.CurrentUpload.Load().(*UploadInfo); ok && uploadInfo != nil { // Show upload progress instead pr.printUploadProgress(uploadInfo) + return } @@ -175,16 +176,18 @@ func (pr *ProgressReporter) printSummaryStatus() { // Calculate ETA if we have total size and are processing etaStr := "" + if totalSize > 0 && bytesProcessed > 0 { processStart, ok := pr.stats.ProcessStartTime.Load().(time.Time) if ok && !processStart.IsZero() { processElapsed := time.Since(processStart) + rate := float64(bytesProcessed) / processElapsed.Seconds() if rate > 0 { remainingBytes := totalSize - bytesProcessed remainingSeconds := float64(remainingBytes) / rate eta := time.Duration(remainingSeconds * float64(time.Second)) - etaStr = fmt.Sprintf(" | ETA: %s", formatDuration(eta)) + etaStr = " | ETA: " + formatDuration(eta) } } } @@ -206,7 +209,7 @@ func (pr *ProgressReporter) printSummaryStatus() { ) if currentFile != "" { - status += fmt.Sprintf(" | Current: %s", truncatePath(currentFile, 40)) + status += " | Current: " + truncatePath(currentFile, 40) } log.Info(status) @@ -242,6 +245,7 @@ func (pr *ProgressReporter) printDetailedStatus() { processStart, ok := pr.stats.ProcessStartTime.Load().(time.Time) if ok && !processStart.IsZero() { processElapsed := time.Since(processStart) + processRate := float64(bytesProcessed) / processElapsed.Seconds() if processRate > 0 { remainingBytes := totalSize - bytesProcessed @@ -276,9 +280,11 @@ func (pr *ProgressReporter) printDetailedStatus() { log.Info("Total uploaded to remote", "uploaded", humanize.Bytes(uint64(bytesUploaded)), "compression_ratio", formatRatio(bytesUploaded, bytesScanned)) + if currentFile != "" { log.Info("Current file", "path", currentFile) } + log.Notice("=============================") } @@ -288,12 +294,15 @@ func formatDuration(d time.Duration) string { if d < 0 { return "unknown" } + if d < time.Minute { return fmt.Sprintf("%ds", int(d.Seconds())) } + if d < time.Hour { return fmt.Sprintf("%dm%ds", int(d.Minutes()), int(d.Seconds())%60) } + return fmt.Sprintf("%dh%dm", int(d.Hours()), int(d.Minutes())%60) } @@ -301,6 +310,7 @@ func formatPercent(numerator, denominator int64) string { if denominator == 0 { return "0.0%" } + return fmt.Sprintf("%.1f%%", float64(numerator)/float64(denominator)*100) } @@ -308,7 +318,9 @@ func formatRatio(compressed, uncompressed int64) string { if uncompressed == 0 { return "1.00" } + ratio := float64(compressed) / float64(uncompressed) + return fmt.Sprintf("%.2f", ratio) } @@ -353,6 +365,7 @@ func (pr *ProgressReporter) ReportUploadComplete(blobHash string, size int64, du if duration < time.Millisecond { duration = time.Millisecond } + bytesPerSec := float64(size) / duration.Seconds() bitsPerSec := bytesPerSec * 8 @@ -398,6 +411,7 @@ func (pr *ProgressReporter) ReportUploadProgress(blobHash string, bytesUploaded, // Calculate ETA based on current speed etaStr := "unknown" + if instantSpeed > 0 && bytesUploaded < totalSize { remainingBytes := totalSize - bytesUploaded remainingSeconds := float64(remainingBytes) / instantSpeed diff --git a/internal/snapshot/remotekey.go b/internal/snapshot/remotekey.go index 031b9ce..7338f07 100644 --- a/internal/snapshot/remotekey.go +++ b/internal/snapshot/remotekey.go @@ -36,5 +36,6 @@ const remoteKeyPrefix = "vaultik|" func RemoteSnapshotKey(snapshotID string) string { first := sha256.Sum256([]byte(remoteKeyPrefix + snapshotID)) second := sha256.Sum256(first[:]) + return hex.EncodeToString(second[:]) } diff --git a/internal/snapshot/scanner.go b/internal/snapshot/scanner.go index c6b2442..06452e6 100644 --- a/internal/snapshot/scanner.go +++ b/internal/snapshot/scanner.go @@ -119,6 +119,7 @@ func NewScanner(cfg ScannerConfig) *Scanner { // Create encryptor (required for blob packing) if len(cfg.AgeRecipients) == 0 { log.Error("No age recipients configured - encryption is required") + return nil } @@ -130,9 +131,11 @@ func NewScanner(cfg ScannerConfig) *Scanner { Repositories: cfg.Repositories, Fs: cfg.FS, } + packer, err := blob.NewPacker(packerCfg) if err != nil { log.Error("Failed to create packer", "error", err) + return nil } @@ -199,11 +202,14 @@ func (s *Scanner) Scan(ctx context.Context, path string, snapshotID string) (*Sc // Phase 1: Scan directory, collect files to process, and track existing files // (builds existingFiles map during walk to avoid double traversal) log.Info("Phase 1/3: Scanning directory structure") + existingFiles := make(map[string]struct{}) + scanResult, err := s.scanPhase(ctx, path, result, existingFiles, knownFiles) if err != nil { return nil, fmt.Errorf("scan phase failed: %w", err) } + filesToProcess := scanResult.FilesToProcess // Phase 1b: Detect deleted files by comparing DB against scanned files @@ -213,8 +219,10 @@ func (s *Scanner) Scan(ctx context.Context, path string, snapshotID string) (*Sc // Phase 1c: Associate unchanged files with this snapshot (no new records needed) if len(scanResult.UnchangedFileIDs) > 0 { - s.ui.Begin("Associating %s unchanged files with the snapshot.", s.ui.Count(len(scanResult.UnchangedFileIDs))) - if err := s.batchAddFilesToSnapshot(ctx, scanResult.UnchangedFileIDs); err != nil { + s.ui.Beginf("Associating %s unchanged files with the snapshot.", s.ui.Count(len(scanResult.UnchangedFileIDs))) + + err := s.batchAddFilesToSnapshot(ctx, scanResult.UnchangedFileIDs) + if err != nil { return nil, fmt.Errorf("associating unchanged files: %w", err) } } @@ -224,13 +232,15 @@ func (s *Scanner) Scan(ctx context.Context, path string, snapshotID string) (*Sc // Phase 2: Process files and create chunks if len(filesToProcess) > 0 { - s.ui.Begin("Backing up %s snapshot source files (chunking, compressing, encrypting, uploading).", s.ui.Count(len(filesToProcess))) + s.ui.Beginf("Backing up %s snapshot source files (chunking, compressing, encrypting, uploading).", s.ui.Count(len(filesToProcess))) log.Info("Phase 2/3: Creating snapshot (chunking, compressing, encrypting, and uploading blobs)") - if err := s.processPhase(ctx, filesToProcess, result); err != nil { + + err := s.processPhase(ctx, filesToProcess, result) + if err != nil { return nil, fmt.Errorf("process phase failed: %w", err) } } else { - s.ui.Info("Snapshot file backup skipped: no changed files (creating metadata-only snapshot).") + s.ui.Infof("Snapshot file backup skipped: no changed files (creating metadata-only snapshot).") log.Info("Phase 2/3: Skipping (no files need processing, metadata-only snapshot)") } @@ -243,18 +253,22 @@ func (s *Scanner) Scan(ctx context.Context, path string, snapshotID string) (*Sc // loadDatabaseState loads known files and chunks from the database into memory for fast lookup // This avoids per-file and per-chunk database queries during the scan and process phases func (s *Scanner) loadDatabaseState(ctx context.Context, path string) (map[string]*database.File, error) { - s.ui.Begin("Loading known files from local index database.") + s.ui.Beginf("Loading known files from local index database.") + knownFiles, err := s.loadKnownFiles(ctx, path) if err != nil { return nil, fmt.Errorf("loading known files: %w", err) } - s.ui.Complete("Loaded %s known files from local index database.", s.ui.Count(len(knownFiles))) - s.ui.Begin("Loading known chunks from local index database.") + s.ui.Completef("Loaded %s known files from local index database.", s.ui.Count(len(knownFiles))) + + s.ui.Beginf("Loading known chunks from local index database.") + if err := s.loadKnownChunks(ctx); err != nil { return nil, fmt.Errorf("loading known chunks: %w", err) } - s.ui.Complete("Loaded %s known chunks from local index database.", s.ui.Count(len(s.knownChunks))) + + s.ui.Completef("Loaded %s known chunks from local index database.", s.ui.Count(len(s.knownChunks))) return knownFiles, nil } @@ -288,7 +302,8 @@ func (s *Scanner) summarizeScanPhase(result *ScanResult, filesToProcess []*FileT s.ui.Count(result.FilesDeleted), s.ui.Size(result.BytesDeleted)) } - s.ui.Complete("%s.", msg) + + s.ui.Completef("%s.", msg) } // finalizeScanResult populates final blob statistics in the scan result @@ -337,6 +352,7 @@ func (s *Scanner) loadKnownChunks(ctx context.Context) error { } s.knownChunksMu.Lock() + s.knownChunks = make(map[string]struct{}, len(chunks)) for _, c := range chunks { s.knownChunks[c.ChunkHash.String()] = struct{}{} @@ -351,6 +367,7 @@ func (s *Scanner) chunkExists(hash string) bool { s.knownChunksMu.RLock() _, exists := s.knownChunks[hash] s.knownChunksMu.RUnlock() + return exists } @@ -371,7 +388,9 @@ func (s *Scanner) addPendingChunkHash(hash string) { // removePendingChunkHashes removes committed chunk hashes from the pending set func (s *Scanner) removePendingChunkHashes(hashes []string) { log.Debug("removePendingChunkHashes: starting", "count", len(hashes)) + start := time.Now() + s.pendingChunkHashesMu.Lock() for _, hash := range hashes { delete(s.pendingChunkHashes, hash) @@ -385,6 +404,7 @@ func (s *Scanner) isChunkPending(hash string) bool { s.pendingChunkHashesMu.Lock() _, pending := s.pendingChunkHashes[hash] s.pendingChunkHashesMu.Unlock() + return pending } @@ -411,37 +431,45 @@ func (s *Scanner) flushPendingFiles(ctx context.Context) error { return s.repos.WithTx(ctx, func(txCtx context.Context, tx *sql.Tx) error { for _, data := range files { // Create or update the file record - if err := s.repos.Files.Create(txCtx, tx, data.file); err != nil { + err := s.repos.Files.Create(txCtx, tx, data.file) + if err != nil { return fmt.Errorf("creating file record: %w", err) } // Delete any existing file_chunks and chunk_files for this file - if err := s.repos.FileChunks.DeleteByFileID(txCtx, tx, data.file.ID); err != nil { + err = s.repos.FileChunks.DeleteByFileID(txCtx, tx, data.file.ID) + if err != nil { return fmt.Errorf("deleting old file chunks: %w", err) } - if err := s.repos.ChunkFiles.DeleteByFileID(txCtx, tx, data.file.ID); err != nil { + + err = s.repos.ChunkFiles.DeleteByFileID(txCtx, tx, data.file.ID) + if err != nil { return fmt.Errorf("deleting old chunk files: %w", err) } // Create file-chunk mappings for i := range data.fileChunks { - if err := s.repos.FileChunks.Create(txCtx, tx, &data.fileChunks[i]); err != nil { + err := s.repos.FileChunks.Create(txCtx, tx, &data.fileChunks[i]) + if err != nil { return fmt.Errorf("creating file chunk: %w", err) } } // Create chunk-file mappings for i := range data.chunkFiles { - if err := s.repos.ChunkFiles.Create(txCtx, tx, &data.chunkFiles[i]); err != nil { + err := s.repos.ChunkFiles.Create(txCtx, tx, &data.chunkFiles[i]) + if err != nil { return fmt.Errorf("creating chunk file: %w", err) } } // Add file to snapshot - if err := s.repos.Snapshots.AddFileByID(txCtx, tx, s.snapshotID, data.file.ID); err != nil { + err = s.repos.Snapshots.AddFileByID(txCtx, tx, s.snapshotID, data.file.ID) + if err != nil { return fmt.Errorf("adding file to snapshot: %w", err) } } + return nil }) } @@ -455,6 +483,7 @@ func (s *Scanner) flushAllPending(ctx context.Context) error { // Files with pending chunks are kept in the queue for later flushing func (s *Scanner) flushCompletedPendingFiles(ctx context.Context) error { flushStart := time.Now() + log.Debug("flushCompletedPendingFiles: starting") // Partition pending files into those ready to flush and those still waiting @@ -462,6 +491,7 @@ func (s *Scanner) flushCompletedPendingFiles(ctx context.Context) error { if len(canFlush) == 0 { log.Debug("flushCompletedPendingFiles: nothing to flush") + return nil } @@ -474,10 +504,13 @@ func (s *Scanner) flushCompletedPendingFiles(ctx context.Context) error { // Execute the batch flush in a single transaction log.Debug("flushCompletedPendingFiles: starting transaction") + txStart := time.Now() err := s.executeBatchFileFlush(ctx, allFiles, allFileIDs, allFileChunks, allChunkFiles) + log.Debug("flushCompletedPendingFiles: transaction done", "duration", time.Since(txStart)) log.Debug("flushCompletedPendingFiles: total duration", "duration", time.Since(flushStart)) + return err } @@ -492,21 +525,27 @@ func (s *Scanner) partitionPendingByChunkStatus() (canFlush []pendingFileData, s var stillPending []pendingFileData log.Debug("flushCompletedPendingFiles: checking which files can flush") + checkStart := time.Now() + for _, data := range s.pendingFiles { allChunksCommitted := true + for _, fc := range data.fileChunks { if s.isChunkPending(fc.ChunkHash.String()) { allChunksCommitted = false + break } } + if allChunksCommitted { canFlush = append(canFlush, data) } else { stillPending = append(stillPending, data) } } + log.Debug("flushCompletedPendingFiles: check done", "duration", time.Since(checkStart), "can_flush", len(canFlush), "still_pending", len(stillPending)) s.pendingFiles = stillPending @@ -520,12 +559,15 @@ func (s *Scanner) partitionPendingByChunkStatus() (canFlush []pendingFileData, s // mappings from the given pending file data for efficient batch database operations func (s *Scanner) collectBatchFlushData(canFlush []pendingFileData) ([]*database.File, []types.FileID, []database.FileChunk, []database.ChunkFile) { log.Debug("flushCompletedPendingFiles: collecting data for batch ops") + collectStart := time.Now() - var allFileChunks []database.FileChunk - var allChunkFiles []database.ChunkFile - var allFileIDs []types.FileID - var allFiles []*database.File + var ( + allFileChunks []database.FileChunk + allChunkFiles []database.ChunkFile + allFileIDs []types.FileID + allFiles []*database.File + ) for _, data := range canFlush { allFileChunks = append(allFileChunks, data.fileChunks...) @@ -551,52 +593,77 @@ func (s *Scanner) executeBatchFileFlush(ctx context.Context, allFiles []*databas // Batch delete old file_chunks and chunk_files log.Debug("flushCompletedPendingFiles: deleting old file_chunks") + opStart := time.Now() - if err := s.repos.FileChunks.DeleteByFileIDs(txCtx, tx, allFileIDs); err != nil { + + err := s.repos.FileChunks.DeleteByFileIDs(txCtx, tx, allFileIDs) + if err != nil { return fmt.Errorf("batch deleting old file chunks: %w", err) } + log.Debug("flushCompletedPendingFiles: deleted file_chunks", "duration", time.Since(opStart)) log.Debug("flushCompletedPendingFiles: deleting old chunk_files") + opStart = time.Now() - if err := s.repos.ChunkFiles.DeleteByFileIDs(txCtx, tx, allFileIDs); err != nil { + + err = s.repos.ChunkFiles.DeleteByFileIDs(txCtx, tx, allFileIDs) + if err != nil { return fmt.Errorf("batch deleting old chunk files: %w", err) } + log.Debug("flushCompletedPendingFiles: deleted chunk_files", "duration", time.Since(opStart)) // Batch create/update file records log.Debug("flushCompletedPendingFiles: creating files") + opStart = time.Now() - if err := s.repos.Files.CreateBatch(txCtx, tx, allFiles); err != nil { + + err = s.repos.Files.CreateBatch(txCtx, tx, allFiles) + if err != nil { return fmt.Errorf("batch creating file records: %w", err) } + log.Debug("flushCompletedPendingFiles: created files", "duration", time.Since(opStart)) // Batch insert file_chunks log.Debug("flushCompletedPendingFiles: inserting file_chunks") + opStart = time.Now() - if err := s.repos.FileChunks.CreateBatch(txCtx, tx, allFileChunks); err != nil { + + err = s.repos.FileChunks.CreateBatch(txCtx, tx, allFileChunks) + if err != nil { return fmt.Errorf("batch creating file chunks: %w", err) } + log.Debug("flushCompletedPendingFiles: inserted file_chunks", "duration", time.Since(opStart)) // Batch insert chunk_files log.Debug("flushCompletedPendingFiles: inserting chunk_files") + opStart = time.Now() - if err := s.repos.ChunkFiles.CreateBatch(txCtx, tx, allChunkFiles); err != nil { + + err = s.repos.ChunkFiles.CreateBatch(txCtx, tx, allChunkFiles) + if err != nil { return fmt.Errorf("batch creating chunk files: %w", err) } + log.Debug("flushCompletedPendingFiles: inserted chunk_files", "duration", time.Since(opStart)) // Batch add files to snapshot log.Debug("flushCompletedPendingFiles: adding files to snapshot") + opStart = time.Now() - if err := s.repos.Snapshots.AddFilesByIDBatch(txCtx, tx, s.snapshotID, allFileIDs); err != nil { + + err = s.repos.Snapshots.AddFilesByIDBatch(txCtx, tx, s.snapshotID, allFileIDs) + if err != nil { return fmt.Errorf("batch adding files to snapshot: %w", err) } + log.Debug("flushCompletedPendingFiles: added files to snapshot", "duration", time.Since(opStart)) log.Debug("flushCompletedPendingFiles: transaction complete") + return nil }) } @@ -616,24 +683,31 @@ func (s *Scanner) scanPhase(ctx context.Context, path string, result *ScanResult estimatedTotal := int64(len(knownFiles)) var filesToProcess []*FileToProcess + var unchangedFileIDs []types.FileID // Just IDs - no new records needed + var mu sync.Mutex // Set up periodic status output startTime := time.Now() lastStatusTime := time.Now() statusInterval := 15 * time.Second + var filesScanned int64 log.Debug("Starting directory walk", "path", path) + err := afero.Walk(s.fs, path, func(filePath string, info os.FileInfo, err error) error { if err != nil { if s.skipErrors { log.Error("Failed to access file (skipping due to --skip-errors)", "path", filePath, "error", err) - s.ui.Error("Failed to access %s: %v. Skipping (--skip-errors).", s.ui.Path(filePath), err) + s.ui.Errorf("Failed to access %s: %v. Skipping (--skip-errors).", s.ui.Path(filePath), err) + return nil // Continue scanning } + log.Debug("Error accessing filesystem entry", "path", filePath, "error", err) + return wrapPermissionError(filePath, err) } @@ -649,6 +723,7 @@ func (s *Scanner) scanPhase(ctx context.Context, path string, result *ScanResult if info.IsDir() { return filepath.SkipDir } + return nil } @@ -657,7 +732,9 @@ func (s *Scanner) scanPhase(ctx context.Context, path string, result *ScanResult file := s.buildSymlinkEntry(filePath, info) if file != nil { existingFiles[filePath] = struct{}{} + mu.Lock() + filesToProcess = append(filesToProcess, &FileToProcess{ Path: filePath, FileInfo: info, @@ -667,6 +744,7 @@ func (s *Scanner) scanPhase(ctx context.Context, path string, result *ScanResult mu.Unlock() s.updateScanEntryStats(result, true, info) } + return nil } @@ -674,7 +752,9 @@ func (s *Scanner) scanPhase(ctx context.Context, path string, result *ScanResult if info.IsDir() { file := s.buildDirectoryEntry(filePath, info) existingFiles[filePath] = struct{}{} + mu.Lock() + filesToProcess = append(filesToProcess, &FileToProcess{ Path: filePath, FileInfo: info, @@ -682,6 +762,7 @@ func (s *Scanner) scanPhase(ctx context.Context, path string, result *ScanResult }) filesScanned++ mu.Unlock() + return nil } @@ -708,6 +789,7 @@ func (s *Scanner) scanPhase(ctx context.Context, path string, result *ScanResult // Unchanged file with existing ID - just need snapshot association unchangedFileIDs = append(unchangedFileIDs, file.ID) } + filesScanned++ changedCount := len(filesToProcess) mu.Unlock() @@ -718,12 +800,12 @@ func (s *Scanner) scanPhase(ctx context.Context, path string, result *ScanResult // Output periodic status if time.Since(lastStatusTime) >= statusInterval { s.printScanProgressLine(filesScanned, changedCount, estimatedTotal, startTime) + lastStatusTime = time.Now() } return nil }) - if err != nil { return nil, err } @@ -745,11 +827,13 @@ func (s *Scanner) updateScanEntryStats(result *ScanResult, needsProcessing bool, } else { result.FilesSkipped++ result.BytesSkipped += info.Size() + if s.progress != nil { s.progress.GetStats().FilesSkipped.Add(1) s.progress.GetStats().BytesSkipped.Add(info.Size()) } } + result.FilesScanned++ if s.progress != nil { s.progress.GetStats().FilesScanned.Add(1) @@ -768,16 +852,16 @@ func (s *Scanner) printScanProgressLine(filesScanned int64, changedCount int, es if pct > 100 { pct = 100 // Cap at 100% for display } - remaining := estimatedTotal - filesScanned - if remaining < 0 { - remaining = 0 - } + + remaining := max(estimatedTotal-filesScanned, 0) + var eta time.Duration if rate > 0 && remaining > 0 { eta = time.Duration(float64(remaining)/rate) * time.Second } + if eta > 0 { - s.ui.Progress("Snapshot source files enumeration: %s files (~%s), %s changed or new, %.0f files/sec, enumeration elapsed: %s, enumeration ETA: %s (est remain %s).", + s.ui.Progressf("Snapshot source files enumeration: %s files (~%s), %s changed or new, %.0f files/sec, enumeration elapsed: %s, enumeration ETA: %s (est remain %s).", s.ui.Count(int(filesScanned)), s.ui.Percent(pct), s.ui.Count(changedCount), @@ -786,7 +870,7 @@ func (s *Scanner) printScanProgressLine(filesScanned int64, changedCount int, es s.ui.Time(time.Now().Add(eta)), s.ui.Duration(eta)) } else { - s.ui.Progress("Snapshot source files enumeration: %s files (~%s), %s changed or new, %.0f files/sec, enumeration elapsed: %s.", + s.ui.Progressf("Snapshot source files enumeration: %s files (~%s), %s changed or new, %.0f files/sec, enumeration elapsed: %s.", s.ui.Count(int(filesScanned)), s.ui.Percent(pct), s.ui.Count(changedCount), @@ -794,7 +878,7 @@ func (s *Scanner) printScanProgressLine(filesScanned int64, changedCount int, es s.ui.Duration(elapsed)) } } else { - s.ui.Progress("Snapshot source files enumeration: %s files seen, %s changed or new, %.0f files/sec, enumeration elapsed: %s.", + s.ui.Progressf("Snapshot source files enumeration: %s files seen, %s changed or new, %.0f files/sec, enumeration elapsed: %s.", s.ui.Count(int(filesScanned)), s.ui.Count(changedCount), rate, @@ -808,6 +892,7 @@ func (s *Scanner) buildSymlinkEntry(path string, info os.FileInfo) *database.Fil target, err := os.Readlink(path) if err != nil { log.Debug("Cannot read symlink target", "path", path, "error", err) + return nil } @@ -860,9 +945,11 @@ func (s *Scanner) buildDirectoryEntry(path string, info os.FileInfo) *database.F // and associates it with the current snapshot. No chunking is performed. func (s *Scanner) recordNonRegularFile(ctx context.Context, ftp *FileToProcess) error { return s.repos.WithTx(ctx, func(txCtx context.Context, tx *sql.Tx) error { - if err := s.repos.Files.Create(txCtx, tx, ftp.File); err != nil { + err := s.repos.Files.Create(txCtx, tx, ftp.File) + if err != nil { return fmt.Errorf("creating non-regular file record: %w", err) } + return s.repos.Snapshots.AddFileByID(txCtx, tx, s.snapshotID, ftp.File.ID) }) } @@ -941,18 +1028,18 @@ func (s *Scanner) batchAddFilesToSnapshot(ctx context.Context, fileIDs []types.F default: } - end := i + batchSize - if end > len(fileIDs) { - end = len(fileIDs) - } + end := min(i+batchSize, len(fileIDs)) + batch := fileIDs[i:end] err := s.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { for _, fileID := range batch { - if err := s.repos.Snapshots.AddFileByID(ctx, tx, s.snapshotID, fileID); err != nil { + err := s.repos.Snapshots.AddFileByID(ctx, tx, s.snapshotID, fileID) + if err != nil { return fmt.Errorf("adding file to snapshot: %w", err) } } + return nil }) if err != nil { @@ -964,15 +1051,16 @@ func (s *Scanner) batchAddFilesToSnapshot(ctx context.Context, fileIDs []types.F elapsed := time.Since(startTime) rate := float64(end) / elapsed.Seconds() pct := float64(end) / float64(len(fileIDs)) * 100 - s.ui.Progress("Snapshot unchanged-file association: %s/%s (%s), %.0f files/sec.", + s.ui.Progressf("Snapshot unchanged-file association: %s/%s (%s), %.0f files/sec.", s.ui.Count(end), s.ui.Count(len(fileIDs)), s.ui.Percent(pct), rate) + lastStatusTime = time.Now() } } elapsed := time.Since(startTime) rate := float64(len(fileIDs)) / elapsed.Seconds() - s.ui.Complete("Associated %s unchanged files with the snapshot in %s (%.0f files/sec).", + s.ui.Completef("Associated %s unchanged files with the snapshot in %s (%.0f files/sec).", s.ui.Count(len(fileIDs)), s.ui.Duration(elapsed), rate) return nil @@ -991,7 +1079,9 @@ func (s *Scanner) processPhase(ctx context.Context, filesToProcess []*FileToProc statusInterval := 15 * time.Second startTime := time.Now() filesProcessed := 0 + var bytesProcessed int64 + totalFiles := len(filesToProcess) // Process each file @@ -1006,6 +1096,7 @@ func (s *Scanner) processPhase(ctx context.Context, filesToProcess []*FileToProc if err != nil { return err } + if skipped { continue } @@ -1021,6 +1112,7 @@ func (s *Scanner) processPhase(ctx context.Context, filesToProcess []*FileToProc // Output periodic status if time.Since(lastStatusTime) >= statusInterval { s.printProcessingProgress(filesProcessed, totalFiles, bytesProcessed, totalBytes, startTime) + lastStatusTime = time.Now() } } @@ -1032,22 +1124,29 @@ func (s *Scanner) processPhase(ctx context.Context, filesToProcess []*FileToProc // processFileWithErrorHandling wraps processFileStreaming with error recovery for // deleted files and skip-errors mode. Returns (skipped, error). func (s *Scanner) processFileWithErrorHandling(ctx context.Context, fileToProcess *FileToProcess, result *ScanResult) (bool, error) { - if err := s.processFileStreaming(ctx, fileToProcess, result); err != nil { + err := s.processFileStreaming(ctx, fileToProcess, result) + if err != nil { // Handle files that were deleted between scan and process phases if errors.Is(err, os.ErrNotExist) { log.Warn("File was deleted during backup, skipping", "path", fileToProcess.Path) + result.FilesSkipped++ + return true, nil } // Skip file read errors if --skip-errors is enabled if s.skipErrors { log.Error("Failed to process file (skipping due to --skip-errors)", "path", fileToProcess.Path, "error", err) - s.ui.Error("Failed to process %s: %v. Skipping (--skip-errors).", s.ui.Path(fileToProcess.Path), err) + s.ui.Errorf("Failed to process %s: %v. Skipping (--skip-errors).", s.ui.Path(fileToProcess.Path), err) + result.FilesSkipped++ + return true, nil } + return false, fmt.Errorf("processing file %s: %w", fileToProcess.Path, err) } + return false, nil } @@ -1061,13 +1160,14 @@ func (s *Scanner) printProcessingProgress(filesProcessed, totalFiles int, bytesP // Calculate ETA based on bytes (more accurate than files) remainingBytes := totalBytes - bytesProcessed + var eta time.Duration if byteRate > 0 { eta = time.Duration(float64(remainingBytes)/byteRate) * time.Second } if eta > 0 { - s.ui.Progress("Snapshot backup: %s/%s files (%s), %s/%s, %s, %.0f files/sec, backup elapsed: %s, backup ETA: %s (est remain %s).", + s.ui.Progressf("Snapshot backup: %s/%s files (%s), %s/%s, %s, %.0f files/sec, backup elapsed: %s, backup ETA: %s (est remain %s).", s.ui.Count(filesProcessed), s.ui.Count(totalFiles), s.ui.Percent(pct), @@ -1079,7 +1179,7 @@ func (s *Scanner) printProcessingProgress(filesProcessed, totalFiles int, bytesP s.ui.Time(time.Now().Add(eta)), s.ui.Duration(eta)) } else { - s.ui.Progress("Snapshot backup: %s/%s files (%s), %s/%s, %s, %.0f files/sec, backup elapsed: %s.", + s.ui.Progressf("Snapshot backup: %s/%s files (%s), %s/%s, %s, %.0f files/sec, backup elapsed: %s.", s.ui.Count(filesProcessed), s.ui.Count(totalFiles), s.ui.Percent(pct), @@ -1097,15 +1197,19 @@ func (s *Scanner) finalizeProcessPhase(ctx context.Context, result *ScanResult) // Final packer flush first - this commits remaining chunks to DB // and handleBlobReady will flush files whose chunks are now committed s.packerMu.Lock() - if err := s.packer.Flush(); err != nil { + + err := s.packer.Flush() + if err != nil { s.packerMu.Unlock() + return fmt.Errorf("flushing packer: %w", err) } s.packerMu.Unlock() // Flush any remaining pending files (e.g., files with only pre-existing chunks // that didn't trigger a blob finalize) - if err := s.flushAllPending(ctx); err != nil { + err = s.flushAllPending(ctx) + if err != nil { return fmt.Errorf("flushing remaining pending files: %w", err) } @@ -1119,6 +1223,7 @@ func (s *Scanner) finalizeProcessPhase(ctx context.Context, result *ScanResult) if err != nil { return fmt.Errorf("parsing blob ID: %w", err) } + err = s.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { return s.repos.Snapshots.AddBlob(ctx, tx, s.snapshotID, blobID, types.BlobHash(b.Hash)) }) @@ -1126,6 +1231,7 @@ func (s *Scanner) finalizeProcessPhase(ctx context.Context, result *ScanResult) return fmt.Errorf("storing blob metadata: %w", err) } } + result.BlobsCreated += len(blobs) } @@ -1148,14 +1254,17 @@ func (s *Scanner) handleBlobReady(blobWithReader *blob.BlobWithReader) error { } blobPath := fmt.Sprintf("blobs/%s/%s/%s", finishedBlob.Hash[:2], finishedBlob.Hash[2:4], finishedBlob.Hash) + blobExists, err := s.uploadBlobIfNeeded(ctx, blobPath, blobWithReader, startTime) if err != nil { s.cleanupBlobTempFile(blobWithReader) + return fmt.Errorf("uploading blob %s: %w", finishedBlob.Hash, err) } if err := s.recordBlobMetadata(ctx, finishedBlob, blobExists, startTime); err != nil { s.cleanupBlobTempFile(blobWithReader) + return err } @@ -1181,25 +1290,28 @@ func (s *Scanner) uploadBlobIfNeeded(ctx context.Context, blobPath string, blobW if _, err := s.storage.Stat(ctx, blobPath); err == nil { log.Info("Blob already exists in storage, skipping upload", "hash", finishedBlob.Hash, "size", humanize.Bytes(uint64(finishedBlob.Compressed))) - s.ui.Info("Blob %s (%s) already exists at %s. Skipping upload.", + s.ui.Infof("Blob %s (%s) already exists at %s. Skipping upload.", s.ui.Hex(finishedBlob.Hash), s.ui.Size(finishedBlob.Compressed), s.ui.Path(destination)) + return true, nil } - s.ui.Begin("Uploading blob %s (%s) to %s.", + s.ui.Beginf("Uploading blob %s (%s) to %s.", s.ui.Hex(finishedBlob.Hash), s.ui.Size(finishedBlob.Compressed), s.ui.Path(destination)) progressCallback := s.makeUploadProgressCallback(ctx, finishedBlob, startTime) - if err := s.storage.PutWithProgress(ctx, blobPath, blobWithReader.Reader, finishedBlob.Compressed, progressCallback); err != nil { + err := s.storage.PutWithProgress(ctx, blobPath, blobWithReader.Reader, finishedBlob.Compressed, progressCallback) + if err != nil { log.Error("Failed to upload blob", "hash", finishedBlob.Hash, "error", err) + return false, fmt.Errorf("uploading blob to storage: %w", err) } uploadDuration := time.Since(startTime) uploadSpeedBps := float64(finishedBlob.Compressed) / uploadDuration.Seconds() - s.ui.Complete("Uploaded blob %s (%s) in %s at %s.", + s.ui.Completef("Uploaded blob %s (%s) in %s at %s.", s.ui.Hex(finishedBlob.Hash), s.ui.Size(finishedBlob.Compressed), s.ui.Duration(uploadDuration), @@ -1228,17 +1340,21 @@ func (s *Scanner) makeUploadProgressCallback(ctx context.Context, finishedBlob * lastProgressTime := time.Now() lastProgressBytes := int64(0) lastStdoutTime := time.Now() + const stdoutInterval = 15 * time.Second return func(uploaded int64) error { now := time.Now() + elapsed := now.Sub(lastProgressTime).Seconds() if elapsed > 0.5 { bytesSinceLastUpdate := uploaded - lastProgressBytes + speed := float64(bytesSinceLastUpdate) / elapsed if s.progress != nil { s.progress.ReportUploadProgress(finishedBlob.Hash, uploaded, finishedBlob.Compressed, speed) } + lastProgressTime = now lastProgressBytes = uploaded } @@ -1248,11 +1364,13 @@ func (s *Scanner) makeUploadProgressCallback(ctx context.Context, finishedBlob * totalElapsed := now.Sub(uploadStart) pct := float64(uploaded) / float64(finishedBlob.Compressed) * 100 avgSpeed := float64(uploaded) / totalElapsed.Seconds() + var eta time.Duration if avgSpeed > 0 { eta = time.Duration(float64(finishedBlob.Compressed-uploaded)/avgSpeed) * time.Second } - s.ui.Progress("Blob upload %s: %s / %s (%s) at %s, blob upload elapsed: %s, blob upload ETA: %s (est remain %s).", + + s.ui.Progressf("Blob upload %s: %s / %s (%s) at %s, blob upload elapsed: %s, blob upload ETA: %s (est remain %s).", s.ui.Hex(finishedBlob.Hash), s.ui.Size(uploaded), s.ui.Size(finishedBlob.Compressed), @@ -1283,11 +1401,13 @@ func (s *Scanner) recordBlobMetadata(ctx context.Context, finishedBlob *blob.Fin uploadDuration := time.Since(startTime) return s.repos.WithTx(ctx, func(txCtx context.Context, tx *sql.Tx) error { - if err := s.repos.Blobs.UpdateUploaded(txCtx, tx, finishedBlob.ID); err != nil { + err := s.repos.Blobs.UpdateUploaded(txCtx, tx, finishedBlob.ID) + if err != nil { return fmt.Errorf("updating blob upload timestamp: %w", err) } - if err := s.repos.Snapshots.AddBlob(txCtx, tx, s.snapshotID, finishedBlobID, types.BlobHash(finishedBlob.Hash)); err != nil { + err = s.repos.Snapshots.AddBlob(txCtx, tx, s.snapshotID, finishedBlobID, types.BlobHash(finishedBlob.Hash)) + if err != nil { return fmt.Errorf("adding blob to snapshot: %w", err) } @@ -1299,7 +1419,9 @@ func (s *Scanner) recordBlobMetadata(ctx context.Context, finishedBlob *blob.Fin Size: finishedBlob.Compressed, DurationMs: uploadDuration.Milliseconds(), } - if err := s.repos.Uploads.Create(txCtx, tx, upload); err != nil { + + err := s.repos.Uploads.Create(txCtx, tx, upload) + if err != nil { return fmt.Errorf("recording upload metrics: %w", err) } } @@ -1312,10 +1434,14 @@ func (s *Scanner) recordBlobMetadata(ctx context.Context, finishedBlob *blob.Fin func (s *Scanner) cleanupBlobTempFile(blobWithReader *blob.BlobWithReader) { if blobWithReader.TempFile != nil { tempName := blobWithReader.TempFile.Name() - if err := blobWithReader.TempFile.Close(); err != nil { + + err := blobWithReader.TempFile.Close() + if err != nil { log.Fatal("Failed to close temp file", "file", tempName, "error", err) } - if err := s.fs.Remove(tempName); err != nil { + + err = s.fs.Remove(tempName) + if err != nil { log.Fatal("Failed to remove temp file", "file", tempName, "error", err) } } @@ -1343,6 +1469,7 @@ func (s *Scanner) processFileStreaming(ctx context.Context, fileToProcess *FileT defer func() { _ = file.Close() }() var chunks []streamingChunkInfo + chunkIndex := 0 fileHash, err := s.chunker.ChunkReaderStreaming(file, func(chunk chunker.Chunk) error { @@ -1372,16 +1499,17 @@ func (s *Scanner) processFileStreaming(ctx context.Context, fileToProcess *FileT s.updateChunkStats(chunkExists, chunk.Size, result) if !chunkExists { - if err := s.addChunkToPacker(chunk); err != nil { + err := s.addChunkToPacker(chunk) + if err != nil { return err } } chunk.Data = nil chunkIndex++ + return nil }) - if err != nil { return fmt.Errorf("chunking file: %w", err) } @@ -1390,6 +1518,7 @@ func (s *Scanner) processFileStreaming(ctx context.Context, fileToProcess *FileT "path", fileToProcess.Path, "file_hash", fileHash, "chunks", len(chunks)) s.queueFileForBatchInsert(ctx, fileToProcess, chunks) + return nil } @@ -1397,6 +1526,7 @@ func (s *Scanner) processFileStreaming(ctx context.Context, fileToProcess *FileT func (s *Scanner) updateChunkStats(chunkExists bool, chunkSize int64, result *ScanResult) { if chunkExists { result.FilesSkipped++ + result.BytesSkipped += chunkSize if s.progress != nil { s.progress.GetStats().BytesSkipped.Add(chunkSize) @@ -1404,6 +1534,7 @@ func (s *Scanner) updateChunkStats(chunkExists bool, chunkSize int64, result *Sc } else { result.ChunksCreated++ result.BytesScanned += chunkSize + if s.progress != nil { s.progress.GetStats().ChunksCreated.Add(1) s.progress.GetStats().BytesProcessed.Add(chunkSize) @@ -1415,27 +1546,36 @@ func (s *Scanner) updateChunkStats(chunkExists bool, chunkSize int64, result *Sc // addChunkToPacker adds a chunk to the blob packer, finalizing the current blob if needed func (s *Scanner) addChunkToPacker(chunk chunker.Chunk) error { s.packerMu.Lock() + err := s.packer.AddChunk(&blob.ChunkRef{Hash: chunk.Hash, Data: chunk.Data}) - if err == blob.ErrBlobSizeLimitExceeded { - if err := s.packer.FinalizeBlob(); err != nil { + if errors.Is(err, blob.ErrBlobSizeLimitExceeded) { + err := s.packer.FinalizeBlob() + if err != nil { s.packerMu.Unlock() + return fmt.Errorf("finalizing blob: %w", err) } - if err := s.packer.AddChunk(&blob.ChunkRef{Hash: chunk.Hash, Data: chunk.Data}); err != nil { + + err = s.packer.AddChunk(&blob.ChunkRef{Hash: chunk.Hash, Data: chunk.Data}) + if err != nil { s.packerMu.Unlock() + return fmt.Errorf("adding chunk after finalize: %w", err) } } else if err != nil { s.packerMu.Unlock() + return fmt.Errorf("adding chunk to packer: %w", err) } s.packerMu.Unlock() + return nil } // queueFileForBatchInsert builds file/chunk associations and queues the file for batch DB insert func (s *Scanner) queueFileForBatchInsert(ctx context.Context, fileToProcess *FileToProcess, chunks []streamingChunkInfo) { fileChunks := make([]database.FileChunk, len(chunks)) + chunkFiles := make([]database.ChunkFile, len(chunks)) for i, ci := range chunks { fileChunks[i] = database.FileChunk{ @@ -1489,7 +1629,7 @@ func (s *Scanner) detectDeletedFilesFromMap(ctx context.Context, knownFiles map[ } if result.FilesDeleted > 0 { - s.ui.Info("Snapshot source files enumeration detected %s deleted files.", s.ui.Count(result.FilesDeleted)) + s.ui.Infof("Snapshot source files enumeration detected %s deleted files.", s.ui.Count(result.FilesDeleted)) } return nil @@ -1503,6 +1643,7 @@ func wrapPermissionError(path string, err error) error { if !errors.Is(err, os.ErrPermission) { return err } + if runtime.GOOS == "darwin" { return fmt.Errorf("cannot read %s: %w\n\n"+ "macOS is blocking access to this path. Grant Full Disk Access to your\n"+ @@ -1510,12 +1651,14 @@ func wrapPermissionError(path string, err error) error { " System Settings → Privacy & Security → Full Disk Access\n\n"+ "then quit and reopen the terminal and re-run the backup", path, err) } + return fmt.Errorf("cannot read %s: %w (check file permissions, or run with --skip-errors to continue past unreadable files)", path, err) } // compileExcludePatterns compiles the exclude patterns into glob matchers func compileExcludePatterns(patterns []string) []compiledPattern { var compiled []compiledPattern + for _, p := range patterns { if p == "" { continue @@ -1523,6 +1666,7 @@ func compileExcludePatterns(patterns []string) []compiledPattern { // Check if pattern is anchored (starts with /) anchored := strings.HasPrefix(p, "/") + pattern := p if anchored { pattern = p[1:] // Remove leading / @@ -1537,6 +1681,7 @@ func compileExcludePatterns(patterns []string) []compiledPattern { g, err := glob.Compile(pattern, '/') if err != nil { log.Warn("Invalid exclude pattern, skipping", "pattern", p, "error", err) + continue } @@ -1546,6 +1691,7 @@ func compileExcludePatterns(patterns []string) []compiledPattern { original: p, }) } + return compiled } diff --git a/internal/snapshot/scanner_test.go b/internal/snapshot/scanner_test.go index 4ceaf84..4ed2d3b 100644 --- a/internal/snapshot/scanner_test.go +++ b/internal/snapshot/scanner_test.go @@ -33,16 +33,22 @@ func TestScannerSimpleDirectory(t *testing.T) { // Create files with specific times testTime := time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC) + for path, content := range testFiles { dir := filepath.Dir(path) - if err := fs.MkdirAll(dir, 0755); err != nil { + + err := fs.MkdirAll(dir, 0755) + if err != nil { t.Fatalf("failed to create directory %s: %v", dir, err) } - if err := afero.WriteFile(fs, path, []byte(content), 0644); err != nil { + + err = afero.WriteFile(fs, path, []byte(content), 0644) + if err != nil { t.Fatalf("failed to write file %s: %v", path, err) } // Set times - if err := fs.Chtimes(path, testTime, testTime); err != nil { + err = fs.Chtimes(path, testTime, testTime) + if err != nil { t.Fatalf("failed to set times for %s: %v", path, err) } } @@ -53,7 +59,8 @@ func TestScannerSimpleDirectory(t *testing.T) { t.Fatalf("failed to create test database: %v", err) } defer func() { - if err := db.Close(); err != nil { + err := db.Close() + if err != nil { t.Errorf("failed to close database: %v", err) } }() @@ -73,6 +80,7 @@ func TestScannerSimpleDirectory(t *testing.T) { // Create a snapshot record for testing ctx := context.Background() snapshotID := "test-snapshot-001" + err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { snapshot := &database.Snapshot{ ID: types.SnapshotID(snapshotID), @@ -87,6 +95,7 @@ func TestScannerSimpleDirectory(t *testing.T) { BlobSize: 0, CompressionRatio: 1.0, } + return repos.Snapshots.Create(ctx, tx, snapshot) }) if err != nil { @@ -95,6 +104,7 @@ func TestScannerSimpleDirectory(t *testing.T) { // Scan the directory var result *snapshot.ScanResult + result, err = scanner.Scan(ctx, "/source", snapshotID) if err != nil { t.Fatalf("scan failed: %v", err) @@ -170,7 +180,7 @@ func TestScannerLargeFile(t *testing.T) { // Use random content to ensure good chunk boundaries largeContent := make([]byte, 1024*1024) // 1MB // Fill with pseudo-random data to ensure chunk boundaries - for i := 0; i < len(largeContent); i++ { + for i := range largeContent { // Simple pseudo-random generator for deterministic tests largeContent[i] = byte((i * 7919) ^ (i >> 3)) } @@ -178,6 +188,7 @@ func TestScannerLargeFile(t *testing.T) { if err := fs.MkdirAll("/source", 0755); err != nil { t.Fatal(err) } + if err := afero.WriteFile(fs, "/source/large.bin", largeContent, 0644); err != nil { t.Fatal(err) } @@ -188,7 +199,8 @@ func TestScannerLargeFile(t *testing.T) { t.Fatalf("failed to create test database: %v", err) } defer func() { - if err := db.Close(); err != nil { + err := db.Close() + if err != nil { t.Errorf("failed to close database: %v", err) } }() @@ -208,6 +220,7 @@ func TestScannerLargeFile(t *testing.T) { // Create a snapshot record for testing ctx := context.Background() snapshotID := "test-snapshot-001" + err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { snapshot := &database.Snapshot{ ID: types.SnapshotID(snapshotID), @@ -222,6 +235,7 @@ func TestScannerLargeFile(t *testing.T) { BlobSize: 0, CompressionRatio: 1.0, } + return repos.Snapshots.Create(ctx, tx, snapshot) }) if err != nil { @@ -230,6 +244,7 @@ func TestScannerLargeFile(t *testing.T) { // Scan the directory var result *snapshot.ScanResult + result, err = scanner.Scan(ctx, "/source", snapshotID) if err != nil { t.Fatalf("scan failed: %v", err) diff --git a/internal/snapshot/snapshot.go b/internal/snapshot/snapshot.go index f3f26d6..545d6e7 100644 --- a/internal/snapshot/snapshot.go +++ b/internal/snapshot/snapshot.go @@ -37,6 +37,8 @@ import ( "bytes" "context" "database/sql" + "encoding/hex" + "errors" "fmt" "io" "os/exec" @@ -97,12 +99,13 @@ func (sm *SnapshotManager) CreateSnapshot(ctx context.Context, hostname, version func (sm *SnapshotManager) CreateSnapshotWithName(ctx context.Context, hostname, name, version, gitRevision string) (string, error) { // Use short hostname (strip domain if present) shortHostname := hostname - if idx := strings.Index(hostname, "."); idx != -1 { - shortHostname = hostname[:idx] + if before, _, ok := strings.Cut(hostname, "."); ok { + shortHostname = before } // Build snapshot ID with optional name timestamp := time.Now().UTC().Format("2006-01-02T15:04:05Z") + var snapshotID string if name != "" { snapshotID = fmt.Sprintf("%s_%s_%s", shortHostname, name, timestamp) @@ -128,12 +131,12 @@ func (sm *SnapshotManager) CreateSnapshotWithName(ctx context.Context, hostname, err := sm.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { return sm.repos.Snapshots.Create(ctx, tx, snapshot) }) - if err != nil { return "", fmt.Errorf("creating snapshot: %w", err) } log.Info("Created snapshot", "snapshot_id", snapshotID) + return snapshotID, nil } @@ -148,7 +151,6 @@ func (sm *SnapshotManager) UpdateSnapshotStats(ctx context.Context, snapshotID s stats.BytesUploaded, ) }) - if err != nil { return fmt.Errorf("updating snapshot stats: %w", err) } @@ -161,13 +163,14 @@ func (sm *SnapshotManager) UpdateSnapshotStats(ctx context.Context, snapshotID s func (sm *SnapshotManager) UpdateSnapshotStatsExtended(ctx context.Context, snapshotID string, stats ExtendedBackupStats) error { return sm.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { // First update basic stats - if err := sm.repos.Snapshots.UpdateCounts(ctx, tx, snapshotID, + err := sm.repos.Snapshots.UpdateCounts(ctx, tx, snapshotID, int64(stats.FilesScanned), int64(stats.ChunksCreated), int64(stats.BlobsCreated), stats.BytesScanned, stats.BytesUploaded, - ); err != nil { + ) + if err != nil { return err } @@ -190,18 +193,20 @@ func (sm *SnapshotManager) CompleteSnapshot(ctx context.Context, snapshotID stri if err != nil { return err } + if added > 0 { log.Info("Populated snapshot_blobs with dedup-referenced blobs", "snapshot_id", snapshotID, "added", added) } + return sm.repos.Snapshots.MarkComplete(ctx, tx, snapshotID) }) - if err != nil { return fmt.Errorf("marking snapshot complete: %w", err) } log.Info("Completed snapshot", "snapshot_id", snapshotID) + return nil } @@ -229,10 +234,13 @@ func (sm *SnapshotManager) ExportSnapshotMetadata(ctx context.Context, dbPath st if err != nil { return fmt.Errorf("creating temp dir: %w", err) } + log.Debug("Created temporary directory", "path", tempDir) defer func() { log.Debug("Cleaning up temporary directory", "path", tempDir) - if err := sm.fs.RemoveAll(tempDir); err != nil { + + err := sm.fs.RemoveAll(tempDir) + if err != nil { log.Debug("Failed to remove temp dir", "path", tempDir, "error", err) } }() @@ -258,6 +266,7 @@ func (sm *SnapshotManager) ExportSnapshotMetadata(ctx context.Context, dbPath st "snapshot_id", snapshotID, "db_size", len(finalData), "manifest_size", len(blobManifest)) + return nil } @@ -268,17 +277,21 @@ func (sm *SnapshotManager) prepareExportDB(ctx context.Context, dbPath, snapshot // The main database should be closed at this point tempDBPath := filepath.Join(tempDir, "snapshot.db") log.Debug("Copying database to temporary location", "source", dbPath, "destination", tempDBPath) + if err := sm.copyFile(dbPath, tempDBPath); err != nil { return nil, "", fmt.Errorf("copying database: %w", err) } + log.Debug("Database copy complete", "size", sm.getFileSize(tempDBPath)) // Step 2: Clean the temp database to only contain current snapshot data log.Debug("Cleaning temporary database", "snapshot_id", snapshotID) + stats, err := sm.cleanSnapshotDB(ctx, tempDBPath, snapshotID) if err != nil { return nil, "", fmt.Errorf("cleaning snapshot database: %w", err) } + log.Info("Temporary database cleanup complete", "db_path", tempDBPath, "size_after_clean", humanize.Bytes(uint64(sm.getFileSize(tempDBPath))), @@ -294,6 +307,7 @@ func (sm *SnapshotManager) prepareExportDB(ctx context.Context, dbPath, snapshot if err := sm.vacuumDatabase(tempDBPath); err != nil { return nil, "", fmt.Errorf("vacuuming database: %w", err) } + log.Debug("Database vacuumed", "size", humanize.Bytes(uint64(sm.getFileSize(tempDBPath)))) // Step 4: Compress and encrypt the binary database file @@ -301,6 +315,7 @@ func (sm *SnapshotManager) prepareExportDB(ctx context.Context, dbPath, snapshot if err := sm.compressFile(tempDBPath, compressedPath); err != nil { return nil, "", fmt.Errorf("compressing database: %w", err) } + log.Debug("Compression complete", "original_size", humanize.Bytes(uint64(sm.getFileSize(tempDBPath))), "compressed_size", humanize.Bytes(uint64(sm.getFileSize(compressedPath)))) @@ -327,9 +342,12 @@ func (sm *SnapshotManager) uploadSnapshotArtifacts(ctx context.Context, snapshot dbKey := fmt.Sprintf("metadata/%s/db.zst.age", remoteKey) dbUploadStart := time.Now() - if err := sm.storage.Put(ctx, dbKey, bytes.NewReader(dbData)); err != nil { + + err := sm.storage.Put(ctx, dbKey, bytes.NewReader(dbData)) + if err != nil { return fmt.Errorf("uploading snapshot database: %w", err) } + dbUploadDuration := time.Since(dbUploadStart) dbUploadSpeed := float64(len(dbData)) * 8 / dbUploadDuration.Seconds() // bits per second log.Info("Uploaded snapshot database", @@ -341,9 +359,12 @@ func (sm *SnapshotManager) uploadSnapshotArtifacts(ctx context.Context, snapshot // Upload blob manifest (compressed only, not encrypted) manifestKey := fmt.Sprintf("metadata/%s/manifest.json.zst", remoteKey) manifestUploadStart := time.Now() - if err := sm.storage.Put(ctx, manifestKey, bytes.NewReader(manifestData)); err != nil { + + err = sm.storage.Put(ctx, manifestKey, bytes.NewReader(manifestData)) + if err != nil { return fmt.Errorf("uploading blob manifest: %w", err) } + manifestUploadDuration := time.Since(manifestUploadStart) manifestUploadSpeed := float64(len(manifestData)) * 8 / manifestUploadDuration.Seconds() // bits per second log.Info("Uploaded blob manifest", @@ -383,7 +404,8 @@ func (sm *SnapshotManager) cleanSnapshotDB(ctx context.Context, dbPath string, s return nil, fmt.Errorf("opening temp database: %w", err) } defer func() { - if err := db.Close(); err != nil { + err := db.Close() + if err != nil { log.Debug("Failed to close temp database", "error", err) } }() @@ -394,7 +416,8 @@ func (sm *SnapshotManager) cleanSnapshotDB(ctx context.Context, dbPath string, s return nil, fmt.Errorf("beginning transaction: %w", err) } defer func() { - if rbErr := tx.Rollback(); rbErr != nil && rbErr != sql.ErrTxDone { + rbErr := tx.Rollback() + if rbErr != nil && !errors.Is(rbErr, sql.ErrTxDone) { log.Debug("Failed to rollback transaction", "error", rbErr) } }() @@ -430,6 +453,7 @@ func (sm *SnapshotManager) cleanSnapshotDB(ctx context.Context, dbPath string, s // Commit transaction log.Debug("[Temp DB Cleanup] Committing cleanup transaction") + if err := tx.Commit(); err != nil { return nil, fmt.Errorf("committing transaction: %w", err) } @@ -439,23 +463,30 @@ func (sm *SnapshotManager) cleanSnapshotDB(ctx context.Context, dbPath string, s // Count files var fileCount int + err = db.QueryRowWithLog(ctx, "SELECT COUNT(*) FROM files").Scan(&fileCount) if err != nil { return nil, fmt.Errorf("counting files: %w", err) } + stats.FileCount = fileCount // Count chunks var chunkCount int + err = db.QueryRowWithLog(ctx, "SELECT COUNT(*) FROM chunks").Scan(&chunkCount) if err != nil { return nil, fmt.Errorf("counting chunks: %w", err) } + stats.ChunkCount = chunkCount // Count blobs and get sizes - var blobCount int - var compressedSize, uncompressedSize sql.NullInt64 + var ( + blobCount int + compressedSize, uncompressedSize sql.NullInt64 + ) + err = db.QueryRowWithLog(ctx, ` SELECT COUNT(*), COALESCE(SUM(compressed_size), 0), COALESCE(SUM(uncompressed_size), 0) FROM blobs @@ -464,6 +495,7 @@ func (sm *SnapshotManager) cleanSnapshotDB(ctx context.Context, dbPath string, s if err != nil { return nil, fmt.Errorf("counting blobs and sizes: %w", err) } + stats.BlobCount = blobCount stats.CompressedSize = compressedSize.Int64 stats.UncompressedSize = uncompressedSize.Int64 @@ -491,7 +523,8 @@ func (sm *SnapshotManager) compressFile(inputPath, outputPath string) error { return fmt.Errorf("opening input file: %w", err) } defer func() { - if err := input.Close(); err != nil { + err := input.Close() + if err != nil { log.Debug("Failed to close input file", "path", inputPath, "error", err) } }() @@ -501,13 +534,15 @@ func (sm *SnapshotManager) compressFile(inputPath, outputPath string) error { return fmt.Errorf("creating output file: %w", err) } defer func() { - if err := output.Close(); err != nil { + err := output.Close() + if err != nil { log.Debug("Failed to close output file", "path", outputPath, "error", err) } }() // Use blobgen for compression and encryption log.Debug("Compressing and encrypting data") + writer, err := blobgen.NewWriter(output, sm.config.CompressionLevel, sm.config.AgeRecipients) if err != nil { return fmt.Errorf("creating blobgen writer: %w", err) @@ -517,7 +552,8 @@ func (sm *SnapshotManager) compressFile(inputPath, outputPath string) error { writerClosed := false defer func() { if !writerClosed { - if err := writer.Close(); err != nil { + err := writer.Close() + if err != nil { log.Debug("Failed to close writer", "error", err) } } @@ -531,9 +567,10 @@ func (sm *SnapshotManager) compressFile(inputPath, outputPath string) error { if err := writer.Close(); err != nil { return fmt.Errorf("closing writer: %w", err) } + writerClosed = true - log.Debug("Compression complete", "hash", fmt.Sprintf("%x", writer.Sum256())) + log.Debug("Compression complete", "hash", hex.EncodeToString(writer.Sum256())) return nil } @@ -541,34 +578,42 @@ func (sm *SnapshotManager) compressFile(inputPath, outputPath string) error { // copyFile copies a file from src to dst func (sm *SnapshotManager) copyFile(src, dst string) error { log.Debug("Opening source file for copy", "path", src) + sourceFile, err := sm.fs.Open(src) if err != nil { return err } defer func() { log.Debug("Closing source file", "path", src) - if err := sourceFile.Close(); err != nil { + + err := sourceFile.Close() + if err != nil { log.Debug("Failed to close source file", "path", src, "error", err) } }() log.Debug("Creating destination file", "path", dst) + destFile, err := sm.fs.Create(dst) if err != nil { return err } defer func() { log.Debug("Closing destination file", "path", dst) - if err := destFile.Close(); err != nil { + + err := destFile.Close() + if err != nil { log.Debug("Failed to close destination file", "path", dst, "error", err) } }() log.Debug("Copying file data") + n, err := io.Copy(destFile, sourceFile) if err != nil { return err } + log.Debug("File copy complete", "bytes_copied", n) return nil @@ -576,7 +621,6 @@ func (sm *SnapshotManager) copyFile(src, dst string) error { // generateBlobManifest creates a compressed JSON list of all blobs in the snapshot func (sm *SnapshotManager) generateBlobManifest(ctx context.Context, dbPath string, snapshotID string) ([]byte, error) { - // Open the cleaned database using the database package db, err := database.New(ctx, dbPath) if err != nil { @@ -589,10 +633,12 @@ func (sm *SnapshotManager) generateBlobManifest(ctx context.Context, dbPath stri // Get all blobs for this snapshot log.Debug("Querying blobs for snapshot", "snapshot_id", snapshotID) + blobHashes, err := repos.Snapshots.GetBlobHashes(ctx, snapshotID) if err != nil { return nil, fmt.Errorf("getting snapshot blobs: %w", err) } + log.Debug("Found blobs", "count", len(blobHashes)) // Get blob details including sizes @@ -603,8 +649,10 @@ func (sm *SnapshotManager) generateBlobManifest(ctx context.Context, dbPath stri blob, err := repos.Blobs.GetByHash(ctx, hash) if err != nil { log.Warn("Failed to get blob details", "hash", hash, "error", err) + continue } + if blob != nil { blobs = append(blobs, BlobInfo{ Hash: hash, @@ -648,6 +696,7 @@ func (sm *SnapshotManager) getFileSize(path string) int64 { if err != nil { return -1 } + return info.Size() } @@ -663,6 +712,7 @@ type BackupStats struct { // ExtendedBackupStats contains additional statistics for comprehensive tracking type ExtendedBackupStats struct { BackupStats + BlobUncompressedSize int64 // Total uncompressed size of all referenced blobs CompressionLevel int // Compression level used for this snapshot UploadDurationMs int64 // Total milliseconds spent uploading to S3 @@ -682,6 +732,7 @@ func (sm *SnapshotManager) CleanupIncompleteSnapshots(ctx context.Context, hostn if len(incompleteSnapshots) == 0 { log.Debug("No incomplete snapshots found") + return nil } @@ -692,14 +743,15 @@ func (sm *SnapshotManager) CleanupIncompleteSnapshots(ctx context.Context, hostn // Check if metadata exists in storage (paths use the hashed // remote key so we don't leak host info to the listing). metadataKey := fmt.Sprintf("metadata/%s/db.zst", RemoteSnapshotKey(snapshot.ID.String())) - _, err := sm.storage.Stat(ctx, metadataKey) + _, err := sm.storage.Stat(ctx, metadataKey) if err != nil { // Metadata doesn't exist in S3 - this is an incomplete snapshot log.Info("Cleaning up incomplete snapshot record", "snapshot_id", snapshot.ID, "started_at", snapshot.StartedAt) // Delete the snapshot and all its associations - if err := sm.deleteSnapshot(ctx, snapshot.ID.String()); err != nil { + err := sm.deleteSnapshot(ctx, snapshot.ID.String()) + if err != nil { return fmt.Errorf("deleting incomplete snapshot %s: %w", snapshot.ID, err) } @@ -708,7 +760,9 @@ func (sm *SnapshotManager) CleanupIncompleteSnapshots(ctx context.Context, hostn // Metadata exists - this snapshot was completed but database wasn't updated // This shouldn't happen in normal operation, but mark it complete log.Warn("Found snapshot with remote metadata but incomplete in database", "snapshot_id", snapshot.ID) - if err := sm.repos.Snapshots.MarkComplete(ctx, nil, snapshot.ID.String()); err != nil { + + err := sm.repos.Snapshots.MarkComplete(ctx, nil, snapshot.ID.String()) + if err != nil { log.Error("Failed to mark snapshot as complete in database", "snapshot_id", snapshot.ID, "error", err) } } @@ -720,28 +774,34 @@ func (sm *SnapshotManager) CleanupIncompleteSnapshots(ctx context.Context, hostn // deleteSnapshot removes a snapshot and all its associations from the database func (sm *SnapshotManager) deleteSnapshot(ctx context.Context, snapshotID string) error { // Delete snapshot_files entries - if err := sm.repos.Snapshots.DeleteSnapshotFiles(ctx, snapshotID); err != nil { + err := sm.repos.Snapshots.DeleteSnapshotFiles(ctx, snapshotID) + if err != nil { return fmt.Errorf("deleting snapshot files: %w", err) } // Delete snapshot_blobs entries - if err := sm.repos.Snapshots.DeleteSnapshotBlobs(ctx, snapshotID); err != nil { + err = sm.repos.Snapshots.DeleteSnapshotBlobs(ctx, snapshotID) + if err != nil { return fmt.Errorf("deleting snapshot blobs: %w", err) } // Delete uploads entries (has foreign key to snapshots without CASCADE) - if err := sm.repos.Snapshots.DeleteSnapshotUploads(ctx, snapshotID); err != nil { + err = sm.repos.Snapshots.DeleteSnapshotUploads(ctx, snapshotID) + if err != nil { return fmt.Errorf("deleting snapshot uploads: %w", err) } // Delete the snapshot itself - if err := sm.repos.Snapshots.Delete(ctx, snapshotID); err != nil { + err = sm.repos.Snapshots.Delete(ctx, snapshotID) + if err != nil { return fmt.Errorf("deleting snapshot: %w", err) } // Clean up orphaned data log.Debug("Cleaning up orphaned records in main database") - if err := sm.CleanupOrphanedData(ctx); err != nil { + + err = sm.CleanupOrphanedData(ctx) + if err != nil { return fmt.Errorf("cleaning up orphaned data: %w", err) } @@ -759,28 +819,36 @@ func (sm *SnapshotManager) CleanupOrphanedData(ctx context.Context) error { // Delete orphaned files (files not in any snapshot) log.Debug("Deleting orphaned file records from database") - if err := sm.repos.Files.DeleteOrphaned(ctx); err != nil { + + err := sm.repos.Files.DeleteOrphaned(ctx) + if err != nil { return fmt.Errorf("deleting orphaned files: %w", err) } // Delete orphaned blobs (blobs not in any snapshot) // This will cascade delete blob_chunks for deleted blobs log.Debug("Deleting orphaned blob records from database") - if err := sm.repos.Blobs.DeleteOrphaned(ctx); err != nil { + + err = sm.repos.Blobs.DeleteOrphaned(ctx) + if err != nil { return fmt.Errorf("deleting orphaned blobs: %w", err) } // Delete orphaned blob_chunks entries // This handles cases where the blob still exists but chunks were deleted log.Debug("Deleting orphaned blob_chunks associations from database") - if err := sm.repos.BlobChunks.DeleteOrphaned(ctx); err != nil { + + err = sm.repos.BlobChunks.DeleteOrphaned(ctx) + if err != nil { return fmt.Errorf("deleting orphaned blob_chunks: %w", err) } // Delete orphaned chunks (chunks not referenced by any file) // This must come after cleaning up blob_chunks to avoid foreign key violations log.Debug("Deleting orphaned chunk records from database") - if err := sm.repos.Chunks.DeleteOrphaned(ctx); err != nil { + + err = sm.repos.Chunks.DeleteOrphaned(ctx) + if err != nil { return fmt.Errorf("deleting orphaned chunks: %w", err) } @@ -793,21 +861,26 @@ func (sm *SnapshotManager) deleteOtherSnapshots(ctx context.Context, tx *sql.Tx, // First delete uploads that reference other snapshots (no CASCADE DELETE on this FK) database.LogSQL("Execute", "DELETE FROM uploads WHERE snapshot_id != ?", currentSnapshotID) + uploadResult, err := tx.ExecContext(ctx, "DELETE FROM uploads WHERE snapshot_id != ?", currentSnapshotID) if err != nil { return fmt.Errorf("deleting uploads for other snapshots: %w", err) } + uploadsDeleted, _ := uploadResult.RowsAffected() log.Debug("[Temp DB Cleanup] Deleted upload records", "count", uploadsDeleted) // Now we can safely delete the snapshots database.LogSQL("Execute", "DELETE FROM snapshots WHERE id != ?", currentSnapshotID) + result, err := tx.ExecContext(ctx, "DELETE FROM snapshots WHERE id != ?", currentSnapshotID) if err != nil { return fmt.Errorf("deleting other snapshots: %w", err) } + rowsAffected, _ := result.RowsAffected() log.Debug("[Temp DB Cleanup] Deleted snapshot records from database", "count", rowsAffected) + return nil } @@ -816,22 +889,27 @@ func (sm *SnapshotManager) deleteOrphanedSnapshotAssociations(ctx context.Contex // Delete orphaned snapshot_files log.Debug("[Temp DB Cleanup] Deleting orphaned snapshot_files associations") database.LogSQL("Execute", "DELETE FROM snapshot_files WHERE snapshot_id != ?", currentSnapshotID) + result, err := tx.ExecContext(ctx, "DELETE FROM snapshot_files WHERE snapshot_id != ?", currentSnapshotID) if err != nil { return fmt.Errorf("deleting orphaned snapshot_files: %w", err) } + rowsAffected, _ := result.RowsAffected() log.Debug("[Temp DB Cleanup] Deleted snapshot_files associations", "count", rowsAffected) // Delete orphaned snapshot_blobs log.Debug("[Temp DB Cleanup] Deleting orphaned snapshot_blobs associations") database.LogSQL("Execute", "DELETE FROM snapshot_blobs WHERE snapshot_id != ?", currentSnapshotID) + result, err = tx.ExecContext(ctx, "DELETE FROM snapshot_blobs WHERE snapshot_id != ?", currentSnapshotID) if err != nil { return fmt.Errorf("deleting orphaned snapshot_blobs: %w", err) } + rowsAffected, _ = result.RowsAffected() log.Debug("[Temp DB Cleanup] Deleted snapshot_blobs associations", "count", rowsAffected) + return nil } @@ -839,6 +917,7 @@ func (sm *SnapshotManager) deleteOrphanedSnapshotAssociations(ctx context.Contex func (sm *SnapshotManager) deleteOrphanedFiles(ctx context.Context, tx *sql.Tx, currentSnapshotID string) error { log.Debug("[Temp DB Cleanup] Deleting file records not referenced by current snapshot") database.LogSQL("Execute", `DELETE FROM files WHERE NOT EXISTS (SELECT 1 FROM snapshot_files WHERE snapshot_files.file_id = files.id AND snapshot_files.snapshot_id = ?)`, currentSnapshotID) + result, err := tx.ExecContext(ctx, ` DELETE FROM files WHERE NOT EXISTS ( @@ -849,11 +928,13 @@ func (sm *SnapshotManager) deleteOrphanedFiles(ctx context.Context, tx *sql.Tx, if err != nil { return fmt.Errorf("deleting orphaned files: %w", err) } + rowsAffected, _ := result.RowsAffected() log.Debug("[Temp DB Cleanup] Deleted file records from database", "count", rowsAffected) // Note: file_chunks will be deleted via CASCADE log.Debug("[Temp DB Cleanup] file_chunks associations deleted via CASCADE") + return nil } @@ -861,6 +942,7 @@ func (sm *SnapshotManager) deleteOrphanedFiles(ctx context.Context, tx *sql.Tx, func (sm *SnapshotManager) deleteOrphanedChunkToFileMappings(ctx context.Context, tx *sql.Tx) error { log.Debug("[Temp DB Cleanup] Deleting orphaned chunk_files associations") database.LogSQL("Execute", `DELETE FROM chunk_files WHERE NOT EXISTS (SELECT 1 FROM files WHERE files.id = chunk_files.file_id)`) + result, err := tx.ExecContext(ctx, ` DELETE FROM chunk_files WHERE NOT EXISTS ( @@ -870,8 +952,10 @@ func (sm *SnapshotManager) deleteOrphanedChunkToFileMappings(ctx context.Context if err != nil { return fmt.Errorf("deleting orphaned chunk_files: %w", err) } + rowsAffected, _ := result.RowsAffected() log.Debug("[Temp DB Cleanup] Deleted chunk_files associations", "count", rowsAffected) + return nil } @@ -879,6 +963,7 @@ func (sm *SnapshotManager) deleteOrphanedChunkToFileMappings(ctx context.Context func (sm *SnapshotManager) deleteOrphanedBlobs(ctx context.Context, tx *sql.Tx, currentSnapshotID string) error { log.Debug("[Temp DB Cleanup] Deleting blob records not referenced by current snapshot") database.LogSQL("Execute", `DELETE FROM blobs WHERE NOT EXISTS (SELECT 1 FROM snapshot_blobs WHERE snapshot_blobs.blob_hash = blobs.blob_hash AND snapshot_blobs.snapshot_id = ?)`, currentSnapshotID) + result, err := tx.ExecContext(ctx, ` DELETE FROM blobs WHERE NOT EXISTS ( @@ -889,8 +974,10 @@ func (sm *SnapshotManager) deleteOrphanedBlobs(ctx context.Context, tx *sql.Tx, if err != nil { return fmt.Errorf("deleting orphaned blobs: %w", err) } + rowsAffected, _ := result.RowsAffected() log.Debug("[Temp DB Cleanup] Deleted blob records from database", "count", rowsAffected) + return nil } @@ -898,6 +985,7 @@ func (sm *SnapshotManager) deleteOrphanedBlobs(ctx context.Context, tx *sql.Tx, func (sm *SnapshotManager) deleteOrphanedBlobToChunkMappings(ctx context.Context, tx *sql.Tx) error { log.Debug("[Temp DB Cleanup] Deleting orphaned blob_chunks associations") database.LogSQL("Execute", `DELETE FROM blob_chunks WHERE NOT EXISTS (SELECT 1 FROM blobs WHERE blobs.id = blob_chunks.blob_id)`) + result, err := tx.ExecContext(ctx, ` DELETE FROM blob_chunks WHERE NOT EXISTS ( @@ -907,14 +995,17 @@ func (sm *SnapshotManager) deleteOrphanedBlobToChunkMappings(ctx context.Context if err != nil { return fmt.Errorf("deleting orphaned blob_chunks: %w", err) } + rowsAffected, _ := result.RowsAffected() log.Debug("[Temp DB Cleanup] Deleted blob_chunks associations", "count", rowsAffected) + return nil } // deleteOrphanedChunks deletes chunks not referenced by any file or blob func (sm *SnapshotManager) deleteOrphanedChunks(ctx context.Context, tx *sql.Tx) error { log.Debug("[Temp DB Cleanup] Deleting orphaned chunk records") + query := ` DELETE FROM chunks WHERE NOT EXISTS ( @@ -926,11 +1017,14 @@ func (sm *SnapshotManager) deleteOrphanedChunks(ctx context.Context, tx *sql.Tx) WHERE blob_chunks.chunk_hash = chunks.chunk_hash )` database.LogSQL("Execute", query) + result, err := tx.ExecContext(ctx, query) if err != nil { return fmt.Errorf("deleting orphaned chunks: %w", err) } + rowsAffected, _ := result.RowsAffected() log.Debug("[Temp DB Cleanup] Deleted chunk records from database", "count", rowsAffected) + return nil } diff --git a/internal/snapshot/snapshot_test.go b/internal/snapshot/snapshot_test.go index a47df4a..969aeed 100644 --- a/internal/snapshot/snapshot_test.go +++ b/internal/snapshot/snapshot_test.go @@ -33,6 +33,7 @@ func copyFile(fs afero.Fs, src, dst string) error { defer func() { _ = destFile.Close() }() _, err = io.Copy(destFile, sourceFile) + return err } @@ -46,6 +47,7 @@ func TestCleanSnapshotDBEmptySnapshot(t *testing.T) { // Create a test database tempDir := t.TempDir() dbPath := filepath.Join(tempDir, "test.db") + db, err := database.New(ctx, dbPath) if err != nil { t.Fatalf("failed to create database: %v", err) @@ -71,9 +73,11 @@ func TestCleanSnapshotDBEmptySnapshot(t *testing.T) { chunk := &database.Chunk{ChunkHash: "orphan-chunk", Size: 500} err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { - if err := repos.Files.Create(ctx, tx, file); err != nil { + err := repos.Files.Create(ctx, tx, file) + if err != nil { return err } + return repos.Chunks.Create(ctx, tx, chunk) }) if err != nil { @@ -111,7 +115,8 @@ func TestCleanSnapshotDBEmptySnapshot(t *testing.T) { t.Fatalf("failed to open cleaned database: %v", err) } defer func() { - if err := cleanedDB.Close(); err != nil { + err := cleanedDB.Close() + if err != nil { t.Errorf("failed to close database: %v", err) } }() @@ -123,6 +128,7 @@ func TestCleanSnapshotDBEmptySnapshot(t *testing.T) { if err != nil { t.Fatalf("failed to get snapshot: %v", err) } + if verifySnapshot == nil { t.Error("snapshot should exist") } @@ -132,6 +138,7 @@ func TestCleanSnapshotDBEmptySnapshot(t *testing.T) { if err != nil { t.Fatalf("failed to check file: %v", err) } + if f != nil { t.Error("orphan file should not exist") } @@ -141,6 +148,7 @@ func TestCleanSnapshotDBEmptySnapshot(t *testing.T) { if err != nil { t.Fatalf("failed to check chunk: %v", err) } + if c != nil { t.Error("orphan chunk should not exist") } @@ -156,6 +164,7 @@ func TestCleanSnapshotDBNonExistentSnapshot(t *testing.T) { // Create a test database tempDir := t.TempDir() dbPath := filepath.Join(tempDir, "test.db") + db, err := database.New(ctx, dbPath) if err != nil { t.Fatalf("failed to create database: %v", err) diff --git a/internal/storage/file.go b/internal/storage/file.go index 1c1fcf6..2d2ecf0 100644 --- a/internal/storage/file.go +++ b/internal/storage/file.go @@ -102,26 +102,32 @@ func (f *FileStorer) PutWithProgress(ctx context.Context, key string, data io.Re // Get retrieves data from the specified key. func (f *FileStorer) Get(ctx context.Context, key string) (io.ReadCloser, error) { path := f.fullPath(key) + file, err := f.fs.Open(path) if err != nil { if os.IsNotExist(err) { return nil, ErrNotFound } + return nil, fmt.Errorf("opening file: %w", err) } + return file, nil } // Stat returns metadata about an object without retrieving its contents. func (f *FileStorer) Stat(ctx context.Context, key string) (*ObjectInfo, error) { path := f.fullPath(key) + info, err := f.fs.Stat(path) if err != nil { if os.IsNotExist(err) { return nil, ErrNotFound } + return nil, fmt.Errorf("stat file: %w", err) } + return &ObjectInfo{ Key: key, Size: info.Size(), @@ -131,19 +137,23 @@ func (f *FileStorer) Stat(ctx context.Context, key string) (*ObjectInfo, error) // Delete removes an object. func (f *FileStorer) Delete(ctx context.Context, key string) error { path := f.fullPath(key) + err := f.fs.Remove(path) if os.IsNotExist(err) { return nil // Match S3 behavior: no error if doesn't exist } + if err != nil { return fmt.Errorf("removing file: %w", err) } + return nil } // List returns all keys with the given prefix. func (f *FileStorer) List(ctx context.Context, prefix string) ([]string, error) { var keys []string + basePath := f.fullPath(prefix) // Check if base path exists @@ -151,6 +161,7 @@ func (f *FileStorer) List(ctx context.Context, prefix string) ([]string, error) if err != nil { return nil, fmt.Errorf("checking path: %w", err) } + if !exists { return keys, nil // Empty list for non-existent prefix } @@ -177,9 +188,9 @@ func (f *FileStorer) List(ctx context.Context, prefix string) ([]string, error) relPath = strings.ReplaceAll(relPath, string(filepath.Separator), "/") keys = append(keys, relPath) } + return nil }) - if err != nil { return nil, fmt.Errorf("walking directory: %w", err) } @@ -192,14 +203,17 @@ func (f *FileStorer) ListStream(ctx context.Context, prefix string) <-chan Objec ch := make(chan ObjectInfo) go func() { defer close(ch) + basePath := f.fullPath(prefix) // Check if base path exists exists, err := afero.Exists(f.fs, basePath) if err != nil { ch <- ObjectInfo{Err: fmt.Errorf("checking path: %w", err)} + return } + if !exists { return // Empty channel for non-existent prefix } @@ -209,12 +223,14 @@ func (f *FileStorer) ListStream(ctx context.Context, prefix string) <-chan Objec select { case <-ctx.Done(): ch <- ObjectInfo{Err: ctx.Err()} + return ctx.Err() default: } if err != nil { ch <- ObjectInfo{Err: err} + return nil // Continue walking despite errors } @@ -222,6 +238,7 @@ func (f *FileStorer) ListStream(ctx context.Context, prefix string) <-chan Objec relPath, err := filepath.Rel(f.basePath, path) if err != nil { ch <- ObjectInfo{Err: fmt.Errorf("computing relative path: %w", err)} + return nil } // Normalize path separators @@ -231,9 +248,11 @@ func (f *FileStorer) ListStream(ctx context.Context, prefix string) <-chan Objec Size: info.Size(), } } + return nil }) }() + return ch } @@ -257,10 +276,12 @@ func (pw *progressWriter) Write(p []byte) (int, error) { if n > 0 { pw.written += int64(n) if pw.callback != nil { - if callbackErr := pw.callback(pw.written); callbackErr != nil { + callbackErr := pw.callback(pw.written) + if callbackErr != nil { return n, callbackErr } } } + return n, err } diff --git a/internal/storage/module.go b/internal/storage/module.go index bf830d6..42cc28a 100644 --- a/internal/storage/module.go +++ b/internal/storage/module.go @@ -24,6 +24,7 @@ func NewStorer(cfg *config.Config) (Storer, error) { if cfg.StorageURL != "" { return storerFromURL(cfg.StorageURL, cfg) } + return storerFromLegacyS3Config(cfg) } @@ -71,6 +72,7 @@ func storerFromURL(rawURL string, cfg *config.Config) (Storer, error) { if err != nil { return nil, fmt.Errorf("creating S3 client: %w", err) } + return NewS3Storer(client), nil case "rclone": @@ -109,5 +111,6 @@ func storerFromLegacyS3Config(cfg *config.Config) (Storer, error) { if err != nil { return nil, fmt.Errorf("creating S3 client: %w", err) } + return NewS3Storer(client), nil } diff --git a/internal/storage/rclone.go b/internal/storage/rclone.go index aff5ba6..19d39d4 100644 --- a/internal/storage/rclone.go +++ b/internal/storage/rclone.go @@ -49,6 +49,7 @@ func NewRcloneStorer(ctx context.Context, remote, path string) (*RcloneStorer, e strings.Contains(err.Error(), "failed to find remote") { return nil, fmt.Errorf("%w: %s", ErrRemoteNotFound, remote) } + return nil, fmt.Errorf("creating rclone filesystem: %w", err) } @@ -101,9 +102,11 @@ func (r *RcloneStorer) Get(ctx context.Context, key string) (io.ReadCloser, erro if errors.Is(err, fs.ErrorObjectNotFound) { return nil, ErrNotFound } + if errors.Is(err, fs.ErrorDirNotFound) { return nil, ErrNotFound } + return nil, fmt.Errorf("getting object: %w", err) } @@ -123,9 +126,11 @@ func (r *RcloneStorer) Stat(ctx context.Context, key string) (*ObjectInfo, error if errors.Is(err, fs.ErrorObjectNotFound) { return nil, ErrNotFound } + if errors.Is(err, fs.ErrorDirNotFound) { return nil, ErrNotFound } + return nil, fmt.Errorf("getting object: %w", err) } @@ -142,9 +147,11 @@ func (r *RcloneStorer) Delete(ctx context.Context, key string) error { if errors.Is(err, fs.ErrorObjectNotFound) { return nil // Match S3 behavior: no error if doesn't exist } + if errors.Is(err, fs.ErrorDirNotFound) { return nil } + return fmt.Errorf("getting object: %w", err) } @@ -209,6 +216,7 @@ func (r *RcloneStorer) Info() StorageInfo { if r.path != "" { location += ":" + r.path } + return StorageInfo{ Type: "rclone", Location: location, @@ -227,10 +235,12 @@ func (pr *progressReader) Read(p []byte) (int, error) { if n > 0 { pr.read += int64(n) if pr.callback != nil { - if callbackErr := pr.callback(pr.read); callbackErr != nil { + callbackErr := pr.callback(pr.read) + if callbackErr != nil { return n, callbackErr } } } + return n, err } diff --git a/internal/storage/s3.go b/internal/storage/s3.go index 534aed7..64b3ede 100644 --- a/internal/storage/s3.go +++ b/internal/storage/s3.go @@ -30,6 +30,7 @@ func (s *S3Storer) PutWithProgress(ctx context.Context, key string, data io.Read if progress != nil { s3Progress = s3.ProgressCallback(progress) } + return s.client.PutObjectWithProgress(ctx, key, data, size, s3Progress) } @@ -44,6 +45,7 @@ func (s *S3Storer) Stat(ctx context.Context, key string) (*ObjectInfo, error) { if err != nil { return nil, err } + return &ObjectInfo{ Key: info.Key, Size: info.Size, @@ -65,6 +67,7 @@ func (s *S3Storer) ListStream(ctx context.Context, prefix string) <-chan ObjectI ch := make(chan ObjectInfo) go func() { defer close(ch) + for info := range s.client.ListObjectsStream(ctx, prefix, false) { ch <- ObjectInfo{ Key: info.Key, @@ -73,6 +76,7 @@ func (s *S3Storer) ListStream(ctx context.Context, prefix string) <-chan ObjectI } } }() + return ch } diff --git a/internal/storage/url.go b/internal/storage/url.go index 2dcca35..4ab169d 100644 --- a/internal/storage/url.go +++ b/internal/storage/url.go @@ -1,6 +1,7 @@ package storage import ( + "errors" "fmt" "net/url" "strings" @@ -24,15 +25,16 @@ type StorageURL struct { // - rclone://remote/path/to/backups func ParseStorageURL(rawURL string) (*StorageURL, error) { if rawURL == "" { - return nil, fmt.Errorf("storage URL is empty") + return nil, errors.New("storage URL is empty") } // Handle file:// URLs - if strings.HasPrefix(rawURL, "file://") { - path := strings.TrimPrefix(rawURL, "file://") + if after, ok := strings.CutPrefix(rawURL, "file://"); ok { + path := after if path == "" { - return nil, fmt.Errorf("file URL path is empty") + return nil, errors.New("file URL path is empty") } + return &StorageURL{ Scheme: "file", Prefix: path, @@ -48,12 +50,13 @@ func ParseStorageURL(rawURL string) (*StorageURL, error) { bucket := u.Host if bucket == "" { - return nil, fmt.Errorf("s3 URL missing bucket name") + return nil, errors.New("s3 URL missing bucket name") } prefix := strings.TrimPrefix(u.Path, "/") query := u.Query() + useSSL := true if query.Get("ssl") == "false" { useSSL = false @@ -78,7 +81,7 @@ func ParseStorageURL(rawURL string) (*StorageURL, error) { remote := u.Host if remote == "" { - return nil, fmt.Errorf("rclone URL missing remote name") + return nil, errors.New("rclone URL missing remote name") } path := strings.TrimPrefix(u.Path, "/") @@ -90,29 +93,32 @@ func ParseStorageURL(rawURL string) (*StorageURL, error) { }, nil } - return nil, fmt.Errorf("unsupported URL scheme: must start with s3://, file://, or rclone://") + return nil, errors.New("unsupported URL scheme: must start with s3://, file://, or rclone://") } // String returns a human-readable representation of the storage URL. func (u *StorageURL) String() string { switch u.Scheme { case "file": - return fmt.Sprintf("file://%s", u.Prefix) + return "file://" + u.Prefix case "s3": endpoint := u.Endpoint if endpoint == "" { endpoint = "s3.amazonaws.com" } + if u.Prefix != "" { return fmt.Sprintf("s3://%s/%s (endpoint: %s)", u.Bucket, u.Prefix, endpoint) } + return fmt.Sprintf("s3://%s (endpoint: %s)", u.Bucket, endpoint) case "rclone": if u.Prefix != "" { return fmt.Sprintf("rclone://%s/%s", u.RcloneRemote, u.Prefix) } - return fmt.Sprintf("rclone://%s", u.RcloneRemote) + + return "rclone://" + u.RcloneRemote default: - return fmt.Sprintf("%s://?", u.Scheme) + return u.Scheme + "://?" } } diff --git a/internal/types/types.go b/internal/types/types.go index 51636e1..9abc02c 100644 --- a/internal/types/types.go +++ b/internal/types/types.go @@ -24,6 +24,7 @@ func ParseFileID(s string) (FileID, error) { if err != nil { return FileID{}, err } + return FileID(id), nil } @@ -38,13 +39,15 @@ func (id FileID) Value() (driver.Value, error) { } // Scan implements sql.Scanner for database deserialization. -func (id *FileID) Scan(src interface{}) error { +func (id *FileID) Scan(src any) error { if src == nil { *id = FileID{} + return nil } var s string + switch v := src.(type) { case string: s = v @@ -58,7 +61,9 @@ func (id *FileID) Scan(src interface{}) error { if err != nil { return fmt.Errorf("invalid FileID: %w", err) } + *id = FileID(parsed) + return nil } @@ -77,6 +82,7 @@ func ParseBlobID(s string) (BlobID, error) { if err != nil { return BlobID{}, err } + return BlobID(id), nil } @@ -91,13 +97,15 @@ func (id BlobID) Value() (driver.Value, error) { } // Scan implements sql.Scanner for database deserialization. -func (id *BlobID) Scan(src interface{}) error { +func (id *BlobID) Scan(src any) error { if src == nil { *id = BlobID{} + return nil } var s string + switch v := src.(type) { case string: s = v @@ -111,7 +119,9 @@ func (id *BlobID) Scan(src interface{}) error { if err != nil { return fmt.Errorf("invalid BlobID: %w", err) } + *id = BlobID(parsed) + return nil } diff --git a/internal/ui/ui.go b/internal/ui/ui.go index 6855b41..83abf4d 100644 --- a/internal/ui/ui.go +++ b/internal/ui/ui.go @@ -94,10 +94,12 @@ func shouldColor(w io.Writer) bool { if os.Getenv("NO_COLOR") != "" { return false } + f, ok := w.(*os.File) if !ok { return false } + return term.IsTerminal(int(f.Fd())) } @@ -106,67 +108,73 @@ func (w *Writer) paint(color, s string) string { if !w.color { return s } + return color + s + ansiReset } // ───────────────────────── message methods ───────────────────────── -// Begin prints an operation-start line, left-aligned with a white marker. -func (w *Writer) Begin(format string, args ...any) { +// Beginf prints an operation-start line, left-aligned with a white marker. +func (w *Writer) Beginf(format string, args ...any) { if w.quiet { return } + w.emit(ansiWhite, Marker, "", format, args) } -// Complete prints an operation-completion line in green, left-aligned. -func (w *Writer) Complete(format string, args ...any) { +// Completef prints an operation-completion line in green, left-aligned. +func (w *Writer) Completef(format string, args ...any) { if w.quiet { return } + w.emit(ansiGreen, Marker, ansiGreen, format, args) } -// Info prints a neutral status line, left-aligned with a white marker. -func (w *Writer) Info(format string, args ...any) { +// Infof prints a neutral status line, left-aligned with a white marker. +func (w *Writer) Infof(format string, args ...any) { if w.quiet { return } + w.emit(ansiWhite, Marker, "", format, args) } -// Notice prints an attention-worthy informational line, marker in cyan. -func (w *Writer) Notice(format string, args ...any) { +// Noticef prints an attention-worthy informational line, marker in cyan. +func (w *Writer) Noticef(format string, args ...any) { if w.quiet { return } + w.emit(ansiCyan, Marker, "", format, args) } -// Warning prints "⚠️ Warning: " in orange/yellow followed by the message. -func (w *Writer) Warning(format string, args ...any) { +// Warningf prints "⚠️ Warning: " in orange/yellow followed by the message. +func (w *Writer) Warningf(format string, args ...any) { w.warnings++ prefix := "⚠️ " + w.paint(ansiYellow+ansiBold, "Warning: ") _, _ = fmt.Fprintln(w.out, prefix+fmt.Sprintf(format, args...)) } -// Error prints "🛑 ERROR: " in red followed by the message. Goes to the +// Errorf prints "🛑 ERROR: " in red followed by the message. Goes to the // same writer as everything else; callers that want stderr should // construct a separate Writer for it. -func (w *Writer) Error(format string, args ...any) { +func (w *Writer) Errorf(format string, args ...any) { w.errors++ prefix := "🛑 " + w.paint(ansiRed+ansiBold, "ERROR: ") _, _ = fmt.Fprintln(w.out, prefix+fmt.Sprintf(format, args...)) } -// Detail prints an indented continuation line under a preceding Complete +// Detailf prints an indented continuation line under a preceding Completef // (or other top-level message). Marker " 》" (white) at column 2. -// Distinct from Progress (semantically a "heartbeat") in usage but +// Distinct from Progressf (semantically a "heartbeat") in usage but // visually identical. -func (w *Writer) Detail(format string, args ...any) { +func (w *Writer) Detailf(format string, args ...any) { if w.quiet { return } + w.emit(ansiWhite, " "+Marker, "", format, args) } @@ -176,24 +184,27 @@ func (w *Writer) WarningCount() int { return w.warnings } // ErrorCount returns the number of Error() calls this writer has emitted. func (w *Writer) ErrorCount() int { return w.errors } -// Progress prints an indented heartbeat / per-item update, marker in white. -func (w *Writer) Progress(format string, args ...any) { +// Progressf prints an indented heartbeat / per-item update, marker in white. +func (w *Writer) Progressf(format string, args ...any) { if w.quiet { return } + w.emit(ansiWhite, " "+Marker, "", format, args) } -// Banner prints a line with no marker, left-aligned. Bold when color +// Bannerf prints a line with no marker, left-aligned. Bold when color // is enabled. Used for the application startup banner only. -func (w *Writer) Banner(format string, args ...any) { +func (w *Writer) Bannerf(format string, args ...any) { if w.quiet { return } + body := fmt.Sprintf(format, args...) if w.color { body = ansiBold + body + ansiReset } + _, _ = fmt.Fprintln(w.out, body) } @@ -204,6 +215,7 @@ func (w *Writer) emit(prefixColor, prefix, bodyColor, format string, args []any) if bodyColor != "" { body = w.paint(bodyColor, body) } + _, _ = fmt.Fprintln(w.out, w.paint(prefixColor, prefix)+" "+body) } @@ -219,6 +231,7 @@ func (w *Writer) Hex(s string) string { if len(s) > 12 { short = s[:12] + "..." } + return w.paint(ansiCyan, short) } @@ -244,8 +257,11 @@ func (w *Writer) Speed(bytesPerSec float64) string { if bytesPerSec <= 0 { return w.paint(ansiMagenta, "N/A") } + bitsPerSec := bytesPerSec * 8 + var s string + switch { case bitsPerSec >= 1e9: s = fmt.Sprintf("%.1f Gbit/sec", bitsPerSec/1e9) @@ -256,6 +272,7 @@ func (w *Writer) Speed(bytesPerSec float64) string { default: s = fmt.Sprintf("%.0f bit/sec", bitsPerSec) } + return w.paint(ansiMagenta, s) } @@ -270,10 +287,12 @@ func (w *Writer) Duration(d time.Duration) string { // displayed in the process's local zone. func (w *Writer) Time(t time.Time) string { t = t.Local() + now := time.Now() if t.Year() == now.Year() && t.YearDay() == now.YearDay() { return w.paint(ansiYellow, t.Format("15:04:05")) } + return w.paint(ansiYellow, t.Format("2006-01-02 15:04:05")) } diff --git a/internal/ui/ui_test.go b/internal/ui/ui_test.go index ac9ad73..0f06b86 100644 --- a/internal/ui/ui_test.go +++ b/internal/ui/ui_test.go @@ -9,6 +9,7 @@ import ( func newTestWriter(color bool) (*Writer, *bytes.Buffer) { buf := &bytes.Buffer{} + return NewWithColor(buf, color), buf } @@ -18,21 +19,22 @@ func TestMessageMethodsPlain(t *testing.T) { fn func(*Writer) want string }{ - {"Begin", func(w *Writer) { w.Begin("starting %s", "thing") }, "》 starting thing\n"}, - {"Complete", func(w *Writer) { w.Complete("done %s", "thing") }, "》 done thing\n"}, - {"Info", func(w *Writer) { w.Info("status") }, "》 status\n"}, - {"Notice", func(w *Writer) { w.Notice("note") }, "》 note\n"}, - {"Warning", func(w *Writer) { w.Warning("oops") }, "⚠️ Warning: oops\n"}, - {"Error", func(w *Writer) { w.Error("boom") }, "🛑 ERROR: boom\n"}, - {"Progress", func(w *Writer) { w.Progress("p") }, " 》 p\n"}, - {"Detail", func(w *Writer) { w.Detail("d") }, " 》 d\n"}, - {"Banner", func(w *Writer) { w.Banner("hello") }, "hello\n"}, // plain mode, no bold + {"Begin", func(w *Writer) { w.Beginf("starting %s", "thing") }, "》 starting thing\n"}, + {"Complete", func(w *Writer) { w.Completef("done %s", "thing") }, "》 done thing\n"}, + {"Info", func(w *Writer) { w.Infof("status") }, "》 status\n"}, + {"Notice", func(w *Writer) { w.Noticef("note") }, "》 note\n"}, + {"Warning", func(w *Writer) { w.Warningf("oops") }, "⚠️ Warning: oops\n"}, + {"Error", func(w *Writer) { w.Errorf("boom") }, "🛑 ERROR: boom\n"}, + {"Progress", func(w *Writer) { w.Progressf("p") }, " 》 p\n"}, + {"Detail", func(w *Writer) { w.Detailf("d") }, " 》 d\n"}, + {"Banner", func(w *Writer) { w.Bannerf("hello") }, "hello\n"}, // plain mode, no bold } for _, tt := range tests { t.Run(tt.method, func(t *testing.T) { w, buf := newTestWriter(false) tt.fn(w) + if got := buf.String(); got != tt.want { t.Errorf("got %q, want %q", got, tt.want) } @@ -45,13 +47,16 @@ func TestWarningErrorCounters(t *testing.T) { if w.WarningCount() != 0 || w.ErrorCount() != 0 { t.Fatalf("expected fresh writer to have zero counts") } - w.Info("normal") - w.Warning("first warn") - w.Warning("second warn") - w.Error("only error") + + w.Infof("normal") + w.Warningf("first warn") + w.Warningf("second warn") + w.Errorf("only error") + if got, want := w.WarningCount(), 2; got != want { t.Errorf("WarningCount: got %d, want %d", got, want) } + if got, want := w.ErrorCount(), 1; got != want { t.Errorf("ErrorCount: got %d, want %d", got, want) } @@ -59,11 +64,13 @@ func TestWarningErrorCounters(t *testing.T) { func TestColorOutputContainsANSI(t *testing.T) { w, buf := newTestWriter(true) - w.Error("boom") + w.Errorf("boom") + out := buf.String() if !strings.Contains(out, "\033[") { t.Errorf("expected ANSI escapes in color output, got %q", out) } + if !strings.Contains(out, "ERROR: ") { t.Errorf("expected 'ERROR: ' text in output, got %q", out) } @@ -71,7 +78,8 @@ func TestColorOutputContainsANSI(t *testing.T) { func TestBannerBoldWhenColor(t *testing.T) { w, buf := newTestWriter(true) - w.Banner("hello") + w.Bannerf("hello") + out := buf.String() if !strings.Contains(out, "\033[1m") { t.Errorf("expected bold ANSI escape in colored Banner output, got %q", out) @@ -84,18 +92,23 @@ func TestValueFormattersPlain(t *testing.T) { if got := w.Hex("0123456789abcdef0123"); got != "0123456789ab..." { t.Errorf("Hex long: got %q", got) } + if got := w.Hex("short"); got != "short" { t.Errorf("Hex short: got %q", got) } + if got := w.Size(1024); got != "1.0 kB" { t.Errorf("Size: got %q", got) } + if got := w.Duration(90 * time.Second); got != "1m30s" { t.Errorf("Duration: got %q", got) } + if got := w.Count(12345); got != "12,345" { t.Errorf("Count: got %q", got) } + if got := w.Percent(12.34); got != "12.3%" { t.Errorf("Percent: got %q", got) } @@ -104,9 +117,11 @@ func TestValueFormattersPlain(t *testing.T) { if got := w.Speed(0); got != "N/A" { t.Errorf("Speed(0): got %q, want N/A", got) } + if got := w.Speed(125_000_000); got != "1.0 Gbit/sec" { // 1 Gbit/s = 125 MB/s t.Errorf("Speed(125e6): got %q", got) } + if got := w.Speed(125_000); got != "1 Mbit/sec" { t.Errorf("Speed(125e3): got %q", got) } @@ -116,6 +131,7 @@ func TestValueFormattersPlain(t *testing.T) { if got := w.Time(today); got != "14:30:45" { t.Errorf("Time today: got %q, want 14:30:45", got) } + other := time.Date(2030, 1, 2, 3, 4, 5, 0, time.Local) if got := w.Time(other); got != "2030-01-02 03:04:05" { t.Errorf("Time other day: got %q", got) @@ -124,10 +140,12 @@ func TestValueFormattersPlain(t *testing.T) { func TestValueFormattersColored(t *testing.T) { w, _ := newTestWriter(true) + hex := w.Hex("0123456789abcdef0123") if !strings.Contains(hex, "\033[") { t.Errorf("expected ANSI in colored Hex output, got %q", hex) } + if !strings.Contains(hex, "0123456789ab") { t.Errorf("expected hex content in output, got %q", hex) } diff --git a/internal/vaultik/blob_fetch.go b/internal/vaultik/blob_fetch.go index dde5f34..1bd5931 100644 --- a/internal/vaultik/blob_fetch.go +++ b/internal/vaultik/blob_fetch.go @@ -4,6 +4,7 @@ import ( "context" "crypto/sha256" "encoding/hex" + "errors" "fmt" "io" "time" @@ -26,9 +27,10 @@ type hashVerifyReader struct { func (h *hashVerifyReader) Read(p []byte) (int, error) { n, err := h.reader.Read(p) - if err == io.EOF { + if errors.Is(err, io.EOF) { h.done = true } + return n, err } @@ -41,6 +43,7 @@ func (h *hashVerifyReader) Close() error { firstHash := h.reader.Sum256() secondHasher := sha256.New() secondHasher.Write(firstHash) + actualHashHex := hex.EncodeToString(secondHasher.Sum(nil)) if actualHashHex != h.blobHash { return fmt.Errorf("blob hash mismatch: expected %s, got %s", h.blobHash[:16], actualHashHex[:16]) @@ -50,6 +53,7 @@ func (h *hashVerifyReader) Close() error { if readerErr != nil { return readerErr } + return fetcherErr } @@ -66,6 +70,7 @@ func (v *Vaultik) FetchAndDecryptBlob(ctx context.Context, blobHash string, expe reader, err := blobgen.NewReader(rc, identity) if err != nil { _ = rc.Close() + return nil, fmt.Errorf("creating blob reader: %w", err) } @@ -86,6 +91,7 @@ func (v *Vaultik) FetchBlob(ctx context.Context, blobHash string, expectedSize i t0 := time.Now() rc, err := v.Storage.Get(ctx, blobPath) getDur := time.Since(t0) + if err != nil { return nil, 0, fmt.Errorf("downloading blob %s: %w", blobHash[:16], err) } @@ -93,8 +99,10 @@ func (v *Vaultik) FetchBlob(ctx context.Context, blobHash string, expectedSize i t0 = time.Now() info, err := v.Storage.Stat(ctx, blobPath) statDur := time.Since(t0) + if err != nil { _ = rc.Close() + return nil, 0, fmt.Errorf("stat blob %s: %w", blobHash[:16], err) } diff --git a/internal/vaultik/blob_fetch_hash_test.go b/internal/vaultik/blob_fetch_hash_test.go index c8f6099..6dee0b6 100644 --- a/internal/vaultik/blob_fetch_hash_test.go +++ b/internal/vaultik/blob_fetch_hash_test.go @@ -24,17 +24,22 @@ func TestFetchAndDecryptBlobVerifiesHash(t *testing.T) { // Create test data and encrypt it using blobgen.Writer plaintext := []byte("hello world test data for blob hash verification") + var encBuf bytes.Buffer + writer, err := blobgen.NewWriter(&encBuf, 1, []string{identity.Recipient().String()}) if err != nil { t.Fatalf("creating blobgen writer: %v", err) } + if _, err := writer.Write(plaintext); err != nil { t.Fatalf("writing plaintext: %v", err) } + if err := writer.Close(); err != nil { t.Fatalf("closing writer: %v", err) } + encryptedData := encBuf.Bytes() // Compute correct double-SHA-256 hash of the plaintext (matches blobgen.Writer.Sum256) @@ -51,6 +56,7 @@ func TestFetchAndDecryptBlobVerifiesHash(t *testing.T) { // Set up mock storage with the blob at the correct path mockStorage := NewMockStorer() blobPath := "blobs/" + correctHash[:2] + "/" + correctHash[2:4] + "/" + correctHash + mockStorage.mu.Lock() mockStorage.data[blobPath] = encryptedData mockStorage.mu.Unlock() @@ -63,13 +69,16 @@ func TestFetchAndDecryptBlobVerifiesHash(t *testing.T) { if err != nil { t.Fatalf("expected success, got error: %v", err) } + data, err := io.ReadAll(rc) if err != nil { t.Fatalf("reading stream: %v", err) } + if err := rc.Close(); err != nil { t.Fatalf("close (hash verification) failed: %v", err) } + if !bytes.Equal(data, plaintext) { t.Fatalf("decrypted data mismatch: got %q, want %q", data, plaintext) } @@ -79,6 +88,7 @@ func TestFetchAndDecryptBlobVerifiesHash(t *testing.T) { // Use a fake hash that doesn't match the actual plaintext fakeHash := strings.Repeat("ab", 32) // 64 hex chars fakePath := "blobs/" + fakeHash[:2] + "/" + fakeHash[2:4] + "/" + fakeHash + mockStorage.mu.Lock() mockStorage.data[fakePath] = encryptedData mockStorage.mu.Unlock() @@ -89,10 +99,12 @@ func TestFetchAndDecryptBlobVerifiesHash(t *testing.T) { } // Read all data — hash is verified on Close _, _ = io.ReadAll(rc) + err = rc.Close() if err == nil { t.Fatal("expected error for mismatched hash, got nil") } + if !strings.Contains(err.Error(), "hash mismatch") { t.Fatalf("expected hash mismatch error, got: %v", err) } diff --git a/internal/vaultik/blobcache.go b/internal/vaultik/blobcache.go index 7ebbefe..2a28f18 100644 --- a/internal/vaultik/blobcache.go +++ b/internal/vaultik/blobcache.go @@ -53,6 +53,7 @@ func newBlobDiskCache(maxBytes int64) (*blobDiskCache, error) { if err != nil { return nil, fmt.Errorf("creating blob cache dir: %w", err) } + return &blobDiskCache{ dir: dir, maxBytes: maxBytes, @@ -70,21 +71,25 @@ func (c *blobDiskCache) unlink(e *blobDiskCacheEntry) { } else { c.head = e.next } + if e.next != nil { e.next.prev = e.prev } else { c.tail = e.prev } + e.prev = nil e.next = nil } func (c *blobDiskCache) pushFront(e *blobDiskCacheEntry) { e.prev = nil + e.next = c.head if c.head != nil { c.head.prev = e } + c.head = e if c.tail == nil { c.tail = e @@ -95,6 +100,7 @@ func (c *blobDiskCache) evictLRU() { if c.tail == nil { return } + victim := c.tail c.unlink(victim) delete(c.items, victim.key) @@ -121,7 +127,8 @@ func (c *blobDiskCache) Put(key string, data []byte) error { delete(c.items, key) } - if err := os.WriteFile(c.path(key), data, 0600); err != nil { + err := os.WriteFile(c.path(key), data, 0600) + if err != nil { return fmt.Errorf("writing blob to cache: %w", err) } @@ -163,14 +170,19 @@ func (c *blobDiskCache) PutFromReader(key string, r io.Reader) (int64, error) { if err != nil { return 0, fmt.Errorf("creating cache file: %w", err) } + written, copyErr := io.Copy(f, r) closeErr := f.Close() + if copyErr != nil { _ = os.Remove(c.path(key)) + return written, fmt.Errorf("streaming to cache file: %w", copyErr) } + if closeErr != nil { _ = os.Remove(c.path(key)) + return written, fmt.Errorf("closing cache file: %w", closeErr) } @@ -182,6 +194,7 @@ func (c *blobDiskCache) PutFromReader(key string, r io.Reader) (int64, error) { // so this branch is effectively unreachable there. if written > c.maxBytes { _ = os.Remove(c.path(key)) + return written, nil } @@ -205,11 +218,14 @@ func (c *blobDiskCache) PutFromReader(key string, r io.Reader) (int64, error) { func (c *blobDiskCache) Get(key string) ([]byte, bool) { c.mu.Lock() c.getCalls++ + e, ok := c.items[key] if !ok { c.mu.Unlock() + return nil, false } + c.unlink(e) c.pushFront(e) c.mu.Unlock() @@ -223,8 +239,10 @@ func (c *blobDiskCache) Get(key string) ([]byte, bool) { c.curBytes -= e.size } c.mu.Unlock() + return nil, false } + return data, true } @@ -232,15 +250,20 @@ func (c *blobDiskCache) Get(key string) ([]byte, bool) { func (c *blobDiskCache) ReadAt(key string, offset, length int64) ([]byte, error) { c.mu.Lock() c.readAtCalls++ + e, ok := c.items[key] if !ok { c.mu.Unlock() + return nil, fmt.Errorf("key %q not in cache", key) } + if offset+length > e.size { c.mu.Unlock() + return nil, fmt.Errorf("read beyond blob size: offset=%d length=%d size=%d", offset, length, e.size) } + c.unlink(e) c.pushFront(e) c.mu.Unlock() @@ -255,6 +278,7 @@ func (c *blobDiskCache) ReadAt(key string, offset, length int64) ([]byte, error) if _, err := f.ReadAt(buf, offset); err != nil { return nil, err } + return buf, nil } @@ -262,7 +286,9 @@ func (c *blobDiskCache) ReadAt(key string, offset, length int64) ([]byte, error) func (c *blobDiskCache) Has(key string) bool { c.mu.Lock() defer c.mu.Unlock() + _, ok := c.items[key] + return ok } @@ -272,10 +298,12 @@ func (c *blobDiskCache) Has(key string) bool { func (c *blobDiskCache) Delete(key string) { c.mu.Lock() defer c.mu.Unlock() + e, ok := c.items[key] if !ok { return } + c.unlink(e) delete(c.items, key) c.curBytes -= e.size @@ -287,10 +315,12 @@ func (c *blobDiskCache) Delete(key string) { func (c *blobDiskCache) Keys() []string { c.mu.Lock() defer c.mu.Unlock() + keys := make([]string, 0, len(c.items)) for k := range c.items { keys = append(keys, k) } + return keys } @@ -298,6 +328,7 @@ func (c *blobDiskCache) Keys() []string { func (c *blobDiskCache) Size() int64 { c.mu.Lock() defer c.mu.Unlock() + return c.curBytes } @@ -305,6 +336,7 @@ func (c *blobDiskCache) Size() int64 { func (c *blobDiskCache) Len() int { c.mu.Lock() defer c.mu.Unlock() + return len(c.items) } @@ -312,6 +344,7 @@ func (c *blobDiskCache) Len() int { func (c *blobDiskCache) GetCalls() int { c.mu.Lock() defer c.mu.Unlock() + return c.getCalls } @@ -319,6 +352,7 @@ func (c *blobDiskCache) GetCalls() int { func (c *blobDiskCache) ReadAtCalls() int { c.mu.Lock() defer c.mu.Unlock() + return c.readAtCalls } @@ -327,6 +361,7 @@ func (c *blobDiskCache) ReadAtCalls() int { func (c *blobDiskCache) PeakLen() int { c.mu.Lock() defer c.mu.Unlock() + return c.peakLen } @@ -334,9 +369,11 @@ func (c *blobDiskCache) PeakLen() int { func (c *blobDiskCache) Close() error { c.mu.Lock() defer c.mu.Unlock() + c.items = nil c.head = nil c.tail = nil c.curBytes = 0 + return os.RemoveAll(c.dir) } diff --git a/internal/vaultik/blobcache_test.go b/internal/vaultik/blobcache_test.go index 778aadd..7467eff 100644 --- a/internal/vaultik/blobcache_test.go +++ b/internal/vaultik/blobcache_test.go @@ -23,6 +23,7 @@ func TestBlobDiskCache_BasicGetPut(t *testing.T) { if !ok { t.Fatal("expected cache hit") } + if !bytes.Equal(got, data) { t.Fatalf("got %q, want %q", got, data) } @@ -35,15 +36,19 @@ func TestBlobDiskCache_BasicGetPut(t *testing.T) { func TestBlobDiskCache_EvictionUnderPressure(t *testing.T) { maxBytes := int64(1000) + cache, err := newBlobDiskCache(maxBytes) if err != nil { t.Fatal(err) } + defer func() { _ = cache.Close() }() - for i := 0; i < 5; i++ { + for i := range 5 { data := make([]byte, 300) - if err := cache.Put(fmt.Sprintf("key%d", i), data); err != nil { + + err := cache.Put(fmt.Sprintf("key%d", i), data) + if err != nil { t.Fatal(err) } } @@ -55,6 +60,7 @@ func TestBlobDiskCache_EvictionUnderPressure(t *testing.T) { if !cache.Has("key4") { t.Fatal("expected key4 to be cached") } + if cache.Has("key0") { t.Fatal("expected key0 to be evicted") } @@ -87,6 +93,7 @@ func TestBlobDiskCache_UpdateInPlace(t *testing.T) { if err := cache.Put("key1", []byte("v1")); err != nil { t.Fatal(err) } + if err := cache.Put("key1", []byte("version2")); err != nil { t.Fatal(err) } @@ -95,12 +102,15 @@ func TestBlobDiskCache_UpdateInPlace(t *testing.T) { if !ok { t.Fatal("expected hit") } + if string(got) != "version2" { t.Fatalf("got %q, want %q", got, "version2") } + if cache.Len() != 1 { t.Fatalf("expected 1 entry, got %d", cache.Len()) } + if cache.Size() != int64(len("version2")) { t.Fatalf("expected size %d, got %d", len("version2"), cache.Size()) } @@ -117,6 +127,7 @@ func TestBlobDiskCache_ReadAt(t *testing.T) { if _, err := rand.Read(data); err != nil { t.Fatal(err) } + if err := cache.Put("blob1", data); err != nil { t.Fatal(err) } @@ -125,6 +136,7 @@ func TestBlobDiskCache_ReadAt(t *testing.T) { if err != nil { t.Fatal(err) } + if !bytes.Equal(chunk, data[100:300]) { t.Fatal("ReadAt returned wrong data") } @@ -149,6 +161,7 @@ func TestBlobDiskCache_Close(t *testing.T) { if err := cache.Put("key1", []byte("data")); err != nil { t.Fatal(err) } + if err := cache.Close(); err != nil { t.Fatal(err) } @@ -165,6 +178,7 @@ func TestBlobDiskCache_LRUOrder(t *testing.T) { if err := cache.Put("a", d); err != nil { t.Fatal(err) } + if err := cache.Put("b", d); err != nil { t.Fatal(err) } @@ -180,9 +194,11 @@ func TestBlobDiskCache_LRUOrder(t *testing.T) { if !cache.Has("a") { t.Fatal("expected 'a' to survive") } + if !cache.Has("c") { t.Fatal("expected 'c' to be present") } + if cache.Has("b") { t.Fatal("expected 'b' to be evicted") } diff --git a/internal/vaultik/helpers.go b/internal/vaultik/helpers.go index deb0d3b..b10e161 100644 --- a/internal/vaultik/helpers.go +++ b/internal/vaultik/helpers.go @@ -1,6 +1,7 @@ package vaultik import ( + "errors" "fmt" "regexp" "strconv" @@ -29,11 +30,13 @@ func formatBytes(bytes int64) string { if bytes < unit { return fmt.Sprintf("%d B", bytes) } + div, exp := int64(unit), 0 for n := bytes / unit; n >= unit; n /= unit { div *= unit exp++ } + return fmt.Sprintf("%.1f %cB", float64(bytes)/float64(div), "KMGTPE"[exp]) } @@ -42,11 +45,12 @@ func formatBytes(bytes int64) string { func parseSnapshotTimestamp(snapshotID string) (time.Time, error) { parts := strings.Split(snapshotID, "_") if len(parts) < 2 { - return time.Time{}, fmt.Errorf("invalid snapshot ID format: expected hostname_snapshotname_timestamp") + return time.Time{}, errors.New("invalid snapshot ID format: expected hostname_snapshotname_timestamp") } // Last part is the RFC3339 timestamp timestampStr := parts[len(parts)-1] + timestamp, err := time.Parse(time.RFC3339, timestampStr) if err != nil { return time.Time{}, fmt.Errorf("invalid timestamp: %w", err) @@ -80,17 +84,20 @@ func parseDuration(s string) (time.Duration, error) { } re := regexp.MustCompile(`(\d+)\s*([a-zA-Z]+)`) + matches := re.FindAllStringSubmatch(s, -1) if len(matches) == 0 { return 0, fmt.Errorf("invalid duration: %q", s) } var total time.Duration + for _, match := range matches { n, err := strconv.Atoi(match[1]) if err != nil { return 0, fmt.Errorf("invalid number %q: %w", match[1], err) } + unit := strings.ToLower(match[2]) switch unit { case "d", "day", "days": @@ -105,5 +112,6 @@ func parseDuration(s string) (time.Duration, error) { return 0, fmt.Errorf("unknown time unit %q", unit) } } + return total, nil } diff --git a/internal/vaultik/helpers_test.go b/internal/vaultik/helpers_test.go index 76a3ea3..e648bec 100644 --- a/internal/vaultik/helpers_test.go +++ b/internal/vaultik/helpers_test.go @@ -61,11 +61,14 @@ func TestParseDuration(t *testing.T) { if err == nil { t.Fatalf("expected error for %q, got %v", tt.input, got) } + return } + if err != nil { t.Fatalf("unexpected error for %q: %v", tt.input, err) } + if got != tt.want { t.Errorf("parseDuration(%q) = %v, want %v", tt.input, got, tt.want) } diff --git a/internal/vaultik/info.go b/internal/vaultik/info.go index ac5f76f..2046bf2 100644 --- a/internal/vaultik/info.go +++ b/internal/vaultik/info.go @@ -15,102 +15,122 @@ import ( // ShowInfo displays system and configuration information func (v *Vaultik) ShowInfo() error { // System Information - v.printfStdout("=== System Information ===\n") - v.printfStdout("OS/Architecture: %s/%s\n", runtime.GOOS, runtime.GOARCH) - v.printfStdout("Version: %s\n", v.Globals.Version) - v.printfStdout("Commit: %s\n", v.Globals.Commit) - v.printfStdout("Go Version: %s\n", runtime.Version()) + v.stdoutf("=== System Information ===\n") + v.stdoutf("OS/Architecture: %s/%s\n", runtime.GOOS, runtime.GOARCH) + v.stdoutf("Version: %s\n", v.Globals.Version) + v.stdoutf("Commit: %s\n", v.Globals.Commit) + v.stdoutf("Go Version: %s\n", runtime.Version()) v.printlnStdout() // Storage Configuration. The backend is selected by storage_url // (s3://, file://, rclone://); the legacy s3.* fields are only // printed when they're actually populated, since the URL scheme // is the primary configuration. - v.printfStdout("=== Storage Configuration ===\n") + v.stdoutf("=== Storage Configuration ===\n") storageInfo := v.Storage.Info() - v.printfStdout("Type: %s\n", storageInfo.Type) - v.printfStdout("Location: %s\n", storageInfo.Location) + v.stdoutf("Type: %s\n", storageInfo.Type) + v.stdoutf("Location: %s\n", storageInfo.Location) + if v.Config.StorageURL != "" { - v.printfStdout("Storage URL: %s\n", v.Config.StorageURL) + v.stdoutf("Storage URL: %s\n", v.Config.StorageURL) } + if v.Config.S3.Bucket != "" { - v.printfStdout("S3 Bucket: %s\n", v.Config.S3.Bucket) + v.stdoutf("S3 Bucket: %s\n", v.Config.S3.Bucket) } + if v.Config.S3.Prefix != "" { - v.printfStdout("S3 Prefix: %s\n", v.Config.S3.Prefix) + v.stdoutf("S3 Prefix: %s\n", v.Config.S3.Prefix) } + if v.Config.S3.Endpoint != "" { - v.printfStdout("S3 Endpoint: %s\n", v.Config.S3.Endpoint) + v.stdoutf("S3 Endpoint: %s\n", v.Config.S3.Endpoint) } + if v.Config.S3.Region != "" { - v.printfStdout("S3 Region: %s\n", v.Config.S3.Region) + v.stdoutf("S3 Region: %s\n", v.Config.S3.Region) } + v.printlnStdout() // Backup Settings - v.printfStdout("=== Backup Settings ===\n") + v.stdoutf("=== Backup Settings ===\n") // Show configured snapshots - v.printfStdout("Snapshots:\n") + v.stdoutf("Snapshots:\n") + for _, name := range v.Config.SnapshotNames() { snap := v.Config.Snapshots[name] - v.printfStdout(" %s:\n", name) + v.stdoutf(" %s:\n", name) + for _, path := range snap.Paths { - v.printfStdout(" - %s\n", path) + v.stdoutf(" - %s\n", path) } + if len(snap.Exclude) > 0 { - v.printfStdout(" exclude: %s\n", strings.Join(snap.Exclude, ", ")) + v.stdoutf(" exclude: %s\n", strings.Join(snap.Exclude, ", ")) } } // Global exclude patterns if len(v.Config.Exclude) > 0 { - v.printfStdout("Global Exclude: %s\n", strings.Join(v.Config.Exclude, ", ")) + v.stdoutf("Global Exclude: %s\n", strings.Join(v.Config.Exclude, ", ")) } - v.printfStdout("Compression: zstd level %d\n", v.Config.CompressionLevel) - v.printfStdout("Chunk Size: %s\n", humanize.Bytes(uint64(v.Config.ChunkSize))) - v.printfStdout("Blob Size Limit: %s\n", humanize.Bytes(uint64(v.Config.BlobSizeLimit))) + v.stdoutf("Compression: zstd level %d\n", v.Config.CompressionLevel) + v.stdoutf("Chunk Size: %s\n", humanize.Bytes(uint64(v.Config.ChunkSize))) + v.stdoutf("Blob Size Limit: %s\n", humanize.Bytes(uint64(v.Config.BlobSizeLimit))) v.printlnStdout() // Encryption Configuration - v.printfStdout("=== Encryption Configuration ===\n") - v.printfStdout("Recipients:\n") + v.stdoutf("=== Encryption Configuration ===\n") + v.stdoutf("Recipients:\n") + for _, recipient := range v.Config.AgeRecipients { - v.printfStdout(" - %s\n", recipient) + v.stdoutf(" - %s\n", recipient) } + v.printlnStdout() // Local Database - v.printfStdout("=== Local Database ===\n") - v.printfStdout("Index Path: %s\n", v.Config.IndexPath) + v.stdoutf("=== Local Database ===\n") + v.stdoutf("Index Path: %s\n", v.Config.IndexPath) // Check if index file exists and get its size if info, err := v.Fs.Stat(v.Config.IndexPath); err == nil { - v.printfStdout("Index Size: %s\n", humanize.Bytes(uint64(info.Size()))) + v.stdoutf("Index Size: %s\n", humanize.Bytes(uint64(info.Size()))) // Get snapshot count from database query := `SELECT COUNT(*) FROM snapshots WHERE completed_at IS NOT NULL` + var snapshotCount int - if err := v.DB.Conn().QueryRowContext(v.ctx, query).Scan(&snapshotCount); err == nil { - v.printfStdout("Snapshots: %d\n", snapshotCount) + + err := v.DB.Conn().QueryRowContext(v.ctx, query).Scan(&snapshotCount) + if err == nil { + v.stdoutf("Snapshots: %d\n", snapshotCount) } // Get blob count from database query = `SELECT COUNT(*) FROM blobs` + var blobCount int - if err := v.DB.Conn().QueryRowContext(v.ctx, query).Scan(&blobCount); err == nil { - v.printfStdout("Blobs: %d\n", blobCount) + + err = v.DB.Conn().QueryRowContext(v.ctx, query).Scan(&blobCount) + if err == nil { + v.stdoutf("Blobs: %d\n", blobCount) } // Get file count from database query = `SELECT COUNT(*) FROM files` + var fileCount int - if err := v.DB.Conn().QueryRowContext(v.ctx, query).Scan(&fileCount); err == nil { - v.printfStdout("Files: %d\n", fileCount) + + err = v.DB.Conn().QueryRowContext(v.ctx, query).Scan(&fileCount) + if err == nil { + v.stdoutf("Files: %d\n", fileCount) } } else { - v.printfStdout("Index Size: (not created)\n") + v.stdoutf("Index Size: (not created)\n") } return nil @@ -153,6 +173,7 @@ type RemoteInfoResult struct { // RemoteInfo displays information about remote storage func (v *Vaultik) RemoteInfo(jsonOutput bool) error { log.Info("Starting remote storage info gathering") + result := &RemoteInfoResult{} storageInfo := v.Storage.Info() @@ -160,11 +181,11 @@ func (v *Vaultik) RemoteInfo(jsonOutput bool) error { result.StorageLocation = storageInfo.Location if !jsonOutput { - v.printfStdout("=== Remote Storage ===\n") - v.printfStdout("Type: %s\n", storageInfo.Type) - v.printfStdout("Location: %s\n", storageInfo.Location) + v.stdoutf("=== Remote Storage ===\n") + v.stdoutf("Type: %s\n", storageInfo.Type) + v.stdoutf("Location: %s\n", storageInfo.Location) v.printlnStdout() - v.printfStdout("Scanning snapshot metadata...\n") + v.stdoutf("Scanning snapshot metadata...\n") } snapshotMetadata, snapshotIDs, err := v.collectSnapshotMetadata() @@ -173,7 +194,7 @@ func (v *Vaultik) RemoteInfo(jsonOutput bool) error { } if !jsonOutput { - v.printfStdout("Downloading %d manifest(s)...\n", len(snapshotIDs)) + v.stdoutf("Downloading %d manifest(s)...\n", len(snapshotIDs)) } referencedBlobs := v.collectReferencedBlobsFromManifests(snapshotIDs, snapshotMetadata) @@ -193,10 +214,12 @@ func (v *Vaultik) RemoteInfo(jsonOutput bool) error { if jsonOutput { enc := json.NewEncoder(v.Stdout) enc.SetIndent("", " ") + return enc.Encode(result) } v.printRemoteInfoTable(result) + return nil } @@ -214,6 +237,7 @@ func (v *Vaultik) collectSnapshotMetadata() (map[string]*SnapshotMetadataInfo, [ if len(parts) < 3 { continue } + snapshotID := parts[1] if _, exists := snapshotMetadata[snapshotID]; !exists { @@ -221,12 +245,14 @@ func (v *Vaultik) collectSnapshotMetadata() (map[string]*SnapshotMetadataInfo, [ } info := snapshotMetadata[snapshotID] + filename := parts[2] if strings.HasPrefix(filename, "manifest") { info.ManifestSize = obj.Size } else if strings.HasPrefix(filename, "db") { info.DatabaseSize = obj.Size } + info.TotalSize = info.ManifestSize + info.DatabaseSize } @@ -234,6 +260,7 @@ func (v *Vaultik) collectSnapshotMetadata() (map[string]*SnapshotMetadataInfo, [ for id := range snapshotMetadata { snapshotIDs = append(snapshotIDs, id) } + sort.Strings(snapshotIDs) return snapshotMetadata, snapshotIDs, nil @@ -245,26 +272,33 @@ func (v *Vaultik) collectReferencedBlobsFromManifests(snapshotIDs []string, snap for _, snapshotID := range snapshotIDs { manifestKey := fmt.Sprintf("metadata/%s/manifest.json.zst", snapshotID) + reader, err := v.Storage.Get(v.ctx, manifestKey) if err != nil { log.Warn("Failed to get manifest", "snapshot", snapshotID, "error", err) + continue } manifest, err := snapshot.DecodeManifest(reader) _ = reader.Close() + if err != nil { log.Warn("Failed to decode manifest", "snapshot", snapshotID, "error", err) + continue } info := snapshotMetadata[snapshotID] info.BlobCount = manifest.BlobCount + var blobsSize int64 + for _, blob := range manifest.Blobs { referencedBlobs[blob.Hash] = blob.CompressedSize blobsSize += blob.CompressedSize } + info.BlobsSize = blobsSize } @@ -274,11 +308,13 @@ func (v *Vaultik) collectReferencedBlobsFromManifests(snapshotIDs []string, snap // populateRemoteInfoResult fills in the result's snapshot and referenced blob stats func (v *Vaultik) populateRemoteInfoResult(result *RemoteInfoResult, snapshotMetadata map[string]*SnapshotMetadataInfo, snapshotIDs []string, referencedBlobs map[string]int64) { var totalMetadataSize int64 + for _, id := range snapshotIDs { info := snapshotMetadata[id] result.Snapshots = append(result.Snapshots, *info) totalMetadataSize += info.TotalSize } + result.TotalMetadataSize = totalMetadataSize result.TotalMetadataCount = len(snapshotIDs) @@ -291,7 +327,7 @@ func (v *Vaultik) populateRemoteInfoResult(result *RemoteInfoResult, snapshotMet // scanRemoteBlobStorage lists all blobs on remote and computes orphan stats func (v *Vaultik) scanRemoteBlobStorage(result *RemoteInfoResult, referencedBlobs map[string]int64, jsonOutput bool) error { if !jsonOutput { - v.printfStdout("Scanning blobs...\n") + v.stdoutf("Scanning blobs...\n") } blobCh := v.Storage.ListStream(v.ctx, "blobs/") @@ -301,10 +337,12 @@ func (v *Vaultik) scanRemoteBlobStorage(result *RemoteInfoResult, referencedBlob if obj.Err != nil { return fmt.Errorf("listing blobs: %w", obj.Err) } + parts := strings.Split(obj.Key, "/") if len(parts) < 4 { continue } + hash := parts[3] allBlobs[hash] = obj.Size result.TotalBlobCount++ @@ -323,14 +361,16 @@ func (v *Vaultik) scanRemoteBlobStorage(result *RemoteInfoResult, referencedBlob // printRemoteInfoTable renders the human-readable remote info output func (v *Vaultik) printRemoteInfoTable(result *RemoteInfoResult) { - v.printfStdout("\n=== Snapshot Metadata ===\n") + v.stdoutf("\n=== Snapshot Metadata ===\n") + if len(result.Snapshots) == 0 { - v.printfStdout("No snapshots found\n") + v.stdoutf("No snapshots found\n") } else { - v.printfStdout("%-45s %12s %12s %12s %10s %12s\n", "SNAPSHOT", "MANIFEST", "DATABASE", "TOTAL", "BLOBS", "BLOB SIZE") - v.printfStdout("%-45s %12s %12s %12s %10s %12s\n", strings.Repeat("-", 45), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 10), strings.Repeat("-", 12)) + v.stdoutf("%-45s %12s %12s %12s %10s %12s\n", "SNAPSHOT", "MANIFEST", "DATABASE", "TOTAL", "BLOBS", "BLOB SIZE") + v.stdoutf("%-45s %12s %12s %12s %10s %12s\n", strings.Repeat("-", 45), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 10), strings.Repeat("-", 12)) + for _, info := range result.Snapshots { - v.printfStdout("%-45s %12s %12s %12s %10s %12s\n", + v.stdoutf("%-45s %12s %12s %12s %10s %12s\n", truncateString(info.SnapshotID, 45), humanize.Bytes(uint64(info.ManifestSize)), humanize.Bytes(uint64(info.DatabaseSize)), @@ -339,20 +379,21 @@ func (v *Vaultik) printRemoteInfoTable(result *RemoteInfoResult) { humanize.Bytes(uint64(info.BlobsSize)), ) } - v.printfStdout("%-45s %12s %12s %12s %10s %12s\n", strings.Repeat("-", 45), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 10), strings.Repeat("-", 12)) - v.printfStdout("%-45s %12s %12s %12s\n", fmt.Sprintf("Total (%d snapshots)", result.TotalMetadataCount), "", "", humanize.Bytes(uint64(result.TotalMetadataSize))) + + v.stdoutf("%-45s %12s %12s %12s %10s %12s\n", strings.Repeat("-", 45), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 10), strings.Repeat("-", 12)) + v.stdoutf("%-45s %12s %12s %12s\n", fmt.Sprintf("Total (%d snapshots)", result.TotalMetadataCount), "", "", humanize.Bytes(uint64(result.TotalMetadataSize))) } - v.printfStdout("\n=== Blob Storage ===\n") - v.printfStdout("Total blobs on remote: %s (%s)\n", + v.stdoutf("\n=== Blob Storage ===\n") + v.stdoutf("Total blobs on remote: %s (%s)\n", humanize.Comma(int64(result.TotalBlobCount)), humanize.Bytes(uint64(result.TotalBlobSize))) - v.printfStdout("Referenced by snapshots: %s (%s)\n", + v.stdoutf("Referenced by snapshots: %s (%s)\n", humanize.Comma(int64(result.ReferencedBlobCount)), humanize.Bytes(uint64(result.ReferencedBlobSize))) - v.printfStdout("Orphaned (unreferenced): %s (%s)\n", + v.stdoutf("Orphaned (unreferenced): %s (%s)\n", humanize.Comma(int64(result.OrphanedBlobCount)), humanize.Bytes(uint64(result.OrphanedBlobSize))) if result.OrphanedBlobCount > 0 { - v.printfStdout("\nRun 'vaultik prune' to remove orphaned blobs.\n") + v.stdoutf("\nRun 'vaultik prune' to remove orphaned blobs.\n") } } @@ -361,8 +402,10 @@ func truncateString(s string, maxLen int) string { if len(s) <= maxLen { return s } + if maxLen <= 3 { return s[:maxLen] } + return s[:maxLen-3] + "..." } diff --git a/internal/vaultik/integration_test.go b/internal/vaultik/integration_test.go index 945d448..78a1c18 100644 --- a/internal/vaultik/integration_test.go +++ b/internal/vaultik/integration_test.go @@ -43,11 +43,14 @@ func (m *MockStorer) Put(ctx context.Context, key string, reader io.Reader) erro defer m.mu.Unlock() m.calls = append(m.calls, "Put:"+key) + data, err := io.ReadAll(reader) if err != nil { return err } + m.data[key] = data + return nil } @@ -60,10 +63,12 @@ func (m *MockStorer) Get(ctx context.Context, key string) (io.ReadCloser, error) defer m.mu.Unlock() m.calls = append(m.calls, "Get:"+key) + data, exists := m.data[key] if !exists { return nil, storage.ErrNotFound } + return io.NopCloser(bytes.NewReader(data)), nil } @@ -72,10 +77,12 @@ func (m *MockStorer) Stat(ctx context.Context, key string) (*storage.ObjectInfo, defer m.mu.Unlock() m.calls = append(m.calls, "Stat:"+key) + data, exists := m.data[key] if !exists { return nil, storage.ErrNotFound } + return &storage.ObjectInfo{ Key: key, Size: int64(len(data)), @@ -88,6 +95,7 @@ func (m *MockStorer) Delete(ctx context.Context, key string) error { m.calls = append(m.calls, "Delete:"+key) delete(m.data, key) + return nil } @@ -96,12 +104,15 @@ func (m *MockStorer) List(ctx context.Context, prefix string) ([]string, error) defer m.mu.Unlock() m.calls = append(m.calls, "List:"+prefix) + var keys []string + for key := range m.data { if len(prefix) == 0 || (len(key) >= len(prefix) && key[:len(prefix)] == prefix) { keys = append(keys, key) } } + return keys, nil } @@ -109,6 +120,7 @@ func (m *MockStorer) ListStream(ctx context.Context, prefix string) <-chan stora ch := make(chan storage.ObjectInfo) go func() { defer close(ch) + m.mu.Lock() defer m.mu.Unlock() @@ -121,6 +133,7 @@ func (m *MockStorer) ListStream(ctx context.Context, prefix string) <-chan stora } } }() + return ch } @@ -138,6 +151,7 @@ func (m *MockStorer) GetCalls() []string { calls := make([]string, len(m.calls)) copy(calls, m.calls) + return calls } @@ -172,14 +186,16 @@ func TestEndToEndBackup(t *testing.T) { "/home/user/code", } for _, dir := range dirs { - if err := fs.MkdirAll(dir, 0755); err != nil { + err := fs.MkdirAll(dir, 0755) + if err != nil { t.Fatalf("failed to create directory %s: %v", dir, err) } } // Create test files for path, content := range testFiles { - if err := afero.WriteFile(fs, path, []byte(content), 0644); err != nil { + err := afero.WriteFile(fs, path, []byte(content), 0644) + if err != nil { t.Fatalf("failed to create test file %s: %v", path, err) } } @@ -216,9 +232,11 @@ func TestEndToEndBackup(t *testing.T) { // Create in-memory database db, err := database.New(ctx, ":memory:") + require.NoError(t, err) defer func() { - if err := db.Close(); err != nil { + err := db.Close() + if err != nil { t.Errorf("failed to close database: %v", err) } }() @@ -246,6 +264,7 @@ func TestEndToEndBackup(t *testing.T) { VaultikVersion: "test-version", StartedAt: time.Now(), } + return repos.Snapshots.Create(ctx, tx, snapshot) }) require.NoError(t, err) @@ -258,9 +277,9 @@ func TestEndToEndBackup(t *testing.T) { // The scanner counts both files and directories, so we have: // 4 files + 4 directories (/home, /home/user, /home/user/documents, /home/user/pictures, /home/user/code) assert.GreaterOrEqual(t, result.FilesScanned, 4, "Should scan at least 4 files") - assert.Greater(t, result.BytesScanned, int64(0), "Should scan some bytes") - assert.Greater(t, result.ChunksCreated, 0, "Should create chunks") - assert.Greater(t, result.BlobsCreated, 0, "Should create blobs") + assert.Positive(t, result.BytesScanned, "Should scan some bytes") + assert.Positive(t, result.ChunksCreated, "Should create chunks") + assert.Positive(t, result.BlobsCreated, "Should create blobs") // Verify storage operations calls := mockStorage.GetCalls() @@ -268,6 +287,7 @@ func TestEndToEndBackup(t *testing.T) { // Should have uploaded at least one blob blobUploads := 0 + for _, call := range calls { if len(call) > 4 && call[:4] == "Put:" { if len(call) > 10 && call[4:10] == "blobs/" { @@ -275,27 +295,30 @@ func TestEndToEndBackup(t *testing.T) { } } } - assert.Greater(t, blobUploads, 0, "Should upload at least one blob") + + assert.Positive(t, blobUploads, "Should upload at least one blob") // Verify files in database files, err := repos.Files.ListByPrefix(ctx, "/home/user") require.NoError(t, err) // Count only regular files (not directories) regularFiles := 0 + for _, f := range files { if f.Mode&0x80000000 == 0 { // Check if regular file (not directory) regularFiles++ } } + assert.Equal(t, 4, regularFiles, "Should have 4 regular files in database") // Verify chunks were created by checking a specific file fileChunks, err := repos.FileChunks.GetByPath(ctx, "/home/user/documents/file1.txt") require.NoError(t, err) - assert.Greater(t, len(fileChunks), 0, "Should have chunks for file1.txt") + assert.NotEmpty(t, fileChunks, "Should have chunks for file1.txt") // Verify blobs were uploaded to storage - assert.Greater(t, mockStorage.GetStorageSize(), 0, "Should have blobs in storage") + assert.Positive(t, mockStorage.GetStorageSize(), "Should have blobs in storage") // Complete the snapshot - just verify we got results // In a real integration test, we'd update the snapshot record @@ -337,9 +360,11 @@ func TestBackupAndVerify(t *testing.T) { // Create test database ctx := context.Background() db, err := database.New(ctx, ":memory:") + require.NoError(t, err) defer func() { - if err := db.Close(); err != nil { + err := db.Close() + if err != nil { t.Errorf("failed to close database: %v", err) } }() @@ -366,6 +391,7 @@ func TestBackupAndVerify(t *testing.T) { VaultikVersion: "test-version", StartedAt: time.Now(), } + return repos.Snapshots.Create(ctx, tx, snapshot) }) require.NoError(t, err) @@ -375,7 +401,7 @@ func TestBackupAndVerify(t *testing.T) { require.NoError(t, err) // Verify backup created blobs - assert.Greater(t, result.BlobsCreated, 0, "Should create at least one blob") + assert.Positive(t, result.BlobsCreated, "Should create at least one blob") assert.Equal(t, mockStorage.GetStorageSize(), result.BlobsCreated, "Storage should have the blobs") // Verify we can retrieve the blob from storage @@ -391,18 +417,19 @@ func TestBackupAndVerify(t *testing.T) { // Get blob info blobInfo, err := mockStorage.Stat(ctx, blobKey) require.NoError(t, err) - assert.Greater(t, blobInfo.Size, int64(0), "Blob should have content") + assert.Positive(t, blobInfo.Size, "Blob should have content") // Get blob content reader, err := mockStorage.Get(ctx, blobKey) require.NoError(t, err) + defer func() { _ = reader.Close() }() // Verify blob data is encrypted (should not contain plaintext) blobData, err := io.ReadAll(reader) require.NoError(t, err) assert.NotContains(t, string(blobData), testContent, "Blob should be encrypted") - assert.Greater(t, len(blobData), 0, "Blob should have data") + assert.NotEmpty(t, blobData, "Blob should have data") } t.Logf("Backup and verify test completed successfully") @@ -418,6 +445,7 @@ func TestBackupAndRestore(t *testing.T) { // Create real temp directory for the database (SQLite needs real filesystem) realTempDir, err := os.MkdirTemp("", "vaultik-test-") require.NoError(t, err) + defer func() { _ = os.RemoveAll(realTempDir) }() // Use real OS filesystem for this test @@ -434,10 +462,14 @@ func TestBackupAndRestore(t *testing.T) { // Create directories and files for path, content := range testFiles { dir := filepath.Dir(path) - if err := fs.MkdirAll(dir, 0755); err != nil { + + err := fs.MkdirAll(dir, 0755) + if err != nil { t.Fatalf("failed to create directory %s: %v", dir, err) } - if err := afero.WriteFile(fs, path, []byte(content), 0644); err != nil { + + err = afero.WriteFile(fs, path, []byte(content), 0644) + if err != nil { t.Fatalf("failed to create test file %s: %v", path, err) } } @@ -455,6 +487,7 @@ func TestBackupAndRestore(t *testing.T) { dbPath := filepath.Join(realTempDir, "test.db") db, err := database.New(ctx, dbPath) require.NoError(t, err) + defer func() { _ = db.Close() }() repos := database.NewRepositories(db) @@ -558,6 +591,7 @@ func TestEndToEndFileStorage(t *testing.T) { fs := afero.NewOsFs() tempDir, err := os.MkdirTemp("", "vaultik-e2e-") require.NoError(t, err) + defer func() { _ = os.RemoveAll(tempDir) }() dataDir := filepath.Join(tempDir, "source") @@ -618,6 +652,7 @@ func TestEndToEndFileStorage(t *testing.T) { db, err := database.New(ctx, dbPath) require.NoError(t, err) + defer func() { _ = db.Close() }() repos := database.NewRepositories(db) @@ -644,8 +679,8 @@ func TestEndToEndFileStorage(t *testing.T) { scanResult, err := scanner.Scan(ctx, dataDir, snapshotID) require.NoError(t, err) - require.Greater(t, scanResult.FilesScanned, 0) - require.Greater(t, scanResult.BlobsCreated, 0) + require.Positive(t, scanResult.FilesScanned) + require.Positive(t, scanResult.BlobsCreated) require.NoError(t, sm.CompleteSnapshot(ctx, snapshotID)) require.NoError(t, sm.ExportSnapshotMetadata(ctx, dbPath, snapshotID)) @@ -656,6 +691,7 @@ func TestEndToEndFileStorage(t *testing.T) { blobInfo, err := os.Stat(filepath.Join(storeDir, "blobs")) require.NoError(t, err) require.True(t, blobInfo.IsDir()) + metaInfo, err := os.Stat(filepath.Join(storeDir, "metadata", snapshot.RemoteSnapshotKey(snapshotID))) require.NoError(t, err) require.True(t, metaInfo.IsDir()) @@ -721,6 +757,7 @@ func TestDedupOnlySnapshotRestores(t *testing.T) { fs := afero.NewOsFs() tempDir, err := os.MkdirTemp("", "vaultik-dedup-") require.NoError(t, err) + defer func() { _ = os.RemoveAll(tempDir) }() dataDir := filepath.Join(tempDir, "source") @@ -756,7 +793,9 @@ func TestDedupOnlySnapshotRestores(t *testing.T) { ctx := context.Background() db, err := database.New(ctx, dbPath) require.NoError(t, err) + defer func() { _ = db.Close() }() + repos := database.NewRepositories(db) makeScanner := func() *snapshot.Scanner { @@ -780,13 +819,14 @@ func TestDedupOnlySnapshotRestores(t *testing.T) { require.NoError(t, err) r1, err := makeScanner().Scan(ctx, dataDir, id1) require.NoError(t, err) - require.Greater(t, r1.BlobsCreated, 0, "first snapshot should upload at least one blob") + require.Positive(t, r1.BlobsCreated, "first snapshot should upload at least one blob") require.NoError(t, sm.CompleteSnapshot(ctx, id1)) require.NoError(t, sm.ExportSnapshotMetadata(ctx, dbPath, id1)) // Second snapshot — same data, every chunk dedups. Sleep past the // second-precision timestamp so the snapshot IDs differ. time.Sleep(1100 * time.Millisecond) + id2, err := sm.CreateSnapshotWithName(ctx, cfg.Hostname, "dedup", "v", "g") require.NoError(t, err) r2, err := makeScanner().Scan(ctx, dataDir, id2) @@ -833,5 +873,6 @@ func bytesPattern(tag string, n int) []byte { for i := range out { out[i] = byte(tag[i%len(tag)] ^ byte(i&0xff)) } + return out } diff --git a/internal/vaultik/prune.go b/internal/vaultik/prune.go index 8b3cfac..67cd06c 100644 --- a/internal/vaultik/prune.go +++ b/internal/vaultik/prune.go @@ -2,6 +2,7 @@ package vaultik import ( "encoding/json" + "errors" "fmt" "strings" @@ -23,20 +24,24 @@ type PruneOptions struct { // confirming with the user. func (v *Vaultik) NukeRemote(force bool) error { if !force { - return fmt.Errorf("nuke requires --force (this deletes ALL remote snapshots and blobs)") + return errors.New("nuke requires --force (this deletes ALL remote snapshots and blobs)") } - v.UI.Begin("Removing all snapshot metadata from backup destination store.") + v.UI.Beginf("Removing all snapshot metadata from backup destination store.") + if _, err := v.RemoveAllSnapshots(&RemoveOptions{Force: true}); err != nil { return fmt.Errorf("removing all snapshots: %w", err) } - v.UI.Begin("Removing any blobs still present in backup destination store.") - if err := v.PruneBlobs(&PruneOptions{Force: true}); err != nil { + v.UI.Beginf("Removing any blobs still present in backup destination store.") + + err := v.PruneBlobs(&PruneOptions{Force: true}) + if err != nil { return fmt.Errorf("pruning blobs: %w", err) } - v.UI.Complete("Backup destination store is now empty.") + v.UI.Completef("Backup destination store is now empty.") + return nil } @@ -55,7 +60,8 @@ type PruneBlobsResult struct { // prefer this method over PruneDatabase or PruneBlobs individually // unless it specifically wants one half. func (v *Vaultik) Prune(opts *PruneOptions) error { - if err := v.EnsureStorageBinding(); err != nil { + err := v.EnsureStorageBinding() + if err != nil { return err } // First reconcile local snapshot records against remote metadata: @@ -63,12 +69,15 @@ func (v *Vaultik) Prune(opts *PruneOptions) error { // store is treated as gone. This used to be the separate 'snapshot // cleanup' command and is now folded in so a single 'vaultik prune' // gets the local index fully back in sync with the destination. - if err := v.CleanupLocalSnapshots(); err != nil { + err = v.CleanupLocalSnapshots() + if err != nil { return fmt.Errorf("reconciling local snapshots with remote: %w", err) } + if _, err := v.PruneDatabase(); err != nil { return fmt.Errorf("pruning local database: %w", err) } + return v.PruneBlobs(opts) } @@ -92,27 +101,35 @@ func (v *Vaultik) PruneBlobs(opts *PruneOptions) error { if len(unreferencedBlobs) == 0 { log.Info("No unreferenced blobs found") + if opts.JSON { return v.outputPruneBlobsJSON(result) } + v.printlnStdout("No unreferenced blobs to remove.") + return nil } log.Info("Found unreferenced blobs", "count", len(unreferencedBlobs), "total_size", humanize.Bytes(uint64(totalSize))) + if !opts.JSON { - v.printfStdout("Found %d unreferenced blob(s) totaling %s\n", len(unreferencedBlobs), humanize.Bytes(uint64(totalSize))) + v.stdoutf("Found %d unreferenced blob(s) totaling %s\n", len(unreferencedBlobs), humanize.Bytes(uint64(totalSize))) } if !opts.Force && !opts.JSON { - v.printfStdout("\nDelete %d unreferenced blob(s)? [y/N] ", len(unreferencedBlobs)) + v.stdoutf("\nDelete %d unreferenced blob(s)? [y/N] ", len(unreferencedBlobs)) + var confirm string if _, err := v.scanStdin(&confirm); err != nil { v.printlnStdout("Cancelled") + return nil } + if strings.ToLower(confirm) != "y" { v.printlnStdout("Cancelled") + return nil } } @@ -123,9 +140,10 @@ func (v *Vaultik) PruneBlobs(opts *PruneOptions) error { return v.outputPruneBlobsJSON(result) } - v.printfStdout("\nDeleted %d blob(s) totaling %s\n", result.BlobsDeleted, humanize.Bytes(uint64(result.BytesFreed))) + v.stdoutf("\nDeleted %d blob(s) totaling %s\n", result.BlobsDeleted, humanize.Bytes(uint64(result.BytesFreed))) + if result.BlobsFailed > 0 { - v.printfStdout("Failed to delete %d blob(s)\n", result.BlobsFailed) + v.stdoutf("Failed to delete %d blob(s)\n", result.BlobsFailed) } return nil @@ -140,6 +158,7 @@ func (v *Vaultik) collectReferencedBlobs() (map[string]bool, error) { if err != nil { return nil, fmt.Errorf("listing snapshot keys: %w", err) } + log.Info("Found manifests in remote storage", "count", len(remoteKeys)) allBlobsReferenced := make(map[string]bool) @@ -147,18 +166,23 @@ func (v *Vaultik) collectReferencedBlobs() (map[string]bool, error) { for _, remoteKey := range remoteKeys { log.Debug("Processing manifest", "remote_key", remoteKey) + manifest, err := v.downloadManifestByKey(remoteKey) if err != nil { log.Error("Failed to download manifest", "remote_key", remoteKey, "error", err) + continue } + for _, blob := range manifest.Blobs { allBlobsReferenced[blob.Hash] = true } + manifestCount++ } log.Info("Processed manifests", "count", manifestCount, "unique_blobs_referenced", len(allBlobsReferenced)) + return allBlobsReferenced, nil } @@ -166,12 +190,14 @@ func (v *Vaultik) collectReferencedBlobs() (map[string]bool, error) { func (v *Vaultik) listUniqueSnapshotIDs() ([]string, error) { objectCh := v.Storage.ListStream(v.ctx, "metadata/") seen := make(map[string]bool) + var snapshotIDs []string for object := range objectCh { if object.Err != nil { return nil, fmt.Errorf("listing metadata objects: %w", object.Err) } + parts := strings.Split(object.Key, "/") if len(parts) >= 2 && parts[0] == "metadata" && parts[1] != "" { if strings.HasSuffix(object.Key, "/") || strings.Contains(object.Key, "/manifest.json.zst") { @@ -183,12 +209,14 @@ func (v *Vaultik) listUniqueSnapshotIDs() ([]string, error) { } } } + return snapshotIDs, nil } // listAllRemoteBlobs returns a map of all blob hashes to their sizes in remote storage func (v *Vaultik) listAllRemoteBlobs() (map[string]int64, error) { log.Info("Listing all blobs in storage") + allBlobs := make(map[string]int64) blobObjectCh := v.Storage.ListStream(v.ctx, "blobs/") @@ -196,6 +224,7 @@ func (v *Vaultik) listAllRemoteBlobs() (map[string]int64, error) { if object.Err != nil { return nil, fmt.Errorf("listing blobs: %w", object.Err) } + parts := strings.Split(object.Key, "/") if len(parts) == 4 && parts[0] == "blobs" { allBlobs[parts[3]] = object.Size @@ -203,19 +232,24 @@ func (v *Vaultik) listAllRemoteBlobs() (map[string]int64, error) { } log.Info("Found blobs in storage", "count", len(allBlobs)) + return allBlobs, nil } // findUnreferencedBlobs returns blob hashes not referenced by any manifest and their total size func (v *Vaultik) findUnreferencedBlobs(allBlobs map[string]int64, referenced map[string]bool) ([]string, int64) { - var unreferenced []string - var totalSize int64 + var ( + unreferenced []string + totalSize int64 + ) + for hash, size := range allBlobs { if !referenced[hash] { unreferenced = append(unreferenced, hash) totalSize += size } } + return unreferenced, totalSize } @@ -226,8 +260,10 @@ func (v *Vaultik) deleteUnreferencedBlobs(unreferencedBlobs []string, allBlobs m for i, hash := range unreferencedBlobs { blobPath := fmt.Sprintf("blobs/%s/%s/%s", hash[:2], hash[2:4], hash) - if err := v.Storage.Delete(v.ctx, blobPath); err != nil { + err := v.Storage.Delete(v.ctx, blobPath) + if err != nil { log.Error("Failed to delete blob", "hash", hash, "error", err) + continue } @@ -256,5 +292,6 @@ func (v *Vaultik) deleteUnreferencedBlobs(unreferencedBlobs []string, allBlobs m func (v *Vaultik) outputPruneBlobsJSON(result *PruneBlobsResult) error { encoder := json.NewEncoder(v.Stdout) encoder.SetIndent("", " ") + return encoder.Encode(result) } diff --git a/internal/vaultik/purge_per_name_test.go b/internal/vaultik/purge_per_name_test.go index ff86be3..b8c4782 100644 --- a/internal/vaultik/purge_per_name_test.go +++ b/internal/vaultik/purge_per_name_test.go @@ -79,16 +79,19 @@ func setupPurgeTest(t *testing.T, snapshotIDs []string) *vaultik.Vaultik { // listRemainingSnapshots returns IDs of all completed snapshots in the database. func listRemainingSnapshots(t *testing.T, v *vaultik.Vaultik) []string { t.Helper() + ctx := context.Background() dbSnaps, err := v.Repositories.Snapshots.ListRecent(ctx, 10000) require.NoError(t, err) var ids []string + for _, s := range dbSnaps { if s.CompletedAt != nil { ids = append(ids, s.ID.String()) } } + return ids } diff --git a/internal/vaultik/remove_snapshot_test.go b/internal/vaultik/remove_snapshot_test.go index 5dd03ec..2bbf33f 100644 --- a/internal/vaultik/remove_snapshot_test.go +++ b/internal/vaultik/remove_snapshot_test.go @@ -37,7 +37,9 @@ func (s *testStorer) Put(ctx context.Context, key string, reader io.Reader) erro if err != nil { return err } + s.data[key] = data + return nil } @@ -53,6 +55,7 @@ func (s *testStorer) Get(ctx context.Context, key string) (io.ReadCloser, error) if !exists { return nil, storage.ErrNotFound } + return io.NopCloser(bytes.NewReader(data)), nil } @@ -64,6 +67,7 @@ func (s *testStorer) Stat(ctx context.Context, key string) (*storage.ObjectInfo, if !exists { return nil, storage.ErrNotFound } + return &storage.ObjectInfo{ Key: key, Size: int64(len(data)), @@ -75,6 +79,7 @@ func (s *testStorer) Delete(ctx context.Context, key string) error { defer s.mu.Unlock() delete(s.data, key) + return nil } @@ -83,11 +88,13 @@ func (s *testStorer) List(ctx context.Context, prefix string) ([]string, error) defer s.mu.Unlock() var keys []string + for key := range s.data { if prefix == "" || strings.HasPrefix(key, prefix) { keys = append(keys, key) } } + return keys, nil } @@ -96,6 +103,7 @@ func (s *testStorer) ListStream(ctx context.Context, prefix string) <-chan stora go func() { defer close(ch) + s.mu.Lock() defer s.mu.Unlock() @@ -115,13 +123,16 @@ func (s *testStorer) ListStream(ctx context.Context, prefix string) <-chan stora func (s *testStorer) hasKey(key string) bool { s.mu.Lock() defer s.mu.Unlock() + _, exists := s.data[key] + return exists } func (s *testStorer) keyCount() int { s.mu.Lock() defer s.mu.Unlock() + return len(s.data) } @@ -175,6 +186,7 @@ func addBlob(t *testing.T, store *testStorer, hash string) { // Create zstd compressed data var buf bytes.Buffer + writer, _ := zstd.NewWriter(&buf) _, _ = writer.Write([]byte("blob data")) _ = writer.Close() @@ -366,7 +378,7 @@ func TestRemoveAllSnapshots_NoSnapshots(t *testing.T) { result, err := tv.RemoveAllSnapshots(opts) require.NoError(t, err) - assert.Len(t, result.SnapshotsRemoved, 0) + assert.Empty(t, result.SnapshotsRemoved) // Verify output assert.Contains(t, tv.Stdout.String(), "No snapshots found") diff --git a/internal/vaultik/restore.go b/internal/vaultik/restore.go index bb7715c..7eebf4f 100644 --- a/internal/vaultik/restore.go +++ b/internal/vaultik/restore.go @@ -5,6 +5,7 @@ import ( "context" "crypto/sha256" "encoding/hex" + "errors" "fmt" "io" "math" @@ -62,16 +63,20 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error { // Step 1: Download and decrypt the snapshot metadata database log.Info("Downloading snapshot metadata...") + tempDB, err := v.downloadSnapshotDB(opts.SnapshotID, identity) if err != nil { return fmt.Errorf("downloading snapshot database: %w", err) } + defer func() { - if err := tempDB.Close(); err != nil { + err := tempDB.Close() + if err != nil { log.Debug("Failed to close temp database", "error", err) } // Clean up temp file - if err := v.Fs.Remove(tempDB.Path()); err != nil { + err = v.Fs.Remove(tempDB.Path()) + if err != nil { log.Debug("Failed to remove temp database", "error", err) } }() @@ -86,12 +91,13 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error { if len(files) == 0 { log.Warn("No files found to restore") - v.UI.Warning("No files found to restore.") + v.UI.Warningf("No files found to restore.") + return nil } log.Info("Found files to restore", "count", len(files)) - v.UI.Info("Found %s files to restore.", v.UI.Count(len(files))) + v.UI.Infof("Found %s files to restore.", v.UI.Count(len(files))) // Step 3: Create target directory if err := v.Fs.MkdirAll(opts.TargetDir, 0755); err != nil { @@ -120,26 +126,28 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error { "duration", result.Duration, ) - v.UI.Complete("Restored %s files (%s) in %s.", + v.UI.Completef("Restored %s files (%s) in %s.", v.UI.Count(result.FilesRestored), v.UI.Size(result.BytesRestored), v.UI.Duration(result.Duration), ) if os.Geteuid() != 0 { - v.UI.Warning("Restore did not preserve file ownership: chown(2) requires root. Re-run as root (e.g. with sudo) if you need original UID/GID preserved.") + v.UI.Warningf("Restore did not preserve file ownership: chown(2) requires root. Re-run as root (e.g. with sudo) if you need original UID/GID preserved.") } if result.FilesFailed > 0 { - v.UI.Warning("%d file(s) failed to restore:", result.FilesFailed) + v.UI.Warningf("%d file(s) failed to restore:", result.FilesFailed) + for _, path := range result.FailedFiles { - v.UI.Detail("%s", v.UI.Path(path)) + v.UI.Detailf("%s", v.UI.Path(path)) } } // Run verification if requested if opts.Verify { - if err := v.handleRestoreVerification(repos, files, opts, result); err != nil { + err := v.handleRestoreVerification(repos, files, opts, result) + if err != nil { return err } } @@ -154,13 +162,14 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error { // prepareRestoreIdentity validates that an age secret key is configured and parses it func (v *Vaultik) prepareRestoreIdentity() (age.Identity, error) { if v.Config.AgeSecretKey == "" { - return nil, fmt.Errorf("decryption key required for restore\n\nSet the VAULTIK_AGE_SECRET_KEY environment variable to your age private key:\n export VAULTIK_AGE_SECRET_KEY='AGE-SECRET-KEY-...'") + return nil, errors.New("decryption key required for restore\n\nSet the VAULTIK_AGE_SECRET_KEY environment variable to your age private key:\n export VAULTIK_AGE_SECRET_KEY='AGE-SECRET-KEY-...'") } identity, err := age.ParseX25519Identity(v.Config.AgeSecretKey) if err != nil { return nil, fmt.Errorf("parsing age secret key: %w", err) } + return identity, nil } @@ -187,13 +196,16 @@ func (v *Vaultik) restoreAllFiles( if err != nil { return nil, fmt.Errorf("creating blob cache: %w", err) } + if v.restoreCacheObserver != nil { v.restoreCacheObserver(blobCache) } + defer func() { if v.restoreCacheObserver != nil { v.restoreCacheObserver(blobCache) } + _ = blobCache.Close() }() @@ -208,7 +220,9 @@ func (v *Vaultik) restoreAllFiles( if err != nil { return nil, fmt.Errorf("fetching blob index: %w", err) } + blobIDToHash := make(map[string]string, len(blobsByID)) + blobByHash := make(map[string]*database.Blob, len(blobsByID)) for id, blob := range blobsByID { hash := blob.Hash.String() @@ -234,7 +248,7 @@ func (v *Vaultik) restoreAllFiles( totalBytesExpected += file.Size } - v.UI.Begin("Restoring %s files (%s) to %s.", + v.UI.Beginf("Restoring %s files (%s) to %s.", v.UI.Count(len(files)), v.UI.Size(totalBytesExpected), v.UI.Path(opts.TargetDir)) @@ -257,9 +271,11 @@ func (v *Vaultik) restoreAllFiles( // Periodic progress output, matching the snapshot create cadence. startTime := time.Now() lastStatusTime := startTime + const statusInterval = 15 * time.Second processed := 0 + for plan.hasPending() { if v.ctx.Err() != nil { return nil, v.ctx.Err() @@ -282,31 +298,44 @@ func (v *Vaultik) restoreAllFiles( if next.IsZero() { break } + for _, hash := range plan.blobsNeeded(next) { blob, ok := blobByHash[hash] if !ok { return nil, fmt.Errorf("blob hash %s missing from blob index", hash[:16]) } - if err := session.downloadBlobToCache(hash, blob.CompressedSize); err != nil { + + err := session.downloadBlobToCache(hash, blob.CompressedSize) + if err != nil { return nil, fmt.Errorf("downloading blob %s: %w", hash[:16], err) } + result.BlobsDownloaded++ result.BytesDownloaded += blob.CompressedSize + plan.markBlobCached(hash) } + continue } file := filesByID[fileID] - if err := session.restoreFile(file); err != nil { + + err := session.restoreFile(file) + if err != nil { log.Error("Failed to restore file", "path", file.Path, "error", err) + if !opts.SkipErrors { return nil, fmt.Errorf("restoring %s: %w (pass --skip-errors to continue past restore failures)", file.Path, err) } - v.UI.Error("Failed to restore %s: %v. Skipping (--skip-errors).", v.UI.Path(file.Path.String()), err) + + v.UI.Errorf("Failed to restore %s: %v. Skipping (--skip-errors).", v.UI.Path(file.Path.String()), err) + result.FilesFailed++ result.FailedFiles = append(result.FailedFiles, file.Path.String()) + plan.finishFile(fileID) + continue } @@ -315,10 +344,12 @@ func (v *Vaultik) restoreAllFiles( // plan's indexes so future picks ignore it. sweeper.fileRestored(fileID.String()) plan.finishFile(fileID) + processed++ if time.Since(lastStatusTime) >= statusInterval { v.printRestoreProgress(processed, len(files), result.BytesRestored, totalBytesExpected, startTime) + lastStatusTime = time.Now() } @@ -344,13 +375,14 @@ func (v *Vaultik) printRestoreProgress(filesDone, totalFiles int, bytesDone, tot fileRate := float64(filesDone) / elapsed.Seconds() remainingBytes := totalBytes - bytesDone + var eta time.Duration if byteRate > 0 && remainingBytes > 0 { eta = time.Duration(float64(remainingBytes)/byteRate) * time.Second } if eta > 0 { - v.UI.Progress("Restore: %s/%s files (%s), %s/%s, %s, %.0f files/sec, restore elapsed: %s, restore ETA: %s (est remain %s).", + v.UI.Progressf("Restore: %s/%s files (%s), %s/%s, %s, %.0f files/sec, restore elapsed: %s, restore ETA: %s (est remain %s).", v.UI.Count(filesDone), v.UI.Count(totalFiles), v.UI.Percent(pct), @@ -361,9 +393,11 @@ func (v *Vaultik) printRestoreProgress(filesDone, totalFiles int, bytesDone, tot v.UI.Duration(elapsed), v.UI.Time(time.Now().Add(eta)), v.UI.Duration(eta)) + return } - v.UI.Progress("Restore: %s/%s files (%s), %s/%s, %s, %.0f files/sec, restore elapsed: %s.", + + v.UI.Progressf("Restore: %s/%s files (%s), %s/%s, %s, %.0f files/sec, restore elapsed: %s.", v.UI.Count(filesDone), v.UI.Count(totalFiles), v.UI.Percent(pct), @@ -381,22 +415,26 @@ func (v *Vaultik) handleRestoreVerification( opts *RestoreOptions, result *RestoreResult, ) error { - if err := v.verifyRestoredFiles(v.ctx, repos, files, opts.TargetDir, result); err != nil { + err := v.verifyRestoredFiles(v.ctx, repos, files, opts.TargetDir, result) + if err != nil { return fmt.Errorf("verification failed: %w", err) } if result.FilesFailed > 0 { - v.UI.Error("Verification failed: %s files did not match expected checksums.", + v.UI.Errorf("Verification failed: %s files did not match expected checksums.", v.UI.Count(result.FilesFailed)) + for _, path := range result.FailedFiles { - v.UI.Detail("%s", v.UI.Path(path)) + v.UI.Detailf("%s", v.UI.Path(path)) } + return fmt.Errorf("%d files failed verification", result.FilesFailed) } - v.UI.Complete("Verified %s files (%s).", + v.UI.Completef("Verified %s files (%s).", v.UI.Count(result.FilesVerified), v.UI.Size(result.BytesVerified)) + return nil } @@ -418,6 +456,7 @@ func (v *Vaultik) downloadSnapshotDB(snapshotID string, identity age.Identity) ( if err != nil { return nil, fmt.Errorf("reading encrypted data: %w", err) } + log.Debug("Downloaded encrypted database", "size", humanize.Bytes(uint64(len(encryptedData)))) // Decrypt and decompress using blobgen.Reader @@ -432,6 +471,7 @@ func (v *Vaultik) downloadSnapshotDB(snapshotID string, identity age.Identity) ( if err != nil { return nil, fmt.Errorf("decrypting and decompressing: %w", err) } + log.Debug("Decrypted database", "size", humanize.Bytes(uint64(len(dbData)))) // Create a temporary database file and write the binary SQLite data directly @@ -439,18 +479,23 @@ func (v *Vaultik) downloadSnapshotDB(snapshotID string, identity age.Identity) ( if err != nil { return nil, fmt.Errorf("creating temp file: %w", err) } + tempPath := tempFile.Name() // Write the binary SQLite database directly if _, err := tempFile.Write(dbData); err != nil { _ = tempFile.Close() _ = v.Fs.Remove(tempPath) + return nil, fmt.Errorf("writing database file: %w", err) } + if err := tempFile.Close(); err != nil { _ = v.Fs.Remove(tempPath) + return nil, fmt.Errorf("closing temp file: %w", err) } + log.Debug("Created restore database", "path", tempPath) // Open the database @@ -471,6 +516,7 @@ func (v *Vaultik) getFilesToRestore(ctx context.Context, repos *database.Reposit // Get files matching the path filters var result []*database.File + seen := make(map[string]bool) for _, filter := range pathFilters { @@ -498,23 +544,30 @@ func (v *Vaultik) getFilesToRestore(ctx context.Context, repos *database.Reposit func (v *Vaultik) buildChunkToBlobMap(ctx context.Context, repos *database.Repositories) (map[string]*database.BlobChunk, error) { // Query all blob_chunks query := `SELECT blob_id, chunk_hash, offset, length FROM blob_chunks` + rows, err := repos.DB().Conn().QueryContext(ctx, query) if err != nil { return nil, fmt.Errorf("querying blob_chunks: %w", err) } + defer func() { _ = rows.Close() }() result := make(map[string]*database.BlobChunk) + for rows.Next() { - var bc database.BlobChunk - var blobIDStr, chunkHashStr string + var ( + bc database.BlobChunk + blobIDStr, chunkHashStr string + ) if err := rows.Scan(&blobIDStr, &chunkHashStr, &bc.Offset, &bc.Length); err != nil { return nil, fmt.Errorf("scanning blob_chunk: %w", err) } + blobID, err := types.ParseBlobID(blobIDStr) if err != nil { return nil, fmt.Errorf("parsing blob ID: %w", err) } + bc.BlobID = blobID bc.ChunkHash = types.ChunkHash(chunkHashStr) result[chunkHashStr] = &bc @@ -553,16 +606,22 @@ type restoreSession struct { // restoreFile dispatches to the right per-kind restorer. func (s *restoreSession) restoreFile(file *database.File) error { targetPath := filepath.Join(s.opts.TargetDir, file.Path.String()) + parentDir := filepath.Dir(targetPath) - if err := s.v.Fs.MkdirAll(parentDir, 0755); err != nil { + + err := s.v.Fs.MkdirAll(parentDir, 0755) + if err != nil { return fmt.Errorf("creating parent directory: %w", err) } + if file.IsSymlink() { return s.restoreSymlink(file, targetPath) } + if file.Mode&uint32(os.ModeDir) != 0 { return s.restoreDirectory(file, targetPath) } + return s.restoreRegularFile(file, targetPath) } @@ -572,37 +631,50 @@ func (s *restoreSession) restoreSymlink(file *database.File, targetPath string) // afero.MemMapFs doesn't support symlinks, so route real-FS // symlinks through os. if _, ok := s.v.Fs.(*afero.OsFs); ok { - if err := os.Symlink(file.LinkTarget.String(), targetPath); err != nil { + err := os.Symlink(file.LinkTarget.String(), targetPath) + if err != nil { return fmt.Errorf("creating symlink: %w", err) } } else { log.Debug("Symlink creation not supported on this filesystem", "path", file.Path, "target", file.LinkTarget) } + s.result.FilesRestored++ + log.Debug("Restored symlink", "path", file.Path, "target", file.LinkTarget) + return nil } // restoreDirectory restores a directory with its permissions, mtime, // and (on real filesystems, with sufficient privileges) ownership. func (s *restoreSession) restoreDirectory(file *database.File, targetPath string) error { - if err := s.v.Fs.MkdirAll(targetPath, os.FileMode(file.Mode)); err != nil { + err := s.v.Fs.MkdirAll(targetPath, os.FileMode(file.Mode)) + if err != nil { return fmt.Errorf("creating directory: %w", err) } - if err := s.v.Fs.Chmod(targetPath, os.FileMode(file.Mode)); err != nil { + + err = s.v.Fs.Chmod(targetPath, os.FileMode(file.Mode)) + if err != nil { log.Debug("Failed to set directory permissions", "path", targetPath, "error", err) } + if s.runningAsRoot { if _, ok := s.v.Fs.(*afero.OsFs); ok { - if err := os.Chown(targetPath, int(file.UID), int(file.GID)); err != nil { + err := os.Chown(targetPath, int(file.UID), int(file.GID)) + if err != nil { log.Debug("Failed to set directory ownership", "path", targetPath, "error", err) } } } - if err := s.v.Fs.Chtimes(targetPath, file.MTime, file.MTime); err != nil { + + err = s.v.Fs.Chtimes(targetPath, file.MTime, file.MTime) + if err != nil { log.Debug("Failed to set directory mtime", "path", targetPath, "error", err) } + s.result.FilesRestored++ + return nil } @@ -617,16 +689,20 @@ func (s *restoreSession) restoreRegularFile(file *database.File, targetPath stri t0 := time.Now() fileChunks, err := s.repos.FileChunks.GetByFileID(s.ctx, file.ID) fileChunksQueryDur := time.Since(t0) + if err != nil { return fmt.Errorf("getting file chunks: %w", err) } t0 = time.Now() + outFile, err := s.v.Fs.Create(targetPath) createDur := time.Since(t0) + if err != nil { return fmt.Errorf("creating output file: %w", err) } + defer func() { _ = outFile.Close() }() var ( @@ -638,10 +714,12 @@ func (s *restoreSession) restoreRegularFile(file *database.File, targetPath stri for _, fc := range fileChunks { chunkHashStr := fc.ChunkHash.String() + blobChunk, ok := s.chunkToBlobMap[chunkHashStr] if !ok { return fmt.Errorf("chunk %s not found in any blob", chunkHashStr[:16]) } + blobHash, ok := s.blobIDToHash[blobChunk.BlobID.String()] if !ok { return fmt.Errorf("blob id %s missing from hash index", blobChunk.BlobID) @@ -650,6 +728,7 @@ func (s *restoreSession) restoreRegularFile(file *database.File, targetPath stri t0 = time.Now() chunkData, err := s.blobCache.ReadAt(blobHash, blobChunk.Offset, blobChunk.Length) readAtDur += time.Since(t0) + if err != nil { return fmt.Errorf("reading chunk %s from cached blob %s: %w", fc.ChunkHash[:16], blobHash[:16], err) } @@ -657,13 +736,17 @@ func (s *restoreSession) restoreRegularFile(file *database.File, targetPath stri t0 = time.Now() n, err := outFile.Write(chunkData) writeDur += time.Since(t0) + if err != nil { return fmt.Errorf("writing chunk: %w", err) } + bytesWritten += int64(n) t0 = time.Now() + s.sweeper.chunkRestored(int64(n)) + sweeperDur += time.Since(t0) } @@ -682,16 +765,20 @@ func (s *restoreSession) restoreRegularFile(file *database.File, targetPath stri if err := outFile.Close(); err != nil { return fmt.Errorf("closing output file: %w", err) } + if err := s.v.Fs.Chmod(targetPath, os.FileMode(file.Mode)); err != nil { log.Debug("Failed to set file permissions", "path", targetPath, "error", err) } + if s.runningAsRoot { if _, ok := s.v.Fs.(*afero.OsFs); ok { - if err := os.Chown(targetPath, int(file.UID), int(file.GID)); err != nil { + err := os.Chown(targetPath, int(file.UID), int(file.GID)) + if err != nil { log.Debug("Failed to set file ownership", "path", targetPath, "error", err) } } } + if err := s.v.Fs.Chtimes(targetPath, file.MTime, file.MTime); err != nil { log.Debug("Failed to set file mtime", "path", targetPath, "error", err) } @@ -700,6 +787,7 @@ func (s *restoreSession) restoreRegularFile(file *database.File, targetPath stri s.result.BytesRestored += bytesWritten log.Debug("Restored file", "path", file.Path, "size", humanize.Bytes(uint64(bytesWritten))) + return nil } @@ -715,6 +803,7 @@ func (s *restoreSession) downloadBlobToCache(blobHash string, expectedSize int64 t0 := time.Now() rc, err := s.v.FetchAndDecryptBlob(s.ctx, blobHash, expectedSize, s.identity) fetchSetupDur := time.Since(t0) + if err != nil { return err } @@ -723,9 +812,11 @@ func (s *restoreSession) downloadBlobToCache(blobHash string, expectedSize int64 written, copyErr := s.blobCache.PutFromReader(blobHash, rc) streamDur := time.Since(t0) closeErr := rc.Close() + if copyErr != nil { return copyErr } + if closeErr != nil { return closeErr } @@ -738,6 +829,7 @@ func (s *restoreSession) downloadBlobToCache(blobHash string, expectedSize int64 "ms_fetch_setup", fetchSetupDur.Milliseconds(), "ms_stream_decrypt_decompress", streamDur.Milliseconds(), ) + return nil } @@ -751,18 +843,21 @@ func (v *Vaultik) verifyRestoredFiles( ) error { // Calculate total bytes to verify for progress bar var totalBytes int64 + regularFiles := make([]*database.File, 0, len(files)) for _, file := range files { // Skip symlinks and directories - only verify regular files if file.IsSymlink() || file.Mode&uint32(os.ModeDir) != 0 { continue } + regularFiles = append(regularFiles, file) totalBytes += file.Size } if len(regularFiles) == 0 { log.Info("No regular files to verify") + return nil } @@ -770,34 +865,40 @@ func (v *Vaultik) verifyRestoredFiles( "files", len(regularFiles), "bytes", humanize.Bytes(uint64(totalBytes)), ) - v.UI.Begin("Verifying %s files (%s).", + v.UI.Beginf("Verifying %s files (%s).", v.UI.Count(len(regularFiles)), v.UI.Size(totalBytes)) startTime := time.Now() lastStatusTime := startTime + const statusInterval = 15 * time.Second var bytesProcessed int64 + for i, file := range regularFiles { if ctx.Err() != nil { return ctx.Err() } targetPath := filepath.Join(targetDir, file.Path.String()) + bytesVerified, err := v.verifyFile(ctx, repos, file, targetPath) if err != nil { log.Error("File verification failed", "path", file.Path, "error", err) + result.FilesFailed++ result.FailedFiles = append(result.FailedFiles, file.Path.String()) } else { result.FilesVerified++ result.BytesVerified += bytesVerified } + bytesProcessed += file.Size if time.Since(lastStatusTime) >= statusInterval { v.printVerifyProgress(i+1, len(regularFiles), bytesProcessed, totalBytes, startTime) + lastStatusTime = time.Now() } } @@ -821,13 +922,14 @@ func (v *Vaultik) printVerifyProgress(filesDone, totalFiles int, bytesDone, tota fileRate := float64(filesDone) / elapsed.Seconds() remainingBytes := totalBytes - bytesDone + var eta time.Duration if byteRate > 0 && remainingBytes > 0 { eta = time.Duration(float64(remainingBytes)/byteRate) * time.Second } if eta > 0 { - v.UI.Progress("Verify: %s/%s files (%s), %s/%s, %s, %.0f files/sec, verify elapsed: %s, verify ETA: %s (est remain %s).", + v.UI.Progressf("Verify: %s/%s files (%s), %s/%s, %s, %.0f files/sec, verify elapsed: %s, verify ETA: %s (est remain %s).", v.UI.Count(filesDone), v.UI.Count(totalFiles), v.UI.Percent(pct), @@ -838,9 +940,11 @@ func (v *Vaultik) printVerifyProgress(filesDone, totalFiles int, bytesDone, tota v.UI.Duration(elapsed), v.UI.Time(time.Now().Add(eta)), v.UI.Duration(eta)) + return } - v.UI.Progress("Verify: %s/%s files (%s), %s/%s, %s, %.0f files/sec, verify elapsed: %s.", + + v.UI.Progressf("Verify: %s/%s files (%s), %s/%s, %s, %.0f files/sec, verify elapsed: %s.", v.UI.Count(filesDone), v.UI.Count(totalFiles), v.UI.Percent(pct), @@ -873,6 +977,7 @@ func (v *Vaultik) verifyFile( // Verify each chunk var bytesVerified int64 + for _, fc := range fileChunks { // Get chunk size from database chunk, err := repos.Chunks.GetByHash(ctx, fc.ChunkHash.String()) @@ -882,10 +987,12 @@ func (v *Vaultik) verifyFile( // Read chunk data from file chunkData := make([]byte, chunk.Size) + n, err := io.ReadFull(f, chunkData) if err != nil { return bytesVerified, fmt.Errorf("reading chunk data: %w", err) } + if int64(n) != chunk.Size { return bytesVerified, fmt.Errorf("short read: expected %d bytes, got %d", chunk.Size, n) } @@ -904,5 +1011,6 @@ func (v *Vaultik) verifyFile( } log.Debug("File verified", "path", file.Path, "bytes", bytesVerified, "chunks", len(fileChunks)) + return bytesVerified, nil } diff --git a/internal/vaultik/restore_locality_test.go b/internal/vaultik/restore_locality_test.go index 4a29bc2..0e7c52d 100644 --- a/internal/vaultik/restore_locality_test.go +++ b/internal/vaultik/restore_locality_test.go @@ -6,6 +6,7 @@ import ( "crypto/rand" "fmt" "io" + "maps" "os" "path/filepath" "sort" @@ -55,6 +56,7 @@ func TestRestoreLocalityAndReadAt(t *testing.T) { fs := afero.NewOsFs() tempDir, err := os.MkdirTemp("", "vaultik-locality-") require.NoError(t, err) + defer func() { _ = os.RemoveAll(tempDir) }() dataDir := filepath.Join(tempDir, "source") @@ -86,8 +88,9 @@ func TestRestoreLocalityAndReadAt(t *testing.T) { path string data []byte } + sources := make([]*source, srcCount) - for i := 0; i < srcCount; i++ { + for i := range srcCount { s := &source{ path: fmt.Sprintf("src-%03d.bin", i+1), data: randomBytes(t, srcBytes), @@ -105,11 +108,14 @@ func TestRestoreLocalityAndReadAt(t *testing.T) { sourceBlob int // 0, 1, or 2 sourceIndex int // index into sources slice } + groupReps := []int{0, perBlob, 2 * perBlob} // 0, 5, 10 letters := []byte{'A', 'B', 'C'} + var copies []copyFile - for i := 0; i < 3; i++ { - for j := 0; j < blobsCount; j++ { + + for i := range 3 { + for j := range blobsCount { seq := i*blobsCount + j + 1 name := fmt.Sprintf("cp-%03d-%c.bin", seq, letters[j]) path := filepath.Join(dataDir, name) @@ -143,6 +149,7 @@ func TestRestoreLocalityAndReadAt(t *testing.T) { db, err := database.New(ctx, dbPath) require.NoError(t, err) + defer func() { _ = db.Close() }() repos := database.NewRepositories(db) @@ -187,6 +194,7 @@ func TestRestoreLocalityAndReadAt(t *testing.T) { // immediately before close) so we read PeakLen and call counters // from the same instance the production code used. var cacheRef *blobDiskCache + v := &Vaultik{ Config: cfg, Storage: counter, @@ -214,6 +222,7 @@ func TestRestoreLocalityAndReadAt(t *testing.T) { require.NoErrorf(t, err, "source missing after restore: %s", s.path) require.Truef(t, bytes.Equal(got, s.data), "byte mismatch for source %s", s.path) } + for _, c := range copies { restored := filepath.Join(restoreDir, c.path) got, err := afero.ReadFile(fs, restored) @@ -226,6 +235,7 @@ func TestRestoreLocalityAndReadAt(t *testing.T) { if !filterBlobKey(key) { continue } + assert.Equalf(t, 1, n, "blob %s fetched %d times, want exactly 1", key, n) } @@ -248,9 +258,11 @@ func TestRestoreLocalityAndReadAt(t *testing.T) { // chunker picks non-degenerate FastCDC boundaries. func randomBytes(t *testing.T, n int) []byte { t.Helper() + b := make([]byte, n) _, err := rand.Read(b) require.NoError(t, err) + return b } @@ -258,21 +270,27 @@ func randomBytes(t *testing.T, n int) []byte { // relative keys for every blob file present. func listBlobKeys(t *testing.T, storeDir string) []string { t.Helper() + var keys []string + root := filepath.Join(storeDir, "blobs") err := filepath.Walk(root, func(p string, info os.FileInfo, err error) error { if err != nil { return err } + if info.IsDir() { return nil } + rel, _ := filepath.Rel(storeDir, p) keys = append(keys, rel) + return nil }) require.NoError(t, err) sort.Strings(keys) + return keys } @@ -289,6 +307,7 @@ func filterBlobKey(key string) bool { // vaultik for access to unexported cache internals. type countingStorerInternal struct { storage.Storer + mu sync.Mutex counts map[string]int } @@ -301,15 +320,16 @@ func (c *countingStorerInternal) Get(ctx context.Context, key string) (io.ReadCl c.mu.Lock() c.counts[key]++ c.mu.Unlock() + return c.Storer.Get(ctx, key) } func (c *countingStorerInternal) snapshot() map[string]int { c.mu.Lock() defer c.mu.Unlock() + out := make(map[string]int, len(c.counts)) - for k, v := range c.counts { - out[k] = v - } + maps.Copy(out, c.counts) + return out } diff --git a/internal/vaultik/restore_plan.go b/internal/vaultik/restore_plan.go index 5b53680..c6cf5cb 100644 --- a/internal/vaultik/restore_plan.go +++ b/internal/vaultik/restore_plan.go @@ -53,26 +53,33 @@ func newRestorePlan( // No chunks to fetch — restore can run immediately. p.fileBlobs[f.ID] = nil p.ready = append(p.ready, f.ID) + continue } + fileChunks, err := repos.FileChunks.GetByFileID(ctx, f.ID) if err != nil { return nil, fmt.Errorf("planning %s: %w", f.Path, err) } + blobs := make(map[string]struct{}) + for _, fc := range fileChunks { bc, ok := chunkToBlobMap[fc.ChunkHash.String()] if !ok { return nil, fmt.Errorf("planning %s: chunk %s missing from blob map", f.Path, fc.ChunkHash.String()[:16]) } + hash, ok := blobIDToHash[bc.BlobID.String()] if !ok { return nil, fmt.Errorf("planning %s: blob id %s missing from id-to-hash map", f.Path, bc.BlobID) } + blobs[hash] = struct{}{} } + p.fileBlobs[f.ID] = blobs for hash := range blobs { set, ok := p.blobFiles[hash] @@ -80,12 +87,15 @@ func newRestorePlan( set = make(map[types.FileID]struct{}) p.blobFiles[hash] = set } + set[f.ID] = struct{}{} } + if len(blobs) == 0 { p.ready = append(p.ready, f.ID) } } + return p, nil } @@ -96,10 +106,12 @@ func (p *restorePlan) markBlobCached(blobHash string) { if _, already := p.cached[blobHash]; already { return } + p.cached[blobHash] = struct{}{} for fileID := range p.blobFiles[blobHash] { blobs := p.fileBlobs[fileID] delete(blobs, blobHash) + if len(blobs) == 0 { p.ready = append(p.ready, fileID) } @@ -112,8 +124,10 @@ func (p *restorePlan) popReady() (types.FileID, bool) { if len(p.ready) == 0 { return types.FileID{}, false } + id := p.ready[0] p.ready = p.ready[1:] + return id, true } @@ -123,17 +137,20 @@ func (p *restorePlan) finishFile(fileID types.FileID) { for hash := range p.fileBlobs[fileID] { if set, ok := p.blobFiles[hash]; ok { delete(set, fileID) + if len(set) == 0 { delete(p.blobFiles, hash) } } } + delete(p.fileBlobs, fileID) // Also scrub the file from any blobFiles entries where it might // still appear even after its uncached-blob set was emptied. for hash, set := range p.blobFiles { if _, ok := set[fileID]; ok { delete(set, fileID) + if len(set) == 0 { delete(p.blobFiles, hash) } @@ -150,8 +167,11 @@ func (p *restorePlan) finishFile(fileID types.FileID) { // The zero FileID return means nothing is pending. func (p *restorePlan) pickNextDownload() types.FileID { var best types.FileID + bestCount := math.MaxInt + var bestID string + for id, blobs := range p.fileBlobs { n := len(blobs) if n == 0 { @@ -159,6 +179,7 @@ func (p *restorePlan) pickNextDownload() types.FileID { // popReady; ignore here just in case. continue } + idStr := id.String() if n < bestCount || (n == bestCount && (best.IsZero() || idStr < bestID)) { best = id @@ -166,16 +187,19 @@ func (p *restorePlan) pickNextDownload() types.FileID { bestID = idStr } } + return best } // blobsNeeded returns the uncached blob hashes for fileID in any order. func (p *restorePlan) blobsNeeded(fileID types.FileID) []string { blobs := p.fileBlobs[fileID] + out := make([]string, 0, len(blobs)) for h := range blobs { out = append(out, h) } + return out } diff --git a/internal/vaultik/restore_sweeper.go b/internal/vaultik/restore_sweeper.go index 2594844..f3a47f0 100644 --- a/internal/vaultik/restore_sweeper.go +++ b/internal/vaultik/restore_sweeper.go @@ -42,6 +42,7 @@ func newRestoreSweeper(ctx context.Context, repos *database.Repositories, cache if threshold <= 0 { threshold = 1 } + return &restoreSweeper{ ctx: ctx, repos: repos, @@ -65,6 +66,7 @@ func (s *restoreSweeper) chunkRestored(n int64) { if s.bytesAccum < s.threshold { return } + s.bytesAccum = 0 s.sweep() } @@ -77,8 +79,10 @@ func (s *restoreSweeper) sweep() { needed, err := s.blobStillNeeded(blobHash) if err != nil { log.Debug("sweeper referencing-files query failed", "blob_hash", blobHash[:16], "error", err) + continue } + if !needed { s.cache.Delete(blobHash) } @@ -104,15 +108,20 @@ func (s *restoreSweeper) blobStillNeeded(blobHash string) (bool, error) { for rows.Next() { var fileID string - if err := rows.Scan(&fileID); err != nil { + + err := rows.Scan(&fileID) + if err != nil { return true, fmt.Errorf("scanning file_id: %w", err) } + if _, ok := s.restored[fileID]; !ok { return true, nil } } + if err := rows.Err(); err != nil { return true, err } + return false, nil } diff --git a/internal/vaultik/restore_sweeper_integration_test.go b/internal/vaultik/restore_sweeper_integration_test.go index 044f3a2..d5b21e0 100644 --- a/internal/vaultik/restore_sweeper_integration_test.go +++ b/internal/vaultik/restore_sweeper_integration_test.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "io" + "maps" "math/rand" "os" "path/filepath" @@ -45,6 +46,7 @@ func TestRestoreSweeperEvictsBlobs(t *testing.T) { fs := afero.NewOsFs() tempDir, err := os.MkdirTemp("", "vaultik-sweeper-") require.NoError(t, err) + defer func() { _ = os.RemoveAll(tempDir) }() dataDir := filepath.Join(tempDir, "source") @@ -63,19 +65,22 @@ func TestRestoreSweeperEvictsBlobs(t *testing.T) { duplicateFiles = 10 fileSize = 1 * 1024 * 1024 ) + rng := rand.New(rand.NewSource(42)) type sourceFile struct { path string data []byte } + uniques := make([]sourceFile, 0, uniqueFiles) expected := make(map[string][]byte, uniqueFiles+duplicateFiles) - for i := 0; i < uniqueFiles; i++ { + for i := range uniqueFiles { data := make([]byte, fileSize) _, err := rng.Read(data) require.NoError(t, err) + path := filepath.Join(dataDir, fmt.Sprintf("unique-%02d.bin", i)) require.NoError(t, afero.WriteFile(fs, path, data, 0o644)) uniques = append(uniques, sourceFile{path: path, data: data}) @@ -112,6 +117,7 @@ func TestRestoreSweeperEvictsBlobs(t *testing.T) { db, err := database.New(ctx, dbPath) require.NoError(t, err) + defer func() { _ = db.Close() }() repos := database.NewRepositories(db) @@ -184,14 +190,18 @@ func TestRestoreSweeperEvictsBlobs(t *testing.T) { // sweeper evicted a still-needed blob; 0 means the cache silently // stopped being consulted. blobDownloads := 0 + for key, count := range counter.snapshot() { if !strings.HasPrefix(key, "blobs/") { continue } + assert.Equalf(t, 1, count, "blob %s should have been downloaded exactly once during restore, got %d", key, count) + blobDownloads++ } + assert.Equal(t, blobCount, blobDownloads, "every blob on disk should have been fetched exactly once during restore") t.Logf("restore downloaded %d blobs, each exactly once", blobDownloads) @@ -202,6 +212,7 @@ func TestRestoreSweeperEvictsBlobs(t *testing.T) { // re-downloading blobs that are evicted while still needed. type countingStorer struct { storage.Storer + mu sync.Mutex counts map[string]int } @@ -214,16 +225,17 @@ func (c *countingStorer) Get(ctx context.Context, key string) (io.ReadCloser, er c.mu.Lock() c.counts[key]++ c.mu.Unlock() + return c.Storer.Get(ctx, key) } func (c *countingStorer) snapshot() map[string]int { c.mu.Lock() defer c.mu.Unlock() + out := make(map[string]int, len(c.counts)) - for k, v := range c.counts { - out[k] = v - } + maps.Copy(out, c.counts) + return out } @@ -232,17 +244,21 @@ func (c *countingStorer) snapshot() map[string]int { // expected number of restore-time downloads. func countBlobsOnDisk(t *testing.T, storeDir string) int { t.Helper() + count := 0 root := filepath.Join(storeDir, "blobs") err := filepath.Walk(root, func(_ string, info os.FileInfo, err error) error { if err != nil { return err } + if !info.IsDir() { count++ } + return nil }) require.NoError(t, err) + return count } diff --git a/internal/vaultik/snapshot.go b/internal/vaultik/snapshot.go index ba5c017..3f9b040 100644 --- a/internal/vaultik/snapshot.go +++ b/internal/vaultik/snapshot.go @@ -2,6 +2,7 @@ package vaultik import ( "encoding/json" + "errors" "fmt" "os" "path/filepath" @@ -36,7 +37,8 @@ func (v *Vaultik) CreateSnapshot(opts *SnapshotCreateOptions) error { "index_path", v.Config.IndexPath, ) - if err := v.EnsureStorageBinding(); err != nil { + err := v.EnsureStorageBinding() + if err != nil { return err } @@ -72,23 +74,25 @@ func (v *Vaultik) CreateSnapshot(opts *SnapshotCreateOptions) error { } if len(snapshotNames) == 0 { - return fmt.Errorf("no snapshots configured") + return errors.New("no snapshots configured") } // Process each named snapshot for snapIdx, snapName := range snapshotNames { - if err := v.createNamedSnapshot(opts, hostname, snapName, snapIdx+1, len(snapshotNames)); err != nil { + err := v.createNamedSnapshot(opts, hostname, snapName, snapIdx+1, len(snapshotNames)) + if err != nil { return err } } // Print overall summary if multiple snapshots if len(snapshotNames) > 1 { - v.UI.Complete("All %d snapshots completed in %s.", len(snapshotNames), v.UI.Duration(time.Since(overallStartTime))) + v.UI.Completef("All %d snapshots completed in %s.", len(snapshotNames), v.UI.Duration(time.Since(overallStartTime))) } if opts.Prune { - if err := v.runPostBackupPrune(snapshotNames, opts.KeepNewerThan); err != nil { + err := v.runPostBackupPrune(snapshotNames, opts.KeepNewerThan) + if err != nil { return fmt.Errorf("post-backup prune: %w", err) } } @@ -99,9 +103,9 @@ func (v *Vaultik) CreateSnapshot(opts *SnapshotCreateOptions) error { // emit the summary via UI.Warning so cron actually delivers // something for the user to look at. if v.UI.WarningCount() > 0 { - v.UI.Warning("Finished with %d warning(s) — review the output above.", v.UI.WarningCount()) + v.UI.Warningf("Finished with %d warning(s) — review the output above.", v.UI.WarningCount()) } else { - v.UI.Complete("Finished successfully.") + v.UI.Completef("Finished successfully.") } return nil @@ -113,7 +117,7 @@ func (v *Vaultik) CreateSnapshot(opts *SnapshotCreateOptions) error { // snapshot of each name is kept. func (v *Vaultik) runPostBackupPrune(snapshotNames []string, keepNewerThan string) error { log.Info("Running post-backup prune", "snapshots", snapshotNames, "keep_newer_than", keepNewerThan) - v.UI.Begin("Running post-backup prune.") + v.UI.Beginf("Running post-backup prune.") purgeOpts := &SnapshotPurgeOptions{ Force: true, @@ -127,11 +131,13 @@ func (v *Vaultik) runPostBackupPrune(snapshotNames []string, keepNewerThan strin purgeOpts.KeepLatest = true } - if err := v.PurgeSnapshotsWithOptions(purgeOpts); err != nil { + err := v.PurgeSnapshotsWithOptions(purgeOpts) + if err != nil { return fmt.Errorf("purging old snapshots: %w", err) } - if err := v.PruneBlobs(&PruneOptions{Force: true}); err != nil { + err = v.PruneBlobs(&PruneOptions{Force: true}) + if err != nil { return fmt.Errorf("pruning orphaned blobs: %w", err) } @@ -158,7 +164,7 @@ func (v *Vaultik) createNamedSnapshot(opts *SnapshotCreateOptions, hostname, sna snapshotStartTime := time.Now() if total > 1 { - v.UI.Info("Snapshot %d/%d: %s.", idx, total, snapName) + v.UI.Infof("Snapshot %d/%d: %s.", idx, total, snapName) } resolvedDirs, err := v.resolveSnapshotPaths(snapName) @@ -178,8 +184,9 @@ func (v *Vaultik) createNamedSnapshot(opts *SnapshotCreateOptions, hostname, sna if err != nil { return fmt.Errorf("creating snapshot: %w", err) } + log.Info("Beginning snapshot", "snapshot_id", snapshotID, "name", snapName) - v.UI.Begin("Creating snapshot %s.", v.UI.Snapshot(snapshotID)) + v.UI.Beginf("Creating snapshot %s.", v.UI.Snapshot(snapshotID)) stats, err := v.scanAllDirectories(scanner, resolvedDirs, snapshotID) if err != nil { @@ -201,6 +208,7 @@ func (v *Vaultik) createNamedSnapshot(opts *SnapshotCreateOptions, hostname, sna "duration", time.Since(snapshotStartTime)) v.printSnapshotSummary(snapshotID, snapshotStartTime, stats) + return nil } @@ -238,12 +246,14 @@ func (v *Vaultik) scanAllDirectories(scanner *snapshot.Scanner, resolvedDirs []s select { case <-v.ctx.Done(): log.Info("Snapshot creation cancelled") + return nil, v.ctx.Err() default: } log.Info("Scanning directory", "path", dir) - v.UI.Begin("Enumerating snapshot source files in %s (%d of %d).", v.UI.Path(dir), i+1, len(resolvedDirs)) + v.UI.Beginf("Enumerating snapshot source files in %s (%d of %d).", v.UI.Path(dir), i+1, len(resolvedDirs)) + result, err := scanner.Scan(v.ctx, dir, snapshotID) if err != nil { return nil, fmt.Errorf("failed to scan %s: %w", dir, err) @@ -297,15 +307,18 @@ func (v *Vaultik) finalizeSnapshotMetadata(snapshotID string, stats *snapshotSta UploadDurationMs: stats.uploadDuration.Milliseconds(), } - if err := v.SnapshotManager.UpdateSnapshotStatsExtended(v.ctx, snapshotID, extStats); err != nil { + err := v.SnapshotManager.UpdateSnapshotStatsExtended(v.ctx, snapshotID, extStats) + if err != nil { return fmt.Errorf("updating snapshot stats: %w", err) } - if err := v.SnapshotManager.CompleteSnapshot(v.ctx, snapshotID); err != nil { + err = v.SnapshotManager.CompleteSnapshot(v.ctx, snapshotID) + if err != nil { return fmt.Errorf("completing snapshot: %w", err) } - if err := v.SnapshotManager.ExportSnapshotMetadata(v.ctx, v.Config.IndexPath, snapshotID); err != nil { + err = v.SnapshotManager.ExportSnapshotMetadata(v.ctx, v.Config.IndexPath, snapshotID) + if err != nil { return fmt.Errorf("exporting snapshot metadata: %w", err) } @@ -318,6 +331,7 @@ func (v *Vaultik) uploadSpeed(bytesUploaded int64, duration time.Duration) strin if bytesUploaded <= 0 || duration <= 0 { return v.UI.Speed(0) } + return v.UI.Speed(float64(bytesUploaded) / duration.Seconds()) } @@ -337,7 +351,8 @@ func (v *Vaultik) printSnapshotSummary(snapshotID string, startTime time.Time, s compressionRatio = 1.0 } - v.UI.Complete("Created snapshot %s.", v.UI.Snapshot(snapshotID)) + v.UI.Completef("Created snapshot %s.", v.UI.Snapshot(snapshotID)) + filesMsg := fmt.Sprintf("Files: %s examined, %s backed up, %s unchanged", v.UI.Count(stats.totalFiles), v.UI.Count(totalFilesChanged), @@ -345,7 +360,8 @@ func (v *Vaultik) printSnapshotSummary(snapshotID string, startTime time.Time, s if stats.totalFilesDeleted > 0 { filesMsg += fmt.Sprintf(", %s deleted", v.UI.Count(stats.totalFilesDeleted)) } - v.UI.Detail("%s.", filesMsg) + + v.UI.Detailf("%s.", filesMsg) dataMsg := fmt.Sprintf("Data: %s total (%s backed up)", v.UI.Size(totalBytesAll), @@ -353,20 +369,22 @@ func (v *Vaultik) printSnapshotSummary(snapshotID string, startTime time.Time, s if stats.totalBytesDeleted > 0 { dataMsg += fmt.Sprintf(", %s deleted", v.UI.Size(stats.totalBytesDeleted)) } - v.UI.Detail("%s.", dataMsg) + + v.UI.Detailf("%s.", dataMsg) if stats.totalBlobsUploaded > 0 { - v.UI.Detail("Storage: %s compressed from %s (%.2fx ratio).", + v.UI.Detailf("Storage: %s compressed from %s (%.2fx ratio).", v.UI.Size(totalBlobSizeCompressed), v.UI.Size(totalBlobSizeUncompressed), compressionRatio) - v.UI.Detail("Upload: %d blobs, %s in %s (%s).", + v.UI.Detailf("Upload: %d blobs, %s in %s (%s).", stats.totalBlobsUploaded, v.UI.Size(stats.totalBytesUploaded), v.UI.Duration(stats.uploadDuration), v.uploadSpeed(stats.totalBytesUploaded, stats.uploadDuration)) } - v.UI.Detail("Snapshot create duration: %s.", v.UI.Duration(snapshotDuration)) + + v.UI.Detailf("Snapshot create duration: %s.", v.UI.Duration(snapshotDuration)) } // getSnapshotBlobSizes returns total compressed and uncompressed blob sizes for a snapshot @@ -375,12 +393,14 @@ func (v *Vaultik) getSnapshotBlobSizes(snapshotID string) (compressed int64, unc if err != nil { return 0, 0 } + for _, hash := range blobHashes { if blob, err := v.Repositories.Blobs.GetByHash(v.ctx, hash); err == nil && blob != nil { compressed += blob.CompressedSize uncompressed += blob.UncompressedSize } } + return compressed, uncompressed } @@ -418,6 +438,7 @@ func (v *Vaultik) ListSnapshots(jsonOutput bool) error { if ls.CompletedAt == nil { continue } + snapshots = append(snapshots, v.snapshotInfoFromLocal(ls)) } @@ -428,6 +449,7 @@ func (v *Vaultik) ListSnapshots(jsonOutput bool) error { if jsonOutput { encoder := json.NewEncoder(v.Stdout) encoder.SetIndent("", " ") + return encoder.Encode(snapshots) } @@ -441,7 +463,8 @@ func (v *Vaultik) ListSnapshots(jsonOutput bool) error { remoteKeys, err := v.listAllRemoteSnapshotKeys() if err != nil { - v.UI.Warning("Could not list backup destination store: %v.", err) + v.UI.Warningf("Could not list backup destination store: %v.", err) + return nil } @@ -450,20 +473,25 @@ func (v *Vaultik) ListSnapshots(jsonOutput bool) error { if ls.CompletedAt == nil { continue } + localKeys[snapshot.RemoteSnapshotKey(ls.ID.String())] = ls.ID.String() } + remoteSet := make(map[string]bool, len(remoteKeys)) for _, k := range remoteKeys { remoteSet[k] = true } var localOnly []string + for key, humanID := range localKeys { if !remoteSet[key] { localOnly = append(localOnly, humanID) } } + var remoteOnlyCount int + for key := range remoteSet { if _, ok := localKeys[key]; !ok { remoteOnlyCount++ @@ -471,14 +499,17 @@ func (v *Vaultik) ListSnapshots(jsonOutput bool) error { } if len(localOnly) > 0 { - v.UI.Warning("%d local snapshot record(s) not found in backup destination store:", len(localOnly)) + v.UI.Warningf("%d local snapshot record(s) not found in backup destination store:", len(localOnly)) + for _, id := range localOnly { - v.UI.Info("%s", v.UI.Snapshot(id)) + v.UI.Infof("%s", v.UI.Snapshot(id)) } - v.UI.Info("Run 'vaultik snapshot cleanup' to remove stale local records.") + + v.UI.Infof("Run 'vaultik snapshot cleanup' to remove stale local records.") } + if remoteOnlyCount > 0 { - v.UI.Notice("NOTE: %d remote snapshot(s) found in backup destination store but not in local database.", remoteOnlyCount) + v.UI.Noticef("NOTE: %d remote snapshot(s) found in backup destination store but not in local database.", remoteOnlyCount) } return nil @@ -493,6 +524,7 @@ func (v *Vaultik) snapshotInfoFromLocal(ls *database.Snapshot) SnapshotInfo { totalSize, err := v.Repositories.Snapshots.GetSnapshotTotalCompressedSize(v.ctx, idStr) if err != nil { log.Warn("Failed to get total compressed size", "id", idStr, "error", err) + totalSize = ls.BlobSize } @@ -523,19 +555,24 @@ func (v *Vaultik) printSnapshotTable(snapshots []SnapshotInfo) error { if _, err := fmt.Fprintln(w, "CONFIGURED SNAPSHOTS:"); err != nil { return err } + if _, err := fmt.Fprintln(w, "NAME\tPATHS"); err != nil { return err } + if _, err := fmt.Fprintln(w, "────\t─────"); err != nil { return err } + for _, name := range v.Config.SnapshotNames() { snap := v.Config.Snapshots[name] + paths := strings.Join(snap.Paths, ", ") if _, err := fmt.Fprintf(w, "%s\t%s\n", name, paths); err != nil { return err } } + if _, err := fmt.Fprintln(w); err != nil { return err } @@ -543,9 +580,11 @@ func (v *Vaultik) printSnapshotTable(snapshots []SnapshotInfo) error { if _, err := fmt.Fprintln(w, "REMOTE SNAPSHOTS:"); err != nil { return err } + if _, err := fmt.Fprintln(w, "SNAPSHOT ID\tTIMESTAMP\tCOMPRESSED SIZE\tUNCOMPRESSED SIZE\tNEW CHUNK SIZE"); err != nil { return err } + if _, err := fmt.Fprintln(w, "───────────\t─────────\t───────────────\t─────────────────\t──────────────"); err != nil { return err } @@ -554,10 +593,12 @@ func (v *Vaultik) printSnapshotTable(snapshots []SnapshotInfo) error { for _, snap := range snapshots { uncompressed := remoteOnlyCell newChunks := remoteOnlyCell + if snap.LocallyTracked { uncompressed = formatBytes(snap.UncompressedSize) newChunks = formatBytes(snap.NewChunkSize) } + if _, err := fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n", snap.ID, snap.Timestamp.Format("2006-01-02 15:04:05"), @@ -611,11 +652,13 @@ func (v *Vaultik) PurgeSnapshotsWithOptions(opts *SnapshotPurgeOptions) error { if s.CompletedAt == nil { continue } + if len(nameFilter) > 0 { if _, ok := nameFilter[parseSnapshotName(s.ID.String())]; !ok { continue } } + snapshots = append(snapshots, SnapshotInfo{ ID: s.ID, Timestamp: s.StartedAt, @@ -634,12 +677,15 @@ func (v *Vaultik) PurgeSnapshotsWithOptions(opts *SnapshotPurgeOptions) error { // Keep the latest snapshot per snapshot name. Snapshots are sorted // newest-first, so the first occurrence of each name is kept. seen := make(map[string]bool) + for _, snap := range snapshots { name := parseSnapshotName(snap.ID.String()) if seen[name] { toDelete = append(toDelete, snap) + continue } + seen[name] = true } } else if opts.OlderThan != "" { @@ -660,6 +706,7 @@ func (v *Vaultik) PurgeSnapshotsWithOptions(opts *SnapshotPurgeOptions) error { if !opts.Quiet { v.printlnStdout("No snapshots to delete") } + return nil } @@ -669,9 +716,10 @@ func (v *Vaultik) PurgeSnapshotsWithOptions(opts *SnapshotPurgeOptions) error { // confirmAndExecutePurge shows deletion candidates, confirms with user, and deletes snapshots func (v *Vaultik) confirmAndExecutePurge(toDelete []SnapshotInfo, force, quiet bool) error { if !quiet { - v.printfStdout("The following snapshots will be deleted:\n\n") + v.stdoutf("The following snapshots will be deleted:\n\n") + for _, snap := range toDelete { - v.printfStdout(" %s (%s, %s)\n", + v.stdoutf(" %s (%s, %s)\n", snap.ID, snap.Timestamp.Format("2006-01-02 15:04:05"), formatBytes(snap.CompressedSize)) @@ -680,29 +728,37 @@ func (v *Vaultik) confirmAndExecutePurge(toDelete []SnapshotInfo, force, quiet b // Confirm unless --force is used if !force { - v.printfStdout("\nDelete %d snapshot(s)? [y/N] ", len(toDelete)) + v.stdoutf("\nDelete %d snapshot(s)? [y/N] ", len(toDelete)) + var confirm string if _, err := v.scanStdin(&confirm); err != nil { // Treat EOF or error as "no" v.printlnStdout("Cancelled") + return nil } + if strings.ToLower(confirm) != "y" { v.printlnStdout("Cancelled") + return nil } } else if !quiet { - v.printfStdout("\nDeleting %d snapshot(s) (--force specified)\n", len(toDelete)) + v.stdoutf("\nDeleting %d snapshot(s) (--force specified)\n", len(toDelete)) } // Delete snapshots (both local and remote) for _, snap := range toDelete { snapshotID := snap.ID.String() log.Info("Deleting snapshot", "id", snapshotID) - if err := v.deleteSnapshotFromLocalDB(snapshotID); err != nil { + + err := v.deleteSnapshotFromLocalDB(snapshotID) + if err != nil { log.Error("Failed to delete from local database", "snapshot_id", snapshotID, "error", err) } - if err := v.deleteRemoteSnapshotByKey(snapshot.RemoteSnapshotKey(snapshotID)); err != nil { + + err = v.deleteRemoteSnapshotByKey(snapshot.RemoteSnapshotKey(snapshotID)) + if err != nil { return fmt.Errorf("deleting snapshot %s from remote: %w", snapshotID, err) } } @@ -711,13 +767,14 @@ func (v *Vaultik) confirmAndExecutePurge(toDelete []SnapshotInfo, force, quiet b // separate command after a purge. Guarded against nil for tests // that don't wire up a SnapshotManager. if v.SnapshotManager != nil { - if err := v.SnapshotManager.CleanupOrphanedData(v.ctx); err != nil { + err := v.SnapshotManager.CleanupOrphanedData(v.ctx) + if err != nil { log.Warn("Failed to clean up orphaned local data after purge", "error", err) } } if !quiet { - v.printfStdout("Deleted %d snapshot(s)\n", len(toDelete)) + v.stdoutf("Deleted %d snapshot(s)\n", len(toDelete)) v.printlnStdout("\nNote: Run 'vaultik prune' to clean up unreferenced remote blobs.") } @@ -730,6 +787,7 @@ func (v *Vaultik) VerifySnapshot(snapshotID string, deep bool) error { if deep { return v.RunDeepVerify(snapshotID, opts) } + return v.VerifySnapshotWithOptions(snapshotID, opts) } @@ -740,6 +798,7 @@ func (v *Vaultik) VerifySnapshotWithOptions(snapshotID string, opts *VerifyOptio if opts.Deep { return v.RunDeepVerify(snapshotID, opts) } + result := &VerifyResult{ SnapshotID: snapshotID, Mode: "shallow", @@ -754,8 +813,10 @@ func (v *Vaultik) VerifySnapshotWithOptions(snapshotID string, opts *VerifyOptio if opts.JSON { result.Status = "failed" result.ErrorMessage = fmt.Sprintf("downloading manifest: %v", err) + return v.outputVerifyJSON(result) } + return fmt.Errorf("downloading manifest: %w", err) } @@ -763,18 +824,20 @@ func (v *Vaultik) VerifySnapshotWithOptions(snapshotID string, opts *VerifyOptio result.TotalSize = manifest.TotalCompressedSize if !opts.JSON { - v.printfStdout("Snapshot information:\n") - v.printfStdout(" Blob count: %d\n", manifest.BlobCount) - v.printfStdout(" Total size: %s\n", humanize.Bytes(uint64(manifest.TotalCompressedSize))) + v.stdoutf("Snapshot information:\n") + v.stdoutf(" Blob count: %d\n", manifest.BlobCount) + v.stdoutf(" Total size: %s\n", humanize.Bytes(uint64(manifest.TotalCompressedSize))) + if manifest.Timestamp != "" { if t, err := time.Parse(time.RFC3339, manifest.Timestamp); err == nil { - v.printfStdout(" Created: %s\n", t.Format("2006-01-02 15:04:05 MST")) + v.stdoutf(" Created: %s\n", t.Format("2006-01-02 15:04:05 MST")) } } + v.printlnStdout() // Check each blob exists - v.printfStdout("Checking blob existence...\n") + v.stdoutf("Checking blob existence...\n") } result.Verified, result.Missing, result.MissingSize = v.verifyManifestBlobsExist(manifest, opts) @@ -791,10 +854,12 @@ func (v *Vaultik) printVerifyHeader(snapshotID string, opts *VerifyOptions) { } if !opts.JSON { - v.printfStdout("Verifying snapshot %s\n", snapshotID) + v.stdoutf("Verifying snapshot %s\n", snapshotID) + if !snapshotTime.IsZero() { - v.printfStdout("Snapshot time: %s\n", snapshotTime.Format("2006-01-02 15:04:05 MST")) + v.stdoutf("Snapshot time: %s\n", snapshotTime.Format("2006-01-02 15:04:05 MST")) } + v.printlnStdout() } } @@ -808,14 +873,16 @@ func (v *Vaultik) verifyManifestBlobsExist(manifest *snapshot.Manifest, opts *Ve _, err := v.Storage.Stat(v.ctx, blobPath) if err != nil { if !opts.JSON { - v.printfStdout(" Missing: %s (%s)\n", blob.Hash, humanize.Bytes(uint64(blob.CompressedSize))) + v.stdoutf(" Missing: %s (%s)\n", blob.Hash, humanize.Bytes(uint64(blob.CompressedSize))) } + missing++ missingSize += blob.CompressedSize } else { verified++ } } + return verified, missing, missingSize } @@ -828,23 +895,29 @@ func (v *Vaultik) formatVerifyResult(result *VerifyResult, manifest *snapshot.Ma } else { result.Status = "ok" } + return v.outputVerifyJSON(result) } - v.printfStdout("\nVerification complete:\n") - v.printfStdout(" Verified: %d blobs (%s)\n", result.Verified, + v.stdoutf("\nVerification complete:\n") + v.stdoutf(" Verified: %d blobs (%s)\n", result.Verified, humanize.Bytes(uint64(manifest.TotalCompressedSize-result.MissingSize))) + if result.Missing > 0 { - v.printfStdout(" Missing: %d blobs (%s)\n", result.Missing, humanize.Bytes(uint64(result.MissingSize))) + v.stdoutf(" Missing: %d blobs (%s)\n", result.Missing, humanize.Bytes(uint64(result.MissingSize))) } else { - v.printfStdout(" Missing: 0 blobs\n") + v.stdoutf(" Missing: 0 blobs\n") } - v.printfStdout(" Status: ") + + v.stdoutf(" Status: ") + if result.Missing > 0 { - v.printfStdout("FAILED - %d blobs are missing\n", result.Missing) + v.stdoutf("FAILED - %d blobs are missing\n", result.Missing) + return fmt.Errorf("%d blobs are missing", result.Missing) } - v.printfStdout("OK - All blobs verified\n") + + v.stdoutf("OK - All blobs verified\n") return nil } @@ -853,12 +926,16 @@ func (v *Vaultik) formatVerifyResult(result *VerifyResult, manifest *snapshot.Ma func (v *Vaultik) outputVerifyJSON(result *VerifyResult) error { encoder := json.NewEncoder(v.Stdout) encoder.SetIndent("", " ") - if err := encoder.Encode(result); err != nil { + + err := encoder.Encode(result) + if err != nil { return fmt.Errorf("encoding JSON: %w", err) } + if result.Status == "failed" { return fmt.Errorf("verification failed: %s", result.ErrorMessage) } + return nil } @@ -871,10 +948,12 @@ func (v *Vaultik) CleanupLocalSnapshots() error { if err := v.EnsureStorageBinding(); err != nil { return err } + remoteKeys, err := v.listAllRemoteSnapshotKeys() if err != nil { return err } + remoteSet := make(map[string]bool, len(remoteKeys)) for _, k := range remoteKeys { remoteSet[k] = true @@ -886,14 +965,19 @@ func (v *Vaultik) CleanupLocalSnapshots() error { } var removed int + for _, snap := range localSnapshots { id := snap.ID.String() if !remoteSet[snapshot.RemoteSnapshotKey(id)] { - v.printfStdout("Removing stale local record: %s\n", id) - if err := v.deleteSnapshotFromLocalDB(id); err != nil { + v.stdoutf("Removing stale local record: %s\n", id) + + err := v.deleteSnapshotFromLocalDB(id) + if err != nil { log.Error("Failed to delete local snapshot", "snapshot_id", id, "error", err) + continue } + removed++ } } @@ -901,8 +985,9 @@ func (v *Vaultik) CleanupLocalSnapshots() error { if removed == 0 { v.printlnStdout("No stale local snapshots found.") } else { - v.printfStdout("Removed %d stale local snapshot record(s).\n", removed) + v.stdoutf("Removed %d stale local snapshot record(s).\n", removed) } + return nil } @@ -948,6 +1033,7 @@ func (v *Vaultik) syncWithRemote() error { if strings.HasPrefix(parts[1], ".") { continue } + remoteSnapshots[parts[1]] = true } } @@ -962,11 +1048,14 @@ func (v *Vaultik) syncWithRemote() error { // Remove local snapshots that don't exist remotely removedCount := 0 + for _, snapshot := range localSnapshots { snapshotIDStr := snapshot.ID.String() if !remoteSnapshots[snapshotIDStr] { log.Info("Removing local snapshot not found in remote", "snapshot_id", snapshot.ID) - if err := v.deleteSnapshotFromLocalDB(snapshotIDStr); err != nil { + + err := v.deleteSnapshotFromLocalDB(snapshotIDStr) + if err != nil { log.Error("Failed to delete local snapshot", "snapshot_id", snapshot.ID, "error", err) } else { removedCount++ @@ -1015,59 +1104,74 @@ func (v *Vaultik) RemoveSnapshot(snapshotID string, opts *RemoveOptions) (*Remov SnapshotID: snapshotID, } - if err := v.EnsureStorageBinding(); err != nil { + err := v.EnsureStorageBinding() + if err != nil { return result, err } if opts.DryRun { result.DryRun = true + if !opts.JSON { - v.printfStdout("Would remove snapshot: %s\n", snapshotID) + v.stdoutf("Would remove snapshot: %s\n", snapshotID) + if !opts.LocalOnly { v.printlnStdout("Would also remove snapshot metadata from remote storage (blobs untouched)") } + v.printlnStdout("[Dry run - no changes made]") } + if opts.JSON { return result, v.outputRemoveJSON(result) } + return result, nil } if !opts.Force && !opts.JSON { if opts.LocalOnly { - v.printfStdout("Remove snapshot '%s' from local database (remote untouched)? [y/N] ", snapshotID) + v.stdoutf("Remove snapshot '%s' from local database (remote untouched)? [y/N] ", snapshotID) } else { - v.printfStdout("Remove snapshot '%s' from local database AND its metadata from remote storage? [y/N] ", snapshotID) + v.stdoutf("Remove snapshot '%s' from local database AND its metadata from remote storage? [y/N] ", snapshotID) } + var confirm string if _, err := v.scanStdin(&confirm); err != nil { v.printlnStdout("Cancelled") + return result, nil } + if strings.ToLower(confirm) != "y" { v.printlnStdout("Cancelled") + return result, nil } } log.Info("Removing snapshot from local database", "snapshot_id", snapshotID) - if err := v.deleteSnapshotFromLocalDB(snapshotID); err != nil { + err = v.deleteSnapshotFromLocalDB(snapshotID) + if err != nil { return result, fmt.Errorf("removing from local database: %w", err) } if !opts.LocalOnly { log.Info("Removing snapshot metadata from remote storage", "snapshot_id", snapshotID) + remoteKey := snapshot.RemoteSnapshotKey(snapshotID) - if err := v.deleteRemoteSnapshotByKey(remoteKey); err != nil { + + err := v.deleteRemoteSnapshotByKey(remoteKey) + if err != nil { // Warn-and-proceed: the local-DB removal has already // happened, so let the user know the remote half didn't // finish and they can retry with `vaultik prune` once the // destination store is reachable. log.Warn("Could not remove snapshot metadata from remote storage", "error", err) + if v.UI != nil { - v.UI.Warning("Could not remove snapshot metadata from remote: %v. Run '%s' once the remote is reachable to finish cleanup.", err, pruneCommandHint) + v.UI.Warningf("Could not remove snapshot metadata from remote: %v. Run '%s' once the remote is reachable to finish cleanup.", err, pruneCommandHint) } } else { result.RemoteRemoved = true @@ -1075,7 +1179,8 @@ func (v *Vaultik) RemoveSnapshot(snapshotID string, opts *RemoveOptions) (*Remov } if v.SnapshotManager != nil { - if err := v.SnapshotManager.CleanupOrphanedData(v.ctx); err != nil { + err := v.SnapshotManager.CleanupOrphanedData(v.ctx) + if err != nil { log.Warn("Failed to clean up orphaned local data after removal", "error", err) } } @@ -1084,10 +1189,11 @@ func (v *Vaultik) RemoveSnapshot(snapshotID string, opts *RemoveOptions) (*Remov return result, v.outputRemoveJSON(result) } - v.printfStdout("Removed snapshot '%s' from local database\n", snapshotID) + v.stdoutf("Removed snapshot '%s' from local database\n", snapshotID) + if !opts.LocalOnly && result.RemoteRemoved { v.printlnStdout("Removed snapshot metadata from remote storage") - v.printfStdout("\nNote: The removed snapshot's blobs remain on the remote. Run '%s' to delete any blobs no longer referenced by any remaining remote snapshot.\n", pruneCommandHint) + v.stdoutf("\nNote: The removed snapshot's blobs remain on the remote. Run '%s' to delete any blobs no longer referenced by any remaining remote snapshot.\n", pruneCommandHint) } return result, nil @@ -1102,6 +1208,7 @@ func (v *Vaultik) RemoveAllSnapshots(opts *RemoveOptions) (*RemoveResult, error) if err := v.EnsureStorageBinding(); err != nil { return nil, err } + localSnaps, err := v.localSnapshotIDs() if err != nil { return nil, fmt.Errorf("listing local snapshots: %w", err) @@ -1114,7 +1221,7 @@ func (v *Vaultik) RemoveAllSnapshots(opts *RemoveOptions) (*RemoveResult, error) remoteKeys, remoteErr := v.listAllRemoteSnapshotKeys() if remoteErr != nil { log.Warn("Could not list remote snapshots", "error", remoteErr) - v.UI.Warning("Could not list remote snapshots: %v.", remoteErr) + v.UI.Warningf("Could not list remote snapshots: %v.", remoteErr) } // Anything visible on the remote that doesn't correspond to a @@ -1124,7 +1231,9 @@ func (v *Vaultik) RemoveAllSnapshots(opts *RemoveOptions) (*RemoveResult, error) for _, id := range localSnaps { knownLocalKeys[snapshot.RemoteSnapshotKey(id)] = id } + var orphanRemoteKeys []string + for _, key := range remoteKeys { if _, known := knownLocalKeys[key]; !known { orphanRemoteKeys = append(orphanRemoteKeys, key) @@ -1135,6 +1244,7 @@ func (v *Vaultik) RemoveAllSnapshots(opts *RemoveOptions) (*RemoveResult, error) if !opts.JSON { v.printlnStdout("No snapshots found") } + return &RemoveResult{}, nil } @@ -1152,15 +1262,19 @@ func (v *Vaultik) localSnapshotIDs() ([]string, error) { if v.Repositories == nil { return nil, nil } + snaps, err := v.Repositories.Snapshots.ListRecent(v.ctx, 100000) if err != nil { return nil, err } + ids := make([]string, 0, len(snaps)) for _, s := range snaps { ids = append(ids, s.ID.String()) } + sort.Strings(ids) + return ids, nil } @@ -1171,10 +1285,13 @@ func (v *Vaultik) localSnapshotIDs() ([]string, error) { // fatal. func (v *Vaultik) listAllRemoteSnapshotKeys() ([]string, error) { log.Info("Listing all remote snapshots") + objectCh := v.Storage.ListStream(v.ctx, "metadata/") seen := make(map[string]bool) + var keys []string + for object := range objectCh { if object.Err != nil { return nil, fmt.Errorf("listing remote snapshots: %w", object.Err) @@ -1186,6 +1303,7 @@ func (v *Vaultik) listAllRemoteSnapshotKeys() ([]string, error) { if strings.HasPrefix(parts[1], ".") { continue } + if strings.HasSuffix(object.Key, "/") || strings.Contains(object.Key, "/manifest.json.zst") { key := parts[1] if !seen[key] { @@ -1202,30 +1320,38 @@ func (v *Vaultik) listAllRemoteSnapshotKeys() ([]string, error) { // handleRemoveAllDryRun handles the dry-run mode for removing all snapshots func (v *Vaultik) handleRemoveAllDryRun(localSnaps, orphanRemoteKeys []string, opts *RemoveOptions) (*RemoveResult, error) { result := &RemoveResult{DryRun: true} + result.SnapshotsRemoved = append(result.SnapshotsRemoved, localSnaps...) if !opts.LocalOnly { result.SnapshotsRemoved = append(result.SnapshotsRemoved, orphanRemoteKeys...) } + if !opts.JSON { - v.printfStdout("Would remove %d local snapshot(s):\n", len(localSnaps)) + v.stdoutf("Would remove %d local snapshot(s):\n", len(localSnaps)) + for _, id := range localSnaps { - v.printfStdout(" %s\n", id) + v.stdoutf(" %s\n", id) } + if !opts.LocalOnly { if len(orphanRemoteKeys) > 0 { - v.printfStdout("Would also remove %d orphan remote snapshot key(s):\n", len(orphanRemoteKeys)) + v.stdoutf("Would also remove %d orphan remote snapshot key(s):\n", len(orphanRemoteKeys)) + for _, key := range orphanRemoteKeys { - v.printfStdout(" %s\n", key) + v.stdoutf(" %s\n", key) } } else { v.printlnStdout("Would also remove snapshot metadata from remote storage (blobs untouched)") } } + v.printlnStdout("[Dry run - no changes made]") } + if opts.JSON { return result, v.outputRemoveJSON(result) } + return result, nil } @@ -1235,24 +1361,29 @@ func (v *Vaultik) handleRemoveAllDryRun(localSnaps, orphanRemoteKeys []string, o // hint is the next step. func (v *Vaultik) executeRemoveAll(localSnaps, orphanRemoteKeys []string, opts *RemoveOptions) (*RemoveResult, error) { if !opts.Force { - return nil, fmt.Errorf("--all requires --force") + return nil, errors.New("--all requires --force") } log.Info("Removing all snapshots", "local_count", len(localSnaps), "orphan_remote_count", len(orphanRemoteKeys)) result := &RemoveResult{} remoteErrors := 0 + for _, snapshotID := range localSnaps { log.Info("Removing snapshot", "snapshot_id", snapshotID) - if err := v.deleteSnapshotFromLocalDB(snapshotID); err != nil { + err := v.deleteSnapshotFromLocalDB(snapshotID) + if err != nil { log.Error("Failed to remove from local database", "snapshot_id", snapshotID, "error", err) + continue } if !opts.LocalOnly { - if err := v.deleteRemoteSnapshotByKey(snapshot.RemoteSnapshotKey(snapshotID)); err != nil { + err := v.deleteRemoteSnapshotByKey(snapshot.RemoteSnapshotKey(snapshotID)) + if err != nil { log.Warn("Failed to remove snapshot metadata from remote", "snapshot_id", snapshotID, "error", err) + remoteErrors++ } } @@ -1263,23 +1394,29 @@ func (v *Vaultik) executeRemoveAll(localSnaps, orphanRemoteKeys []string, opts * if !opts.LocalOnly { for _, key := range orphanRemoteKeys { log.Info("Removing orphan remote snapshot", "remote_key", key) - if err := v.deleteRemoteSnapshotByKey(key); err != nil { + + err := v.deleteRemoteSnapshotByKey(key) + if err != nil { log.Warn("Failed to remove orphan from remote", "remote_key", key, "error", err) + remoteErrors++ + continue } + result.SnapshotsRemoved = append(result.SnapshotsRemoved, key) } if remoteErrors == 0 { result.RemoteRemoved = true } else if v.UI != nil { - v.UI.Warning("Some remote metadata deletions failed. Run '%s' once the remote is healthy to clean up unreferenced blobs.", pruneCommandHint) + v.UI.Warningf("Some remote metadata deletions failed. Run '%s' once the remote is healthy to clean up unreferenced blobs.", pruneCommandHint) } } if v.SnapshotManager != nil { - if err := v.SnapshotManager.CleanupOrphanedData(v.ctx); err != nil { + err := v.SnapshotManager.CleanupOrphanedData(v.ctx) + if err != nil { log.Warn("Failed to clean up orphaned local data after bulk removal", "error", err) } } @@ -1288,10 +1425,11 @@ func (v *Vaultik) executeRemoveAll(localSnaps, orphanRemoteKeys []string, opts * return result, v.outputRemoveJSON(result) } - v.printfStdout("Removed %d snapshot(s)\n", len(result.SnapshotsRemoved)) + v.stdoutf("Removed %d snapshot(s)\n", len(result.SnapshotsRemoved)) + if !opts.LocalOnly && result.RemoteRemoved { v.printlnStdout("Removed snapshot metadata from remote storage") - v.printfStdout("\nNote: Removed snapshots' blobs remain on the remote. Run '%s' to delete any blobs no longer referenced by any remaining remote snapshot.\n", pruneCommandHint) + v.stdoutf("\nNote: Removed snapshots' blobs remain on the remote. Run '%s' to delete any blobs no longer referenced by any remaining remote snapshot.\n", pruneCommandHint) } return result, nil @@ -1304,16 +1442,23 @@ func (v *Vaultik) deleteSnapshotFromLocalDB(snapshotID string) error { } // Delete related records first to avoid foreign key constraints - if err := v.Repositories.Snapshots.DeleteSnapshotFiles(v.ctx, snapshotID); err != nil { + err := v.Repositories.Snapshots.DeleteSnapshotFiles(v.ctx, snapshotID) + if err != nil { return fmt.Errorf("deleting snapshot files for %s: %w", snapshotID, err) } - if err := v.Repositories.Snapshots.DeleteSnapshotBlobs(v.ctx, snapshotID); err != nil { + + err = v.Repositories.Snapshots.DeleteSnapshotBlobs(v.ctx, snapshotID) + if err != nil { return fmt.Errorf("deleting snapshot blobs for %s: %w", snapshotID, err) } - if err := v.Repositories.Snapshots.DeleteSnapshotUploads(v.ctx, snapshotID); err != nil { + + err = v.Repositories.Snapshots.DeleteSnapshotUploads(v.ctx, snapshotID) + if err != nil { return fmt.Errorf("deleting snapshot uploads for %s: %w", snapshotID, err) } - if err := v.Repositories.Snapshots.Delete(v.ctx, snapshotID); err != nil { + + err = v.Repositories.Snapshots.Delete(v.ctx, snapshotID) + if err != nil { return fmt.Errorf("deleting snapshot record %s: %w", snapshotID, err) } @@ -1330,17 +1475,21 @@ func (v *Vaultik) deleteRemoteSnapshotByKey(remoteKey string) error { objectCh := v.Storage.ListStream(v.ctx, prefix) var objectsToDelete []string + for object := range objectCh { if object.Err != nil { return fmt.Errorf("listing objects: %w", object.Err) } + objectsToDelete = append(objectsToDelete, object.Key) } for _, key := range objectsToDelete { - if err := v.Storage.Delete(v.ctx, key); err != nil { + err := v.Storage.Delete(v.ctx, key) + if err != nil { return fmt.Errorf("removing %s: %w", key, err) } + log.Debug("Deleted remote object", "key", key) } @@ -1351,6 +1500,7 @@ func (v *Vaultik) deleteRemoteSnapshotByKey(remoteKey string) error { func (v *Vaultik) outputRemoveJSON(result *RemoveResult) error { encoder := json.NewEncoder(v.Stdout) encoder.SetIndent("", " ") + return encoder.Encode(result) } @@ -1367,7 +1517,7 @@ type PruneResult struct { // before starting a new backup or on-demand via the prune command. func (v *Vaultik) PruneDatabase() (*PruneResult, error) { log.Info("Pruning local database: removing incomplete snapshots and orphaned data") - v.UI.Begin("Pruning local index database (removing incomplete snapshots and orphaned data).") + v.UI.Beginf("Pruning local index database (removing incomplete snapshots and orphaned data).") result := &PruneResult{} @@ -1384,16 +1534,23 @@ func (v *Vaultik) PruneDatabase() (*PruneResult, error) { snapshotIDStr := snapshot.ID.String() log.Info("Deleting incomplete snapshot", "snapshot_id", snapshot.ID) // Delete related records first - if err := v.Repositories.Snapshots.DeleteSnapshotFiles(v.ctx, snapshotIDStr); err != nil { + err := v.Repositories.Snapshots.DeleteSnapshotFiles(v.ctx, snapshotIDStr) + if err != nil { log.Error("Failed to delete snapshot files", "snapshot_id", snapshot.ID, "error", err) } - if err := v.Repositories.Snapshots.DeleteSnapshotBlobs(v.ctx, snapshotIDStr); err != nil { + + err = v.Repositories.Snapshots.DeleteSnapshotBlobs(v.ctx, snapshotIDStr) + if err != nil { log.Error("Failed to delete snapshot blobs", "snapshot_id", snapshot.ID, "error", err) } - if err := v.Repositories.Snapshots.DeleteSnapshotUploads(v.ctx, snapshotIDStr); err != nil { + + err = v.Repositories.Snapshots.DeleteSnapshotUploads(v.ctx, snapshotIDStr) + if err != nil { log.Error("Failed to delete snapshot uploads", "snapshot_id", snapshot.ID, "error", err) } - if err := v.Repositories.Snapshots.Delete(v.ctx, snapshotIDStr); err != nil { + + err = v.Repositories.Snapshots.Delete(v.ctx, snapshotIDStr) + if err != nil { log.Error("Failed to delete snapshot", "snapshot_id", snapshot.ID, "error", err) } else { result.SnapshotsDeleted++ @@ -1427,11 +1584,12 @@ func (v *Vaultik) PruneDatabase() (*PruneResult, error) { ) snapshotCountAfter := snapshotCountBefore - result.SnapshotsDeleted - v.UI.Complete("Pruned local index database.") - v.UI.Detail("Incomplete snapshots: %d removed (%d remain).", result.SnapshotsDeleted, snapshotCountAfter) - v.UI.Detail("Orphaned files: %d removed (%d remain).", result.FilesDeleted, fileCountAfter) - v.UI.Detail("Orphaned chunks: %d removed (%d remain).", result.ChunksDeleted, chunkCountAfter) - v.UI.Detail("Orphaned blobs: %d removed (%d remain).", result.BlobsDeleted, blobCountAfter) + + v.UI.Completef("Pruned local index database.") + v.UI.Detailf("Incomplete snapshots: %d removed (%d remain).", result.SnapshotsDeleted, snapshotCountAfter) + v.UI.Detailf("Orphaned files: %d removed (%d remain).", result.FilesDeleted, fileCountAfter) + v.UI.Detailf("Orphaned chunks: %d removed (%d remain).", result.ChunksDeleted, chunkCountAfter) + v.UI.Detailf("Orphaned blobs: %d removed (%d remain).", result.BlobsDeleted, blobCountAfter) return result, nil } @@ -1451,10 +1609,13 @@ func (v *Vaultik) getTableCount(tableName string) (int64, error) { } var count int64 - query := fmt.Sprintf("SELECT COUNT(*) FROM %s", tableName) + + query := "SELECT COUNT(*) FROM " + tableName + err := v.DB.Conn().QueryRowContext(v.ctx, query).Scan(&count) if err != nil { return 0, err } + return count, nil } diff --git a/internal/vaultik/storage_bind.go b/internal/vaultik/storage_bind.go index 8c1083d..27f79c2 100644 --- a/internal/vaultik/storage_bind.go +++ b/internal/vaultik/storage_bind.go @@ -54,10 +54,13 @@ func (v *Vaultik) EnsureStorageBinding() error { } if stored == "" { - if err := v.Repositories.LocalMeta.Set(v.ctx, database.LocalMetaKeyStorageURL, configured); err != nil { + err := v.Repositories.LocalMeta.Set(v.ctx, database.LocalMetaKeyStorageURL, configured) + if err != nil { return fmt.Errorf("recording local storage binding: %w", err) } + log.Info("Bound local index to storage destination", "storage_url", configured) + return nil } diff --git a/internal/vaultik/vaultik.go b/internal/vaultik/vaultik.go index 91c93c1..7679229 100644 --- a/internal/vaultik/vaultik.go +++ b/internal/vaultik/vaultik.go @@ -3,6 +3,7 @@ package vaultik import ( "bytes" "context" + "errors" "fmt" "io" "os" @@ -125,8 +126,9 @@ func (v *Vaultik) CanDecrypt() bool { // Returns an error if no recipients are configured func (v *Vaultik) GetEncryptor() (*crypto.Encryptor, error) { if len(v.Config.AgeRecipients) == 0 { - return nil, fmt.Errorf("no age recipients configured") + return nil, errors.New("no age recipients configured") } + return crypto.NewEncryptor(v.Config.AgeRecipients) } @@ -134,8 +136,9 @@ func (v *Vaultik) GetEncryptor() (*crypto.Encryptor, error) { // Returns an error if no secret key is configured func (v *Vaultik) GetDecryptor() (*crypto.Decryptor, error) { if v.Config.AgeSecretKey == "" { - return nil, fmt.Errorf("no age secret key configured") + return nil, errors.New("no age secret key configured") } + return crypto.NewDecryptor(v.Config.AgeSecretKey) } @@ -144,8 +147,8 @@ func (v *Vaultik) GetFilesystem() afero.Fs { return v.Fs } -// printfStdout writes formatted output to stdout. -func (v *Vaultik) printfStdout(format string, args ...any) { +// stdoutf writes formatted output to stdout. +func (v *Vaultik) stdoutf(format string, args ...any) { _, _ = fmt.Fprintf(v.Stdout, format, args...) } @@ -162,6 +165,7 @@ func (v *Vaultik) scanStdin(a ...any) (int, error) { // TestVaultik wraps a Vaultik with captured stdout/stderr for testing type TestVaultik struct { *Vaultik + Stdout *bytes.Buffer Stderr *bytes.Buffer Stdin *bytes.Buffer @@ -175,6 +179,7 @@ func NewForTesting(storage storage.Storer) *TestVaultik { stdout := &bytes.Buffer{} stderr := &bytes.Buffer{} stdin := &bytes.Buffer{} + return &TestVaultik{ Vaultik: &Vaultik{ Storage: storage, diff --git a/internal/vaultik/verify.go b/internal/vaultik/verify.go index 9ad2f42..61d1089 100644 --- a/internal/vaultik/verify.go +++ b/internal/vaultik/verify.go @@ -39,13 +39,16 @@ type VerifyResult struct { // deepVerifyFailure records a failure in the result and returns it appropriately func (v *Vaultik) deepVerifyFailure(result *VerifyResult, opts *VerifyOptions, msg string, err error) error { result.Status = "failed" + result.ErrorMessage = msg if opts.JSON { return v.outputVerifyJSON(result) } + if err != nil { return err } + return fmt.Errorf("%s", msg) } @@ -58,12 +61,14 @@ func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error { if !v.CanDecrypt() { msg := "VAULTIK_AGE_SECRET_KEY not set; required for deep verification" + return v.deepVerifyFailure(result, opts, msg, fmt.Errorf("%s", msg)) } log.Info("Starting snapshot verification", "snapshot_id", snapshotID, "mode", "deep") + if !opts.JSON { - v.printfStdout("Deep verification of snapshot: %s\n\n", snapshotID) + v.stdoutf("Deep verification of snapshot: %s\n\n", snapshotID) } manifest, tempDB, dbBlobs, err := v.loadVerificationData(snapshotID, opts, result) @@ -77,10 +82,12 @@ func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error { }() result.BlobCount = len(dbBlobs) + var totalSize int64 for _, blob := range dbBlobs { totalSize += blob.CompressedSize } + result.TotalSize = totalSize if err := v.runVerificationSteps(manifest, dbBlobs, tempDB, opts, result, totalSize); err != nil { @@ -96,10 +103,10 @@ func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error { log.Info("✓ Verification completed successfully", "snapshot_id", snapshotID, "mode", "deep", "blobs_verified", len(dbBlobs)) - v.printfStdout("\n✓ Verification completed successfully\n") - v.printfStdout(" Snapshot: %s\n", snapshotID) - v.printfStdout(" Blobs verified: %d\n", len(dbBlobs)) - v.printfStdout(" Total size: %s\n", humanize.Bytes(uint64(totalSize))) + v.stdoutf("\n✓ Verification completed successfully\n") + v.stdoutf(" Snapshot: %s\n", snapshotID) + v.stdoutf(" Blobs verified: %d\n", len(dbBlobs)) + v.stdoutf(" Total size: %s\n", humanize.Bytes(uint64(totalSize))) return nil } @@ -112,15 +119,18 @@ func (v *Vaultik) loadVerificationData(snapshotID string, opts *VerifyOptions, r // Download manifest manifestPath := fmt.Sprintf("metadata/%s/manifest.json.zst", remoteKey) log.Info("Downloading manifest", "path", manifestPath) + if !opts.JSON { - v.printfStdout("Downloading manifest...\n") + v.stdoutf("Downloading manifest...\n") } + manifestReader, err := v.Storage.Get(v.ctx, manifestPath) if err != nil { return nil, nil, nil, v.deepVerifyFailure(result, opts, fmt.Sprintf("failed to download manifest: %v", err), fmt.Errorf("failed to download manifest: %w", err)) } + defer func() { _ = manifestReader.Close() }() manifest, err := snapshot.DecodeManifest(manifestReader) @@ -133,20 +143,23 @@ func (v *Vaultik) loadVerificationData(snapshotID string, opts *VerifyOptions, r log.Info("Manifest loaded", "manifest_blob_count", manifest.BlobCount, "manifest_total_size", humanize.Bytes(uint64(manifest.TotalCompressedSize))) + if !opts.JSON { - v.printfStdout("Manifest loaded: %d blobs (%s)\n", manifest.BlobCount, humanize.Bytes(uint64(manifest.TotalCompressedSize))) - v.printfStdout("Downloading and decrypting database...\n") + v.stdoutf("Manifest loaded: %d blobs (%s)\n", manifest.BlobCount, humanize.Bytes(uint64(manifest.TotalCompressedSize))) + v.stdoutf("Downloading and decrypting database...\n") } // Download and decrypt database dbPath := fmt.Sprintf("metadata/%s/db.zst.age", remoteKey) log.Info("Downloading encrypted database", "path", dbPath) + dbReader, err := v.Storage.Get(v.ctx, dbPath) if err != nil { return nil, nil, nil, v.deepVerifyFailure(result, opts, fmt.Sprintf("failed to download database: %v", err), fmt.Errorf("failed to download database: %w", err)) } + defer func() { _ = dbReader.Close() }() tdb, err := v.decryptAndLoadDatabase(dbReader, v.Config.AgeSecretKey) @@ -159,6 +172,7 @@ func (v *Vaultik) loadVerificationData(snapshotID string, opts *VerifyOptions, r dbBlobs, err := v.getBlobsFromDatabase(snapshotID, tdb.DB) if err != nil { _ = tdb.Close() + return nil, nil, nil, v.deepVerifyFailure(result, opts, fmt.Sprintf("failed to get blobs from database: %v", err), fmt.Errorf("failed to get blobs from database: %w", err)) @@ -172,8 +186,9 @@ func (v *Vaultik) loadVerificationData(snapshotID string, opts *VerifyOptions, r log.Info("Database loaded", "db_blob_count", len(dbBlobs), "db_total_size", humanize.Bytes(uint64(dbTotalSize))) + if !opts.JSON { - v.printfStdout("Database loaded: %d blobs (%s)\n", len(dbBlobs), humanize.Bytes(uint64(dbTotalSize))) + v.stdoutf("Database loaded: %d blobs (%s)\n", len(dbBlobs), humanize.Bytes(uint64(dbTotalSize))) } return manifest, tdb, dbBlobs, nil @@ -182,25 +197,31 @@ func (v *Vaultik) loadVerificationData(snapshotID string, opts *VerifyOptions, r // runVerificationSteps executes manifest verification, blob existence check, and deep content verification func (v *Vaultik) runVerificationSteps(manifest *snapshot.Manifest, dbBlobs []snapshot.BlobInfo, tdb *tempDB, opts *VerifyOptions, result *VerifyResult, totalSize int64) error { if !opts.JSON { - v.printfStdout("Verifying manifest against database...\n") + v.stdoutf("Verifying manifest against database...\n") } - if err := v.verifyManifestAgainstDatabase(manifest, dbBlobs); err != nil { + + err := v.verifyManifestAgainstDatabase(manifest, dbBlobs) + if err != nil { return v.deepVerifyFailure(result, opts, err.Error(), err) } if !opts.JSON { - v.printfStdout("Manifest verified.\n") - v.printfStdout("Checking blob existence in remote storage...\n") + v.stdoutf("Manifest verified.\n") + v.stdoutf("Checking blob existence in remote storage...\n") } - if err := v.verifyBlobExistenceFromDB(dbBlobs); err != nil { + + err = v.verifyBlobExistenceFromDB(dbBlobs) + if err != nil { return v.deepVerifyFailure(result, opts, err.Error(), err) } if !opts.JSON { - v.printfStdout("All blobs exist.\n") - v.printfStdout("Downloading and verifying blob contents (%d blobs, %s)...\n", len(dbBlobs), humanize.Bytes(uint64(totalSize))) + v.stdoutf("All blobs exist.\n") + v.stdoutf("Downloading and verifying blob contents (%d blobs, %s)...\n", len(dbBlobs), humanize.Bytes(uint64(totalSize))) } - if err := v.performDeepVerificationFromDB(dbBlobs, tdb.DB, opts); err != nil { + + err = v.performDeepVerificationFromDB(dbBlobs, tdb.DB, opts) + if err != nil { return v.deepVerifyFailure(result, opts, err.Error(), err) } @@ -210,12 +231,14 @@ func (v *Vaultik) runVerificationSteps(manifest *snapshot.Manifest, dbBlobs []sn // tempDB wraps sql.DB with cleanup type tempDB struct { *sql.DB + tempPath string } func (t *tempDB) Close() error { err := t.DB.Close() _ = os.Remove(t.tempPath) + return err } @@ -245,16 +268,20 @@ func (v *Vaultik) decryptAndLoadDatabase(reader io.ReadCloser, secretKey string) if err != nil { return nil, fmt.Errorf("failed to create temp file: %w", err) } + tempPath := tempFile.Name() // Stream decompress directly to file log.Info("Decompressing database...") + written, err := io.Copy(tempFile, decompressor) if err != nil { _ = tempFile.Close() _ = os.Remove(tempPath) + return nil, fmt.Errorf("failed to decompress database: %w", err) } + _ = tempFile.Close() log.Info("Database decompressed", "size", humanize.Bytes(uint64(written))) @@ -263,6 +290,7 @@ func (v *Vaultik) decryptAndLoadDatabase(reader io.ReadCloser, secretKey string) db, err := sql.Open("sqlite", tempPath) if err != nil { _ = os.Remove(tempPath) + return nil, fmt.Errorf("failed to open database: %w", err) } @@ -332,21 +360,28 @@ func (v *Vaultik) verifyBlobChunks(db *sql.DB, blobHash string, decompressor io. WHERE b.blob_hash = ? ORDER BY bc.offset ` + rows, err := db.QueryContext(v.ctx, query, blobHash) if err != nil { return 0, fmt.Errorf("failed to query blob chunks: %w", err) } + defer func() { _ = rows.Close() }() var lastOffset int64 = -1 + chunkCount := 0 totalRead := int64(0) // Verify each chunk in the blob for rows.Next() { - var chunkHash string - var offset, length int64 - if err := rows.Scan(&chunkHash, &offset, &length); err != nil { + var ( + chunkHash string + offset, length int64 + ) + + err := rows.Scan(&chunkHash, &offset, &length) + if err != nil { return 0, fmt.Errorf("failed to scan chunk row: %w", err) } @@ -354,6 +389,7 @@ func (v *Vaultik) verifyBlobChunks(db *sql.DB, blobHash string, decompressor io. if offset <= lastOffset { return 0, fmt.Errorf("chunks out of order: offset %d after %d", offset, lastOffset) } + lastOffset = offset // Read chunk data from decompressed stream @@ -363,6 +399,7 @@ func (v *Vaultik) verifyBlobChunks(db *sql.DB, blobHash string, decompressor io. if _, err := io.CopyN(io.Discard, decompressor, skipBytes); err != nil { return 0, fmt.Errorf("failed to skip to offset %d: %w", offset, err) } + totalRead = offset } @@ -371,6 +408,7 @@ func (v *Vaultik) verifyBlobChunks(db *sql.DB, blobHash string, decompressor io. if _, err := io.ReadFull(decompressor, chunkData); err != nil { return 0, fmt.Errorf("failed to read chunk at offset %d: %w", offset, err) } + totalRead += length // Verify chunk hash @@ -401,6 +439,7 @@ func (v *Vaultik) verifyBlobFinalIntegrity(decompressor io.Reader, blobHasher ha if err != nil { return fmt.Errorf("failed to check for remaining blob data: %w", err) } + if remaining > 0 { return fmt.Errorf("blob has %d unexpected trailing bytes not covered by chunk list", remaining) } @@ -424,19 +463,27 @@ func (v *Vaultik) getBlobsFromDatabase(snapshotID string, db *sql.DB) ([]snapsho WHERE sb.snapshot_id = ? ORDER BY b.blob_hash ` + rows, err := db.QueryContext(v.ctx, query, snapshotID) if err != nil { return nil, fmt.Errorf("failed to query snapshot blobs: %w", err) } + defer func() { _ = rows.Close() }() var blobs []snapshot.BlobInfo + for rows.Next() { - var hash string - var size int64 - if err := rows.Scan(&hash, &size); err != nil { + var ( + hash string + size int64 + ) + + err := rows.Scan(&hash, &size) + if err != nil { return nil, fmt.Errorf("failed to scan blob row: %w", err) } + blobs = append(blobs, snapshot.BlobInfo{ Hash: hash, CompressedSize: size, @@ -481,6 +528,7 @@ func (v *Vaultik) verifyManifestAgainstDatabase(manifest *snapshot.Manifest, dbB if !exists { return fmt.Errorf("manifest contains blob %s not in database", hash) } + if dbSize != manifestSize { return fmt.Errorf("blob %s size mismatch: database has %d bytes, manifest has %d bytes", hash, dbSize, manifestSize) @@ -491,6 +539,7 @@ func (v *Vaultik) verifyManifestAgainstDatabase(manifest *snapshot.Manifest, dbB "manifest_blobs", len(manifestBlobMap), "database_blobs", len(dbBlobMap), ) + return nil } @@ -525,6 +574,7 @@ func (v *Vaultik) verifyBlobExistenceFromDB(blobs []snapshot.BlobInfo) error { } log.Info("✓ All blobs exist in storage") + return nil } @@ -546,7 +596,8 @@ func (v *Vaultik) performDeepVerificationFromDB(blobs []snapshot.BlobInfo, db *s for i, blobInfo := range blobs { // Verify individual blob - if err := v.verifyBlob(blobInfo, db); err != nil { + err := v.verifyBlob(blobInfo, db) + if err != nil { return fmt.Errorf("blob %s verification failed: %w", blobInfo.Hash, err) } @@ -556,8 +607,10 @@ func (v *Vaultik) performDeepVerificationFromDB(blobs []snapshot.BlobInfo, db *s // Calculate ETA based on bytes processed var eta time.Duration + if bytesProcessed > 0 { bytesPerSec := float64(bytesProcessed) / elapsed.Seconds() + bytesRemaining := totalBytesExpected - bytesProcessed if bytesPerSec > 0 { eta = time.Duration(float64(bytesRemaining)/bytesPerSec) * time.Second @@ -577,7 +630,7 @@ func (v *Vaultik) performDeepVerificationFromDB(blobs []snapshot.BlobInfo, db *s ) if !opts.JSON { - v.printfStdout(" Verified %d/%d blobs (%d remaining) - %s/%s - elapsed %s, eta %s\n", + v.stdoutf(" Verified %d/%d blobs (%d remaining) - %s/%s - elapsed %s, eta %s\n", i+1, len(blobs), remaining, humanize.Bytes(uint64(bytesProcessed)), humanize.Bytes(uint64(totalBytesExpected)), diff --git a/internal/vaultik/verify_test.go b/internal/vaultik/verify_test.go index 6ff6adc..c1fc1c3 100644 --- a/internal/vaultik/verify_test.go +++ b/internal/vaultik/verify_test.go @@ -25,6 +25,7 @@ func TestTeeReaderWithDecryption(t *testing.T) { // Compress the data var compressedBuf bytes.Buffer + compressor, err := zstd.NewWriter(&compressedBuf, zstd.WithEncoderLevel(zstd.SpeedDefault)) require.NoError(t, err) _, err = compressor.Write(testData) @@ -40,6 +41,7 @@ func TestTeeReaderWithDecryption(t *testing.T) { require.NoError(t, err) var encryptedBuf bytes.Buffer + err = encryptor.EncryptStream(&encryptedBuf, bytes.NewReader(compressedBuf.Bytes())) require.NoError(t, err) @@ -68,6 +70,7 @@ func TestTeeReaderWithDecryption(t *testing.T) { // Decompress decompressor, err := zstd.NewReader(decryptedReader) require.NoError(t, err) + defer decompressor.Close() // Read all decompressed data (simulating chunk verification) -- 2.49.1 From 7ae470e530e9edcb0716394c48e872380321264c Mon Sep 17 00:00:00 2001 From: sneak Date: Fri, 7 Aug 2026 18:51:21 +0000 Subject: [PATCH 2/2] Remediate all lint findings under the canonical golangci-lint config Fix every finding surfaced by the canonical .golangci.yml with golangci-lint v2.12.2 (refs #61), behavior-preserving throughout: - err113: dynamic errors replaced with package-level sentinels and %w wrapping; direct comparisons converted to errors.Is - goprintffuncname: printf-style helpers renamed with an f suffix (ui.Writer message methods, cli.ReportErrorf, database.Fatalf, vaultik stdoutf) and all call sites updated - revive: stuttering type names renamed (blob.Handler, blob.WithReader, blob.ChunkPosition, storage.URL, storage.Info), doc comments added, unused parameters blanked, package comments added - contextcheck/noctx: ctx threaded through blob.Packer (AddChunk/Flush/FinalizeBlob/PackChunks) and scanner call sites; context-aware exec and sql variants used - funlen/cyclop/gocognit/nestif/dupl: oversized or duplicated functions split into focused helpers across production and test code - paralleltest/tparallel/thelper/usetesting/testpackage: tests parallelized where safe (global log.Initialize kept in the serial phase), helpers marked, t.TempDir adopted, external test packages where only exported API is used - gosec: integer conversions clamped or justified, header timeouts added, remaining findings suppressed with per-site justifications - mnd/goconst/lll/wsl_v5/nlreturn/noinlineerr/errcheck and other mechanical findings fixed directly Remove the deprecated log.LogOptions alias (callers migrated to log.Options). make check is green. --- TODO.md | 13 +- cmd/vaultik/main.go | 13 +- internal/blob/errors.go | 15 +- internal/blob/packer.go | 251 +++-- internal/blob/packer_test.go | 664 ++++++------- internal/blobgen/compress.go | 26 +- internal/blobgen/compress_test.go | 23 +- internal/blobgen/reader.go | 4 +- internal/blobgen/writer.go | 41 +- internal/blobgen/writer_test.go | 13 +- internal/chunker/chunker.go | 65 +- internal/chunker/chunker_isolated_test.go | 13 +- internal/chunker/chunker_test.go | 200 ++-- internal/chunker/fastcdc.go | 106 ++- internal/cli/app.go | 112 ++- internal/cli/app_test.go | 20 +- internal/cli/config.go | 205 ++-- internal/cli/config_test.go | 34 +- internal/cli/database.go | 24 +- internal/cli/duration.go | 114 ++- internal/cli/duration_test.go | 99 +- internal/cli/entry.go | 12 +- internal/cli/entry_test.go | 16 +- internal/cli/info.go | 8 +- internal/cli/prune.go | 8 +- internal/cli/remote.go | 69 +- internal/cli/root.go | 51 +- internal/cli/snapshot.go | 258 ++--- internal/cli/snapshot_restore.go | 31 +- internal/cli/vaultik_snapshot_types.go | 2 + internal/cli/version.go | 31 +- internal/config/config.go | 159 ++-- internal/config/config_test.go | 79 +- internal/config/size.go | 27 +- internal/crypto/encryption.go | 24 +- internal/crypto/encryption_test.go | 38 +- internal/database/blob_chunks.go | 52 +- internal/database/blob_chunks_test.go | 196 ++-- internal/database/blobs.go | 160 ++-- internal/database/blobs_test.go | 41 +- internal/database/cascade_debug_test.go | 92 +- internal/database/chunk_files.go | 250 ++--- internal/database/chunk_files_test.go | 219 +++-- internal/database/chunks.go | 25 +- internal/database/chunks_ext.go | 1 + internal/database/chunks_test.go | 19 +- internal/database/database.go | 163 ++-- internal/database/database_test.go | 86 +- internal/database/file_chunks.go | 329 ++++--- internal/database/file_chunks_test.go | 112 ++- internal/database/files.go | 205 ++-- internal/database/files_test.go | 84 +- internal/database/helpers_internal_test.go | 81 ++ internal/database/helpers_test.go | 52 + internal/database/local_meta.go | 1 + internal/database/local_meta_test.go | 15 +- internal/database/models.go | 60 +- internal/database/module.go | 12 +- internal/database/repositories_test.go | 206 ++-- .../database/repository_comprehensive_test.go | 214 +++-- internal/database/repository_debug_test.go | 132 +-- .../database/repository_edge_cases_test.go | 407 ++++---- internal/database/snapshots.go | 357 ++++--- internal/database/snapshots_test.go | 103 +- internal/database/uploads.go | 39 +- internal/globals/globals.go | 20 +- internal/globals/globals_test.go | 8 +- internal/log/log.go | 40 +- internal/log/module.go | 8 +- internal/log/tty_handler.go | 20 +- internal/models/models.go | 2 + internal/models/models_test.go | 14 +- internal/pidlock/pidlock.go | 21 +- internal/pidlock/pidlock_test.go | 40 +- internal/s3/client.go | 19 +- internal/s3/client_test.go | 26 +- internal/s3/module.go | 4 +- internal/s3/s3_test.go | 58 +- internal/snapshot/backup_test.go | 524 +++++----- internal/snapshot/exclude_test.go | 129 ++- internal/snapshot/file_change_test.go | 136 ++- internal/snapshot/manifest.go | 17 +- internal/snapshot/module.go | 6 +- internal/snapshot/permission_error_test.go | 27 +- internal/snapshot/progress.go | 357 ++++--- internal/snapshot/scanner.go | 897 +++++++++++------- internal/snapshot/scanner_test.go | 219 +++-- internal/snapshot/snapshot.go | 557 ++++++----- internal/snapshot/snapshot_test.go | 119 ++- internal/storage/file.go | 47 +- internal/storage/module.go | 110 ++- internal/storage/rclone.go | 21 +- internal/storage/s3.go | 9 +- internal/storage/storer.go | 12 +- internal/storage/url.go | 54 +- internal/types/types.go | 30 +- internal/ui/ui.go | 93 +- internal/ui/ui_test.go | 54 +- internal/vaultik/blob_fetch.go | 15 +- internal/vaultik/blob_fetch_hash_test.go | 65 +- internal/vaultik/blobcache.go | 121 +-- internal/vaultik/blobcache_test.go | 55 +- internal/vaultik/helpers.go | 64 +- internal/vaultik/helpers_test.go | 20 +- internal/vaultik/info.go | 226 +++-- internal/vaultik/integration_test.go | 693 ++++++++------ internal/vaultik/prune.go | 71 +- internal/vaultik/purge_per_name_test.go | 70 +- internal/vaultik/remove_snapshot_test.go | 82 +- internal/vaultik/restore.go | 615 +++++++----- internal/vaultik/restore_locality_test.go | 173 ++-- internal/vaultik/restore_plan.go | 15 +- internal/vaultik/restore_sweeper.go | 15 +- .../restore_sweeper_integration_test.go | 166 ++-- internal/vaultik/snapshot.go | 751 ++++++++++----- internal/vaultik/snapshot_prune_test.go | 10 +- internal/vaultik/storage_bind.go | 24 +- internal/vaultik/storage_bind_test.go | 10 +- internal/vaultik/vaultik.go | 22 +- internal/vaultik/verify.go | 177 ++-- internal/vaultik/verify_test.go | 11 +- 121 files changed, 8344 insertions(+), 5406 deletions(-) create mode 100644 internal/database/helpers_internal_test.go create mode 100644 internal/database/helpers_test.go diff --git a/TODO.md b/TODO.md index 1458942..17d8258 100644 --- a/TODO.md +++ b/TODO.md @@ -14,13 +14,16 @@ pre-1.0 # Next Step -Remediate the 2,990 lint findings surfaced by the standard -`.golangci.yml` (issue #61): behavior-preserving fixes in per-linter or -per-package chunks, mechanical linters first, until `make check` is -green on `main`. +Reconcile the uncommitted ARCHITECTURE.md edits on main: finish and +commit, or revert. # Completed Steps +- 2026-08-07: Updated golangci-lint to v2.12.2 everywhere it is pinned + (`Dockerfile` lint stage, `Makefile` deps target), replaced + `.golangci.yml` with the canonical config (v2 schema, `default: all`), + and remediated all lint findings it surfaced (issue #61): + behavior-preserving fixes across every package, `make check` green. - 2026-08-07: Added the standard `.golangci.yml` and `.editorconfig` (issue #59); lint findings under the new config are tracked in issue #61. `script/bootstrap` now installs sqlite3 (needed by tests). @@ -46,8 +49,6 @@ green on `main`. # Future Steps -- Reconcile the uncommitted ARCHITECTURE.md edits on main: finish and - commit, or revert. - Review stale local branches (add-godoc-to-cli-package, feature/pluggable-storage-backend) and merge or delete them. - Define remaining scope for a first tagged release and cut v0.1.0. diff --git a/cmd/vaultik/main.go b/cmd/vaultik/main.go index 234de24..c9cbacc 100644 --- a/cmd/vaultik/main.go +++ b/cmd/vaultik/main.go @@ -1,3 +1,4 @@ +// Package main is the vaultik command-line entry point. package main import ( @@ -11,13 +12,14 @@ import ( func main() { // CPU profiling: set VAULTIK_CPUPROFILE=/path/to/cpu.prof if cpuProfile := os.Getenv("VAULTIK_CPUPROFILE"); cpuProfile != "" { - f, err := os.Create(cpuProfile) + f, err := os.Create(cpuProfile) //nolint:gosec // G304: operator-set path if err != nil { panic("could not create CPU profile: " + err.Error()) } defer func() { _ = f.Close() }() - if err := pprof.StartCPUProfile(f); err != nil { + err = pprof.StartCPUProfile(f) + if err != nil { panic("could not start CPU profile: " + err.Error()) } @@ -27,7 +29,7 @@ func main() { // Memory profiling: set VAULTIK_MEMPROFILE=/path/to/mem.prof if memProfile := os.Getenv("VAULTIK_MEMPROFILE"); memProfile != "" { defer func() { - f, err := os.Create(memProfile) + f, err := os.Create(memProfile) //nolint:gosec // G304: operator-set path if err != nil { panic("could not create memory profile: " + err.Error()) } @@ -35,11 +37,12 @@ func main() { runtime.GC() // get up-to-date statistics - if err := pprof.WriteHeapProfile(f); err != nil { + err = pprof.WriteHeapProfile(f) + if err != nil { panic("could not write memory profile: " + err.Error()) } }() } - cli.CLIEntry() + cli.Entry() } diff --git a/internal/blob/errors.go b/internal/blob/errors.go index ceaa0ad..7817486 100644 --- a/internal/blob/errors.go +++ b/internal/blob/errors.go @@ -2,5 +2,18 @@ package blob import "errors" -// ErrBlobSizeLimitExceeded is returned when adding a chunk would exceed the blob size limit +// ErrBlobSizeLimitExceeded is returned when adding a chunk would exceed +// the blob size limit. var ErrBlobSizeLimitExceeded = errors.New("adding chunk would exceed blob size limit") + +// ErrNoRecipients is returned when a Packer is created without any age +// recipients; blobs must always be encrypted. +var ErrNoRecipients = errors.New("recipients are required - blobs must be encrypted") + +// ErrInvalidMaxBlobSize is returned when the configured maximum blob size +// is zero or negative. +var ErrInvalidMaxBlobSize = errors.New("max blob size must be positive") + +// ErrNoFilesystem is returned when a Packer is created without a filesystem +// for temporary files. +var ErrNoFilesystem = errors.New("filesystem is required") diff --git a/internal/blob/packer.go b/internal/blob/packer.go index 1e02096..cfbb081 100644 --- a/internal/blob/packer.go +++ b/internal/blob/packer.go @@ -32,21 +32,28 @@ import ( "sneak.berlin/go/vaultik/internal/types" ) -// BlobHandler is a callback function invoked when a blob is finalized and ready for upload. -// The handler receives a BlobWithReader containing the blob metadata and a reader for -// the compressed and encrypted blob content. The handler is responsible for uploading -// the blob to storage and cleaning up any temporary files. -type BlobHandler func(blob *BlobWithReader) error +// Handler is a callback function invoked when a blob is finalized and +// ready for upload. The handler receives a WithReader containing the +// blob metadata and a reader for the compressed and encrypted blob content. +// The handler is responsible for uploading the blob to storage and cleaning +// up any temporary files. +type Handler func(blob *WithReader) error // PackerConfig holds configuration for creating a Packer. // All fields except BlobHandler are required. type PackerConfig struct { - MaxBlobSize int64 // Maximum size of a blob before forcing finalization - CompressionLevel int // Zstd compression level (1-19, higher = better compression) - Recipients []string // Age recipients for encryption - Repositories *database.Repositories // Database repositories for tracking blob metadata - BlobHandler BlobHandler // Optional callback when blob is ready for upload - Fs afero.Fs // Filesystem for temporary files + // MaxBlobSize is the maximum size of a blob before forcing finalization. + MaxBlobSize int64 + // CompressionLevel is the zstd level (1-19, higher = better compression). + CompressionLevel int + // Recipients holds the age recipients for encryption. + Recipients []string + // Repositories provides database access for tracking blob metadata. + Repositories *database.Repositories + // BlobHandler is an optional callback when a blob is ready for upload. + BlobHandler Handler + // Fs is the filesystem used for temporary files. + Fs afero.Fs } // PendingChunk represents a chunk waiting to be inserted into the database. @@ -62,7 +69,7 @@ type Packer struct { maxBlobSize int64 compressionLevel int recipients []string // Age recipients for encryption - blobHandler BlobHandler // Called when blob is ready + blobHandler Handler // Called when blob is ready repos *database.Repositories // For creating blob records fs afero.Fs // Filesystem for temporary files @@ -109,21 +116,21 @@ type FinishedBlob struct { ID string Hash string Data []byte // Compressed data - Chunks []*BlobChunkRef + Chunks []*ChunkPosition CreatedTS time.Time Uncompressed int64 Compressed int64 } -// BlobChunkRef represents a chunk's position within a blob -type BlobChunkRef struct { +// ChunkPosition represents a chunk's position within a blob +type ChunkPosition struct { ChunkHash string Offset int64 Length int64 } -// BlobWithReader wraps a FinishedBlob with its data reader -type BlobWithReader struct { +// WithReader wraps a FinishedBlob with its data reader +type WithReader struct { *FinishedBlob Reader io.ReadSeeker @@ -136,15 +143,15 @@ type BlobWithReader struct { // Returns an error if required configuration fields are missing or invalid. func NewPacker(cfg PackerConfig) (*Packer, error) { if len(cfg.Recipients) == 0 { - return nil, errors.New("recipients are required - blobs must be encrypted") + return nil, ErrNoRecipients } if cfg.MaxBlobSize <= 0 { - return nil, errors.New("max blob size must be positive") + return nil, ErrInvalidMaxBlobSize } if cfg.Fs == nil { - return nil, errors.New("filesystem is required") + return nil, ErrNoFilesystem } return &Packer{ @@ -162,7 +169,7 @@ func NewPacker(cfg PackerConfig) (*Packer, error) { // The handler is responsible for uploading the blob to storage. // If no handler is set, finalized blobs are stored in memory and can be // retrieved with GetFinishedBlobs(). -func (p *Packer) SetBlobHandler(handler BlobHandler) { +func (p *Packer) SetBlobHandler(handler Handler) { p.mu.Lock() defer p.mu.Unlock() @@ -184,13 +191,13 @@ func (p *Packer) AddPendingChunk(hash string, size int64) { // In this case, the caller should finalize the current blob and retry. // The chunk data is written immediately and can be garbage collected after this call. // Thread-safe. -func (p *Packer) AddChunk(chunk *ChunkRef) error { +func (p *Packer) AddChunk(ctx context.Context, chunk *ChunkRef) error { p.mu.Lock() defer p.mu.Unlock() // Initialize new blob if needed if p.currentBlob == nil { - err := p.startNewBlob() + err := p.startNewBlob(ctx) if err != nil { return fmt.Errorf("starting new blob: %w", err) } @@ -222,12 +229,12 @@ func (p *Packer) AddChunk(chunk *ChunkRef) error { // This should be called after all chunks have been added to ensure no data is lost. // If a BlobHandler is set, it will be called with the finalized blob. // Thread-safe. -func (p *Packer) Flush() error { +func (p *Packer) Flush(ctx context.Context) error { p.mu.Lock() defer p.mu.Unlock() if p.currentBlob != nil && len(p.currentBlob.chunks) > 0 { - err := p.finalizeCurrentBlob() + err := p.finalizeCurrentBlob(ctx) if err != nil { return fmt.Errorf("finalizing blob: %w", err) } @@ -242,7 +249,7 @@ func (p *Packer) Flush() error { // BlobHandler (if set) or stored internally. // Caller must handle retrying any chunk that triggered size limit exceeded. // Not thread-safe - caller must hold the lock. -func (p *Packer) FinalizeBlob() error { +func (p *Packer) FinalizeBlob(ctx context.Context) error { p.mu.Lock() defer p.mu.Unlock() @@ -250,7 +257,7 @@ func (p *Packer) FinalizeBlob() error { return nil } - return p.finalizeCurrentBlob() + return p.finalizeCurrentBlob(ctx) } // GetFinishedBlobs returns all completed blobs and clears the internal list. @@ -267,8 +274,33 @@ func (p *Packer) GetFinishedBlobs() []*FinishedBlob { return blobs } +// PackChunks is a convenience method to pack multiple chunks at once. +func (p *Packer) PackChunks(ctx context.Context, chunks []*ChunkRef) error { + for _, chunk := range chunks { + err := p.AddChunk(ctx, chunk) + if errors.Is(err, ErrBlobSizeLimitExceeded) { + // Finalize current blob and retry + err = p.FinalizeBlob(ctx) + if err != nil { + return fmt.Errorf("finalizing blob before retry: %w", err) + } + + // Retry the chunk + err = p.AddChunk(ctx, chunk) + if err != nil { + return fmt.Errorf( + "adding chunk %s after finalize: %w", chunk.Hash, err) + } + } else if err != nil { + return fmt.Errorf("adding chunk %s: %w", chunk.Hash, err) + } + } + + return p.Flush(ctx) +} + // startNewBlob initializes a new blob (must be called with lock held) -func (p *Packer) startNewBlob() error { +func (p *Packer) startNewBlob(ctx context.Context) error { // Generate UUID for the blob blobID := uuid.New().String() @@ -280,8 +312,9 @@ func (p *Packer) startNewBlob() error { } blob := &database.Blob{ - ID: blobIDTyped, - Hash: types.BlobHash("temp-placeholder-" + blobID), // Temporary placeholder until finalized + ID: blobIDTyped, + // Temporary placeholder hash until finalized. + Hash: types.BlobHash("temp-placeholder-" + blobID), CreatedTS: time.Now().UTC(), FinishedTS: nil, UncompressedSize: 0, @@ -289,9 +322,12 @@ func (p *Packer) startNewBlob() error { UploadedTS: nil, } - if err := p.repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error { - return p.repos.Blobs.Create(ctx, tx, blob) - }); err != nil { + err = p.repos.WithTx( + ctx, + func(txCtx context.Context, tx *sql.Tx) error { + return p.repos.Blobs.Create(txCtx, tx, blob) + }) + if err != nil { return fmt.Errorf("creating blob record: %w", err) } } @@ -321,16 +357,19 @@ func (p *Packer) startNewBlob() error { size: 0, } - log.Debug("Created new blob container", "blob_id", blobID, "temp_file", tempFile.Name()) + log.Debug("Created new blob container", + "blob_id", blobID, "temp_file", tempFile.Name()) return nil } -// addChunkToCurrentBlob adds a chunk to the current blob (must be called with lock held) +// addChunkToCurrentBlob adds a chunk to the current blob (must be called +// with lock held). func (p *Packer) addChunkToCurrentBlob(chunk *ChunkRef) error { // Skip if chunk already in current blob if p.currentBlob.chunkSet[chunk.Hash] { - log.Debug("Skipping duplicate chunk already in current blob", "chunk_hash", chunk.Hash) + log.Debug("Skipping duplicate chunk already in current blob", + "chunk_hash", chunk.Hash) return nil } @@ -339,7 +378,8 @@ func (p *Packer) addChunkToCurrentBlob(chunk *ChunkRef) error { offset := p.currentBlob.size // Write to the blobgen writer (compression -> encryption -> disk) - if _, err := p.currentBlob.writer.Write(chunk.Data); err != nil { + _, err := p.currentBlob.writer.Write(chunk.Data) + if err != nil { return fmt.Errorf("writing to blob stream: %w", err) } @@ -372,7 +412,7 @@ func (p *Packer) addChunkToCurrentBlob(chunk *ChunkRef) error { } // finalizeCurrentBlob completes the current blob (must be called with lock held) -func (p *Packer) finalizeCurrentBlob() error { +func (p *Packer) finalizeCurrentBlob(ctx context.Context) error { if p.currentBlob == nil { return nil } @@ -387,7 +427,8 @@ func (p *Packer) finalizeCurrentBlob() error { chunksToInsert := p.pendingChunks p.pendingChunks = nil - if err := p.commitBlobToDatabase(blobHash, finalSize, chunksToInsert); err != nil { + err = p.commitBlobToDatabase(ctx, blobHash, finalSize, chunksToInsert) + if err != nil { return err } @@ -415,15 +456,18 @@ func (p *Packer) finalizeCurrentBlob() error { return p.deliverFinishedBlob(finished, insertedChunkHashes) } -// closeBlobWriter closes the writer, syncs to disk, and returns the blob hash and final size +// closeBlobWriter closes the writer, syncs to disk, and returns the blob +// hash and final size. func (p *Packer) closeBlobWriter() (string, int64, error) { - if err := p.currentBlob.writer.Close(); err != nil { + err := p.currentBlob.writer.Close() + if err != nil { p.cleanupTempFile() return "", 0, fmt.Errorf("closing blobgen writer: %w", err) } - if err := p.currentBlob.tempFile.Sync(); err != nil { + err = p.currentBlob.tempFile.Sync() + if err != nil { p.cleanupTempFile() return "", 0, fmt.Errorf("syncing temp file: %w", err) @@ -436,7 +480,8 @@ func (p *Packer) closeBlobWriter() (string, int64, error) { return "", 0, fmt.Errorf("getting file size: %w", err) } - if _, err := p.currentBlob.tempFile.Seek(0, io.SeekStart); err != nil { + _, err = p.currentBlob.tempFile.Seek(0, io.SeekStart) + if err != nil { p.cleanupTempFile() return "", 0, fmt.Errorf("seeking to start: %w", err) @@ -447,11 +492,11 @@ func (p *Packer) closeBlobWriter() (string, int64, error) { return hex.EncodeToString(finalHash), finalSize, nil } -// buildChunkRefs creates BlobChunkRef entries from the current blob's chunks -func (p *Packer) buildChunkRefs() []*BlobChunkRef { - refs := make([]*BlobChunkRef, 0, len(p.currentBlob.chunks)) +// buildChunkRefs creates ChunkPosition entries from the current blob's chunks +func (p *Packer) buildChunkRefs() []*ChunkPosition { + refs := make([]*ChunkPosition, 0, len(p.currentBlob.chunks)) for _, chunk := range p.currentBlob.chunks { - refs = append(refs, &BlobChunkRef{ + refs = append(refs, &ChunkPosition{ ChunkHash: chunk.Hash, Offset: chunk.Offset, Length: chunk.Size, }) } @@ -460,7 +505,10 @@ func (p *Packer) buildChunkRefs() []*BlobChunkRef { } // commitBlobToDatabase inserts pending chunks, blob_chunks, and updates the blob record -func (p *Packer) commitBlobToDatabase(blobHash string, finalSize int64, chunksToInsert []PendingChunk) error { +func (p *Packer) commitBlobToDatabase( + ctx context.Context, + blobHash string, finalSize int64, chunksToInsert []PendingChunk, +) error { if p.repos == nil { return nil } @@ -472,30 +520,12 @@ func (p *Packer) commitBlobToDatabase(blobHash string, finalSize int64, chunksTo return fmt.Errorf("parsing blob ID: %w", parseErr) } - err := p.repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error { - for _, chunk := range chunksToInsert { - dbChunk := &database.Chunk{ChunkHash: types.ChunkHash(chunk.Hash), Size: chunk.Size} - - err := p.repos.Chunks.Create(ctx, tx, dbChunk) - if err != nil { - return fmt.Errorf("creating chunk: %w", err) - } - } - - for _, chunk := range p.currentBlob.chunks { - blobChunk := &database.BlobChunk{ - BlobID: blobIDTyped, ChunkHash: types.ChunkHash(chunk.Hash), - Offset: chunk.Offset, Length: chunk.Size, - } - - err := p.repos.BlobChunks.Create(ctx, tx, blobChunk) - if err != nil { - return fmt.Errorf("creating blob_chunk: %w", err) - } - } - - return p.repos.Blobs.UpdateFinished(ctx, tx, p.currentBlob.id, blobHash, p.currentBlob.size, finalSize) - }) + err := p.repos.WithTx( + ctx, + func(txCtx context.Context, tx *sql.Tx) error { + return p.insertBlobRecords(txCtx, tx, blobIDTyped, blobHash, + finalSize, chunksToInsert) + }) if err != nil { p.cleanupTempFile() @@ -503,28 +533,69 @@ func (p *Packer) commitBlobToDatabase(blobHash string, finalSize int64, chunksTo } log.Debug("Committed blob transaction", - "chunks_inserted", len(chunksToInsert), "blob_chunks_inserted", len(p.currentBlob.chunks)) + "chunks_inserted", len(chunksToInsert), + "blob_chunks_inserted", len(p.currentBlob.chunks)) return nil } +// insertBlobRecords inserts pending chunks and blob_chunk rows, then marks +// the blob finished, all within the supplied transaction. +func (p *Packer) insertBlobRecords( + ctx context.Context, + tx *sql.Tx, + blobIDTyped types.BlobID, + blobHash string, + finalSize int64, + chunksToInsert []PendingChunk, +) error { + for _, chunk := range chunksToInsert { + dbChunk := &database.Chunk{ + ChunkHash: types.ChunkHash(chunk.Hash), Size: chunk.Size, + } + + err := p.repos.Chunks.Create(ctx, tx, dbChunk) + if err != nil { + return fmt.Errorf("creating chunk: %w", err) + } + } + + for _, chunk := range p.currentBlob.chunks { + blobChunk := &database.BlobChunk{ + BlobID: blobIDTyped, ChunkHash: types.ChunkHash(chunk.Hash), + Offset: chunk.Offset, Length: chunk.Size, + } + + err := p.repos.BlobChunks.Create(ctx, tx, blobChunk) + if err != nil { + return fmt.Errorf("creating blob_chunk: %w", err) + } + } + + return p.repos.Blobs.UpdateFinished(ctx, tx, p.currentBlob.id, blobHash, + p.currentBlob.size, finalSize) +} + // deliverFinishedBlob passes the blob to the handler or stores it internally -func (p *Packer) deliverFinishedBlob(finished *FinishedBlob, insertedChunkHashes []string) error { +func (p *Packer) deliverFinishedBlob( + finished *FinishedBlob, insertedChunkHashes []string, +) error { if p.blobHandler != nil { - if _, err := p.currentBlob.tempFile.Seek(0, io.SeekStart); err != nil { + _, err := p.currentBlob.tempFile.Seek(0, io.SeekStart) + if err != nil { p.cleanupTempFile() return fmt.Errorf("seeking for handler: %w", err) } - blobWithReader := &BlobWithReader{ + blobWithReader := &WithReader{ FinishedBlob: finished, Reader: p.currentBlob.tempFile, TempFile: p.currentBlob.tempFile, InsertedChunkHashes: insertedChunkHashes, } - err := p.blobHandler(blobWithReader) + err = p.blobHandler(blobWithReader) if err != nil { p.cleanupTempFile() @@ -539,7 +610,8 @@ func (p *Packer) deliverFinishedBlob(finished *FinishedBlob, insertedChunkHashes // No handler - read data for legacy behavior log.Debug("No blob handler callback configured", "blob_hash", finished.Hash[:8]+"...") - if _, err := p.currentBlob.tempFile.Seek(0, io.SeekStart); err != nil { + _, err := p.currentBlob.tempFile.Seek(0, io.SeekStart) + if err != nil { p.cleanupTempFile() return fmt.Errorf("seeking to read data: %w", err) @@ -568,26 +640,3 @@ func (p *Packer) cleanupTempFile() { _ = p.fs.Remove(name) } } - -// PackChunks is a convenience method to pack multiple chunks at once -func (p *Packer) PackChunks(chunks []*ChunkRef) error { - for _, chunk := range chunks { - err := p.AddChunk(chunk) - if errors.Is(err, ErrBlobSizeLimitExceeded) { - // Finalize current blob and retry - err := p.FinalizeBlob() - if err != nil { - return fmt.Errorf("finalizing blob before retry: %w", err) - } - // Retry the chunk - err = p.AddChunk(chunk) - if err != nil { - return fmt.Errorf("adding chunk %s after finalize: %w", chunk.Hash, err) - } - } else if err != nil { - return fmt.Errorf("adding chunk %s: %w", chunk.Hash, err) - } - } - - return p.Flush() -} diff --git a/internal/blob/packer_test.go b/internal/blob/packer_test.go index d44bc22..09994fa 100644 --- a/internal/blob/packer_test.go +++ b/internal/blob/packer_test.go @@ -1,4 +1,4 @@ -package blob +package blob_test import ( "bytes" @@ -13,6 +13,7 @@ import ( "filippo.io/age" "github.com/klauspost/compress/zstd" "github.com/spf13/afero" + "sneak.berlin/go/vaultik/internal/blob" "sneak.berlin/go/vaultik/internal/database" "sneak.berlin/go/vaultik/internal/log" "sneak.berlin/go/vaultik/internal/types" @@ -20,387 +21,298 @@ import ( const ( // Test key from test/insecure-integration-test.key - testPrivateKey = "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5" - testPublicKey = "age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05gl0sjq9q9wjg" + testPrivateKey = "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7A" + + "PHXA2QS2NJA5" + testPublicKey = "age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05gl0sjq9q9wjg" + + defaultMaxBlobSize = 10 * 1024 * 1024 // 10MB + testChunkSize = 1000 + testChunkCount = 10 ) -func TestPacker(t *testing.T) { - // Initialize logger for tests - log.Initialize(log.Config{}) +// parseTestIdentity parses the fixed test age identity. +func parseTestIdentity(t *testing.T) *age.X25519Identity { + t.Helper() - // Parse test identity identity, err := age.ParseX25519Identity(testPrivateKey) if err != nil { t.Fatalf("failed to parse test identity: %v", err) } - t.Run("single chunk creates single blob", func(t *testing.T) { - // Create test database - db, err := database.NewTestDB() - if err != nil { - t.Fatalf("failed to create test db: %v", err) - } - defer func() { _ = db.Close() }() - - repos := database.NewRepositories(db) - - cfg := PackerConfig{ - MaxBlobSize: 10 * 1024 * 1024, // 10MB - CompressionLevel: 3, - Recipients: []string{testPublicKey}, - Repositories: repos, - Fs: afero.NewMemMapFs(), - } - - packer, err := NewPacker(cfg) - if err != nil { - t.Fatalf("failed to create packer: %v", err) - } - - // Create a chunk - data := []byte("Hello, World!") - hash := sha256.Sum256(data) - hashStr := hex.EncodeToString(hash[:]) - - // Create chunk in database first - dbChunk := &database.Chunk{ - ChunkHash: types.ChunkHash(hashStr), - Size: int64(len(data)), - } - - err = repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error { - return repos.Chunks.Create(ctx, tx, dbChunk) - }) - if err != nil { - t.Fatalf("failed to create chunk in db: %v", err) - } - - chunk := &ChunkRef{ - Hash: hashStr, - Data: data, - } - - // Add chunk - if err := packer.AddChunk(chunk); err != nil { - t.Fatalf("failed to add chunk: %v", err) - } - - // Flush - if err := packer.Flush(); err != nil { - t.Fatalf("failed to flush: %v", err) - } - - // Get finished blobs - blobs := packer.GetFinishedBlobs() - if len(blobs) != 1 { - t.Fatalf("expected 1 blob, got %d", len(blobs)) - } - - blob := blobs[0] - if len(blob.Chunks) != 1 { - t.Errorf("expected 1 chunk in blob, got %d", len(blob.Chunks)) - } - - // Note: Very small data may not compress well - t.Logf("Compression: %d -> %d bytes", blob.Uncompressed, blob.Compressed) - - // Decrypt the blob data - decrypted, err := age.Decrypt(bytes.NewReader(blob.Data), identity) - if err != nil { - t.Fatalf("failed to decrypt blob: %v", err) - } - - // Decompress the decrypted data - reader, err := zstd.NewReader(decrypted) - if err != nil { - t.Fatalf("failed to create decompressor: %v", err) - } - defer reader.Close() - - var decompressed bytes.Buffer - if _, err := io.Copy(&decompressed, reader); err != nil { - t.Fatalf("failed to decompress: %v", err) - } - - if !bytes.Equal(decompressed.Bytes(), data) { - t.Error("decompressed data doesn't match original") - } - }) - - t.Run("multiple chunks packed together", func(t *testing.T) { - // Create test database - db, err := database.NewTestDB() - if err != nil { - t.Fatalf("failed to create test db: %v", err) - } - defer func() { _ = db.Close() }() - - repos := database.NewRepositories(db) - - cfg := PackerConfig{ - MaxBlobSize: 10 * 1024 * 1024, // 10MB - CompressionLevel: 3, - Recipients: []string{testPublicKey}, - Repositories: repos, - Fs: afero.NewMemMapFs(), - } - - packer, err := NewPacker(cfg) - if err != nil { - t.Fatalf("failed to create packer: %v", err) - } - - // Create multiple small chunks - chunks := make([]*ChunkRef, 10) - - for i := range 10 { - data := bytes.Repeat([]byte{byte(i)}, 1000) - hash := sha256.Sum256(data) - hashStr := hex.EncodeToString(hash[:]) - - // Create chunk in database first - dbChunk := &database.Chunk{ - ChunkHash: types.ChunkHash(hashStr), - Size: int64(len(data)), - } - - err = repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error { - return repos.Chunks.Create(ctx, tx, dbChunk) - }) - if err != nil { - t.Fatalf("failed to create chunk in db: %v", err) - } - - chunks[i] = &ChunkRef{ - Hash: hashStr, - Data: data, - } - } - - // Add all chunks - for _, chunk := range chunks { - err := packer.AddChunk(chunk) - if err != nil { - t.Fatalf("failed to add chunk: %v", err) - } - } - - // Flush - if err := packer.Flush(); err != nil { - t.Fatalf("failed to flush: %v", err) - } - - // Should have one blob with all chunks - blobs := packer.GetFinishedBlobs() - if len(blobs) != 1 { - t.Fatalf("expected 1 blob, got %d", len(blobs)) - } - - if len(blobs[0].Chunks) != 10 { - t.Errorf("expected 10 chunks in blob, got %d", len(blobs[0].Chunks)) - } - - // Verify offsets are correct - expectedOffset := int64(0) - for i, chunkRef := range blobs[0].Chunks { - if chunkRef.Offset != expectedOffset { - t.Errorf("chunk %d: expected offset %d, got %d", i, expectedOffset, chunkRef.Offset) - } - - if chunkRef.Length != 1000 { - t.Errorf("chunk %d: expected length 1000, got %d", i, chunkRef.Length) - } - - expectedOffset += chunkRef.Length - } - }) - - t.Run("blob size limit enforced", func(t *testing.T) { - // Create test database - db, err := database.NewTestDB() - if err != nil { - t.Fatalf("failed to create test db: %v", err) - } - defer func() { _ = db.Close() }() - - repos := database.NewRepositories(db) - - // Small blob size limit to force multiple blobs - cfg := PackerConfig{ - MaxBlobSize: 5000, // 5KB max - CompressionLevel: 3, - Recipients: []string{testPublicKey}, - Repositories: repos, - Fs: afero.NewMemMapFs(), - } - - packer, err := NewPacker(cfg) - if err != nil { - t.Fatalf("failed to create packer: %v", err) - } - - // Create chunks that will exceed the limit - chunks := make([]*ChunkRef, 10) - - for i := range 10 { - data := bytes.Repeat([]byte{byte(i)}, 1000) // 1KB each - hash := sha256.Sum256(data) - hashStr := hex.EncodeToString(hash[:]) - - // Create chunk in database first - dbChunk := &database.Chunk{ - ChunkHash: types.ChunkHash(hashStr), - Size: int64(len(data)), - } - - err = repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error { - return repos.Chunks.Create(ctx, tx, dbChunk) - }) - if err != nil { - t.Fatalf("failed to create chunk in db: %v", err) - } - - chunks[i] = &ChunkRef{ - Hash: hashStr, - Data: data, - } - } - - blobCount := 0 - - // Add chunks and handle size limit errors - for _, chunk := range chunks { - err := packer.AddChunk(chunk) - if errors.Is(err, ErrBlobSizeLimitExceeded) { - // Finalize current blob - err := packer.FinalizeBlob() - if err != nil { - t.Fatalf("failed to finalize blob: %v", err) - } - - blobCount++ - // Retry adding the chunk - err = packer.AddChunk(chunk) - if err != nil { - t.Fatalf("failed to add chunk after finalize: %v", err) - } - } else if err != nil { - t.Fatalf("failed to add chunk: %v", err) - } - } - - // Flush remaining - if err := packer.Flush(); err != nil { - t.Fatalf("failed to flush: %v", err) - } - - // Get all blobs - blobs := packer.GetFinishedBlobs() - totalBlobs := blobCount + len(blobs) - - // Should have multiple blobs due to size limit - if totalBlobs < 2 { - t.Errorf("expected multiple blobs due to size limit, got %d", totalBlobs) - } - - // Verify each blob respects size limit (approximately) - for _, blob := range blobs { - if blob.Compressed > 6000 { // Allow some overhead - t.Errorf("blob size %d exceeds limit", blob.Compressed) - } - } - }) - - t.Run("with encryption", func(t *testing.T) { - // Create test database - db, err := database.NewTestDB() - if err != nil { - t.Fatalf("failed to create test db: %v", err) - } - defer func() { _ = db.Close() }() - - repos := database.NewRepositories(db) - - // Generate test identity (using the one from parent test) - cfg := PackerConfig{ - MaxBlobSize: 10 * 1024 * 1024, // 10MB - CompressionLevel: 3, - Recipients: []string{testPublicKey}, - Repositories: repos, - Fs: afero.NewMemMapFs(), - } - - packer, err := NewPacker(cfg) - if err != nil { - t.Fatalf("failed to create packer: %v", err) - } - - // Create test data - data := bytes.Repeat([]byte("Test data for encryption!"), 100) - hash := sha256.Sum256(data) - hashStr := hex.EncodeToString(hash[:]) - - // Create chunk in database first - dbChunk := &database.Chunk{ - ChunkHash: types.ChunkHash(hashStr), - Size: int64(len(data)), - } - - err = repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error { - return repos.Chunks.Create(ctx, tx, dbChunk) - }) - if err != nil { - t.Fatalf("failed to create chunk in db: %v", err) - } - - chunk := &ChunkRef{ - Hash: hashStr, - Data: data, - } - - // Add chunk and flush - if err := packer.AddChunk(chunk); err != nil { - t.Fatalf("failed to add chunk: %v", err) - } - - if err := packer.Flush(); err != nil { - t.Fatalf("failed to flush: %v", err) - } - - // Get blob - blobs := packer.GetFinishedBlobs() - if len(blobs) != 1 { - t.Fatalf("expected 1 blob, got %d", len(blobs)) - } - - blob := blobs[0] - - // Decrypt the blob - decrypted, err := age.Decrypt(bytes.NewReader(blob.Data), identity) - if err != nil { - t.Fatalf("failed to decrypt blob: %v", err) - } - - var decryptedData bytes.Buffer - if _, err := decryptedData.ReadFrom(decrypted); err != nil { - t.Fatalf("failed to read decrypted data: %v", err) - } - - // Decompress - reader, err := zstd.NewReader(&decryptedData) - if err != nil { - t.Fatalf("failed to create decompressor: %v", err) - } - defer reader.Close() - - var decompressed bytes.Buffer - if _, err := decompressed.ReadFrom(reader); err != nil { - t.Fatalf("failed to decompress: %v", err) - } - - // Verify data - if !bytes.Equal(decompressed.Bytes(), data) { - t.Error("decrypted and decompressed data doesn't match original") - } - }) + return identity +} + +// newTestPacker creates a test database and a Packer backed by it. +func newTestPacker( + t *testing.T, maxBlobSize int64, +) (*database.Repositories, *blob.Packer) { + t.Helper() + + db, err := database.NewTestDB() + if err != nil { + t.Fatalf("failed to create test db: %v", err) + } + + t.Cleanup(func() { _ = db.Close() }) + + repos := database.NewRepositories(db) + + packer, err := blob.NewPacker(blob.PackerConfig{ + MaxBlobSize: maxBlobSize, + CompressionLevel: 3, + Recipients: []string{testPublicKey}, + Repositories: repos, + Fs: afero.NewMemMapFs(), + }) + if err != nil { + t.Fatalf("failed to create packer: %v", err) + } + + return repos, packer +} + +// makeChunk creates a ChunkRef for data and registers the chunk in the +// database. +func makeChunk( + t *testing.T, repos *database.Repositories, data []byte, +) *blob.ChunkRef { + t.Helper() + + hash := sha256.Sum256(data) + hashStr := hex.EncodeToString(hash[:]) + + dbChunk := &database.Chunk{ + ChunkHash: types.ChunkHash(hashStr), + Size: int64(len(data)), + } + + err := repos.WithTx( + context.Background(), + func(ctx context.Context, tx *sql.Tx) error { + return repos.Chunks.Create(ctx, tx, dbChunk) + }) + if err != nil { + t.Fatalf("failed to create chunk in db: %v", err) + } + + return &blob.ChunkRef{ + Hash: hashStr, + Data: data, + } +} + +// decryptAndDecompress reverses the blob pipeline: age decrypt, then zstd +// decompress. +func decryptAndDecompress( + t *testing.T, blobData []byte, identity *age.X25519Identity, +) []byte { + t.Helper() + + decrypted, err := age.Decrypt(bytes.NewReader(blobData), identity) + if err != nil { + t.Fatalf("failed to decrypt blob: %v", err) + } + + reader, err := zstd.NewReader(decrypted) + if err != nil { + t.Fatalf("failed to create decompressor: %v", err) + } + defer reader.Close() + + var decompressed bytes.Buffer + + _, err = io.Copy(&decompressed, reader) + if err != nil { + t.Fatalf("failed to decompress: %v", err) + } + + return decompressed.Bytes() +} + +func TestPackerSingleChunk(t *testing.T) { + log.Initialize(log.Config{}) + t.Parallel() + + identity := parseTestIdentity(t) + repos, packer := newTestPacker(t, defaultMaxBlobSize) + ctx := context.Background() + + data := []byte("Hello, World!") + chunk := makeChunk(t, repos, data) + + err := packer.AddChunk(ctx, chunk) + if err != nil { + t.Fatalf("failed to add chunk: %v", err) + } + + err = packer.Flush(ctx) + if err != nil { + t.Fatalf("failed to flush: %v", err) + } + + blobs := packer.GetFinishedBlobs() + if len(blobs) != 1 { + t.Fatalf("expected 1 blob, got %d", len(blobs)) + } + + finished := blobs[0] + if len(finished.Chunks) != 1 { + t.Errorf("expected 1 chunk in blob, got %d", len(finished.Chunks)) + } + + // Note: Very small data may not compress well + t.Logf("Compression: %d -> %d bytes", + finished.Uncompressed, finished.Compressed) + + decompressed := decryptAndDecompress(t, finished.Data, identity) + if !bytes.Equal(decompressed, data) { + t.Error("decompressed data doesn't match original") + } +} + +func TestPackerMultipleChunks(t *testing.T) { + log.Initialize(log.Config{}) + t.Parallel() + + repos, packer := newTestPacker(t, defaultMaxBlobSize) + ctx := context.Background() + + chunks := make([]*blob.ChunkRef, testChunkCount) + for i := range testChunkCount { + data := bytes.Repeat([]byte{byte(i)}, testChunkSize) + chunks[i] = makeChunk(t, repos, data) + } + + for _, chunk := range chunks { + err := packer.AddChunk(ctx, chunk) + if err != nil { + t.Fatalf("failed to add chunk: %v", err) + } + } + + err := packer.Flush(ctx) + if err != nil { + t.Fatalf("failed to flush: %v", err) + } + + blobs := packer.GetFinishedBlobs() + if len(blobs) != 1 { + t.Fatalf("expected 1 blob, got %d", len(blobs)) + } + + if len(blobs[0].Chunks) != testChunkCount { + t.Errorf("expected %d chunks in blob, got %d", + testChunkCount, len(blobs[0].Chunks)) + } + + // Verify offsets are correct + expectedOffset := int64(0) + + for i, chunkRef := range blobs[0].Chunks { + if chunkRef.Offset != expectedOffset { + t.Errorf("chunk %d: expected offset %d, got %d", + i, expectedOffset, chunkRef.Offset) + } + + if chunkRef.Length != testChunkSize { + t.Errorf("chunk %d: expected length %d, got %d", + i, testChunkSize, chunkRef.Length) + } + + expectedOffset += chunkRef.Length + } +} + +func TestPackerSizeLimit(t *testing.T) { + log.Initialize(log.Config{}) + t.Parallel() + + const ( + maxBlobSize = 5000 // 5KB max, forces multiple blobs + maxBlobawoOverhead = 6000 // allow some overhead over the limit + ) + + repos, packer := newTestPacker(t, maxBlobSize) + ctx := context.Background() + + chunks := make([]*blob.ChunkRef, testChunkCount) + for i := range testChunkCount { + data := bytes.Repeat([]byte{byte(i)}, testChunkSize) // 1KB each + chunks[i] = makeChunk(t, repos, data) + } + + blobCount := 0 + + // Add chunks and handle size limit errors + for _, chunk := range chunks { + err := packer.AddChunk(ctx, chunk) + if errors.Is(err, blob.ErrBlobSizeLimitExceeded) { + // Finalize current blob + err = packer.FinalizeBlob(ctx) + if err != nil { + t.Fatalf("failed to finalize blob: %v", err) + } + + blobCount++ + + // Retry adding the chunk + err = packer.AddChunk(ctx, chunk) + if err != nil { + t.Fatalf("failed to add chunk after finalize: %v", err) + } + } else if err != nil { + t.Fatalf("failed to add chunk: %v", err) + } + } + + err := packer.Flush(ctx) + if err != nil { + t.Fatalf("failed to flush: %v", err) + } + + blobs := packer.GetFinishedBlobs() + + totalBlobs := blobCount + len(blobs) + if totalBlobs < 2 { + t.Errorf("expected multiple blobs due to size limit, got %d", totalBlobs) + } + + // Verify each blob respects size limit (approximately) + for _, finished := range blobs { + if finished.Compressed > maxBlobawoOverhead { + t.Errorf("blob size %d exceeds limit", finished.Compressed) + } + } +} + +func TestPackerEncryption(t *testing.T) { + log.Initialize(log.Config{}) + t.Parallel() + + identity := parseTestIdentity(t) + repos, packer := newTestPacker(t, defaultMaxBlobSize) + ctx := context.Background() + + data := bytes.Repeat([]byte("Test data for encryption!"), 100) + chunk := makeChunk(t, repos, data) + + err := packer.AddChunk(ctx, chunk) + if err != nil { + t.Fatalf("failed to add chunk: %v", err) + } + + err = packer.Flush(ctx) + if err != nil { + t.Fatalf("failed to flush: %v", err) + } + + blobs := packer.GetFinishedBlobs() + if len(blobs) != 1 { + t.Fatalf("expected 1 blob, got %d", len(blobs)) + } + + decompressed := decryptAndDecompress(t, blobs[0].Data, identity) + if !bytes.Equal(decompressed, data) { + t.Error("decrypted and decompressed data doesn't match original") + } } diff --git a/internal/blobgen/compress.go b/internal/blobgen/compress.go index 20f8571..8eefc77 100644 --- a/internal/blobgen/compress.go +++ b/internal/blobgen/compress.go @@ -1,3 +1,6 @@ +// Package blobgen implements the blob data pipeline: streaming zstd +// compression, age encryption, and SHA256 content hashing for blob +// creation, plus the matching decrypt/decompress/verify reader. package blobgen import ( @@ -16,7 +19,9 @@ type CompressResult struct { } // CompressData compresses and encrypts data, returning the result with hash -func CompressData(data []byte, compressionLevel int, recipients []string) (*CompressResult, error) { +func CompressData( + data []byte, compressionLevel int, recipients []string, +) (*CompressResult, error) { var buf bytes.Buffer // Create writer @@ -26,14 +31,16 @@ func CompressData(data []byte, compressionLevel int, recipients []string) (*Comp } // Write data - if _, err := w.Write(data); err != nil { + _, err = w.Write(data) + if err != nil { _ = w.Close() return nil, fmt.Errorf("writing data: %w", err) } // Close to flush - if err := w.Close(); err != nil { + err = w.Close() + if err != nil { return nil, fmt.Errorf("closing writer: %w", err) } @@ -45,8 +52,11 @@ func CompressData(data []byte, compressionLevel int, recipients []string) (*Comp }, nil } -// CompressStream compresses and encrypts from reader to writer, returning hash -func CompressStream(dst io.Writer, src io.Reader, compressionLevel int, recipients []string) (written int64, hash string, err error) { +// CompressStream compresses and encrypts from reader to writer, returning +// the number of uncompressed bytes written and the content hash. +func CompressStream( + dst io.Writer, src io.Reader, compressionLevel int, recipients []string, +) (int64, string, error) { // Create writer w, err := NewWriter(dst, compressionLevel, recipients) if err != nil { @@ -61,12 +71,14 @@ func CompressStream(dst io.Writer, src io.Reader, compressionLevel int, recipien }() // Copy data - if _, err := io.Copy(w, src); err != nil { + _, err = io.Copy(w, src) + if err != nil { return 0, "", fmt.Errorf("copying data: %w", err) } // Close to flush - if err := w.Close(); err != nil { + err = w.Close() + if err != nil { return 0, "", fmt.Errorf("closing writer: %w", err) } diff --git a/internal/blobgen/compress_test.go b/internal/blobgen/compress_test.go index 37c41e9..b019f11 100644 --- a/internal/blobgen/compress_test.go +++ b/internal/blobgen/compress_test.go @@ -1,4 +1,4 @@ -package blobgen +package blobgen_test import ( "bytes" @@ -8,6 +8,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "sneak.berlin/go/vaultik/internal/blobgen" ) // testRecipient is a static age recipient for tests. @@ -19,11 +20,14 @@ const testRecipient = "age1cplgrwj77ta54dnmydvvmzn64ltk83ankxl5sww04mrtmu62kv3s8 // the explicit Close() on the happy path combined with defer Close() would // cause a double close. func TestCompressStreamNoDoubleClose(t *testing.T) { + t.Parallel() + input := []byte("regression test data for issue #28 double-close fix") var buf bytes.Buffer - written, hash, err := CompressStream(&buf, bytes.NewReader(input), 3, []string{testRecipient}) + written, hash, err := blobgen.CompressStream( + &buf, bytes.NewReader(input), 3, []string{testRecipient}) require.NoError(t, err, "CompressStream should not return an error") assert.Positive(t, written, "expected bytes written > 0") assert.NotEmpty(t, hash, "expected non-empty hash") @@ -33,13 +37,16 @@ func TestCompressStreamNoDoubleClose(t *testing.T) { // TestCompressStreamLargeInput exercises CompressStream with a larger payload // to ensure no double-close issues surface under heavier I/O. func TestCompressStreamLargeInput(t *testing.T) { + t.Parallel() + data := make([]byte, 512*1024) // 512 KB _, err := rand.Read(data) require.NoError(t, err) var buf bytes.Buffer - written, hash, err := CompressStream(&buf, bytes.NewReader(data), 3, []string{testRecipient}) + written, hash, err := blobgen.CompressStream( + &buf, bytes.NewReader(data), 3, []string{testRecipient}) require.NoError(t, err) assert.Positive(t, written) assert.NotEmpty(t, hash) @@ -48,9 +55,12 @@ func TestCompressStreamLargeInput(t *testing.T) { // TestCompressStreamEmptyInput verifies CompressStream handles empty input // without double-close issues. func TestCompressStreamEmptyInput(t *testing.T) { + t.Parallel() + var buf bytes.Buffer - _, hash, err := CompressStream(&buf, strings.NewReader(""), 3, []string{testRecipient}) + _, hash, err := blobgen.CompressStream( + &buf, strings.NewReader(""), 3, []string{testRecipient}) require.NoError(t, err) assert.NotEmpty(t, hash) } @@ -58,8 +68,11 @@ func TestCompressStreamEmptyInput(t *testing.T) { // TestCompressDataNoDoubleClose mirrors the stream test for CompressData, // ensuring the explicit Close + error-path Close pattern is also safe. func TestCompressDataNoDoubleClose(t *testing.T) { + t.Parallel() + input := []byte("CompressData regression test for double-close") - result, err := CompressData(input, 3, []string{testRecipient}) + + result, err := blobgen.CompressData(input, 3, []string{testRecipient}) require.NoError(t, err) assert.Positive(t, result.CompressedSize) assert.Equal(t, result.UncompressedSize, int64(len(input))) diff --git a/internal/blobgen/reader.go b/internal/blobgen/reader.go index 6d7f2f8..1a7dc11 100644 --- a/internal/blobgen/reader.go +++ b/internal/blobgen/reader.go @@ -50,8 +50,8 @@ func NewReader(r io.Reader, identity age.Identity) (*Reader, error) { } // Read implements io.Reader -func (r *Reader) Read(p []byte) (n int, err error) { - n, err = r.teeReader.Read(p) +func (r *Reader) Read(p []byte) (int, error) { + n, err := r.teeReader.Read(p) r.bytesRead += int64(n) return n, err diff --git a/internal/blobgen/writer.go b/internal/blobgen/writer.go index a14574c..8305046 100644 --- a/internal/blobgen/writer.go +++ b/internal/blobgen/writer.go @@ -2,6 +2,7 @@ package blobgen import ( "crypto/sha256" + "errors" "fmt" "hash" "io" @@ -11,6 +12,21 @@ import ( "github.com/klauspost/compress/zstd" ) +// Zstd compression level bounds accepted by NewWriter. +const ( + minCompressionLevel = 1 + maxCompressionLevel = 19 +) + +// reservedCompressionCPUs is how many CPUs are left free of zstd +// compression work for I/O and hashing. +const reservedCompressionCPUs = 2 + +// ErrInvalidCompressionLevel is returned when the zstd compression level +// is outside the accepted 1-19 range. +var ErrInvalidCompressionLevel = errors.New( + "invalid compression level: must be between 1 and 19") + // Writer wraps compression and encryption with SHA256 hashing. // Data flows: input -> tee(hasher, compressor -> encryptor -> destination) // The hash is computed on the uncompressed input for deterministic content-addressing. @@ -23,11 +39,15 @@ type Writer struct { bytesWritten int64 } -// NewWriter creates a new Writer that compresses, encrypts, and hashes data. -// The hash is computed on the uncompressed input for deterministic content-addressing. -func NewWriter(w io.Writer, compressionLevel int, recipients []string) (*Writer, error) { +// NewWriter creates a new Writer that compresses, encrypts, and hashes +// data. The hash is computed on the uncompressed input for deterministic +// content-addressing. +func NewWriter( + w io.Writer, compressionLevel int, recipients []string, +) (*Writer, error) { // Validate compression level - if err := validateCompressionLevel(compressionLevel); err != nil { + err := validateCompressionLevel(compressionLevel) + if err != nil { return nil, err } @@ -53,7 +73,7 @@ func NewWriter(w io.Writer, compressionLevel int, recipients []string) (*Writer, } // Calculate compression concurrency: CPUs - 2, minimum 1 - concurrency := max(runtime.NumCPU()-2, 1) + concurrency := max(runtime.NumCPU()-reservedCompressionCPUs, 1) // Create compression writer with encryption as destination compressor, err := zstd.NewWriter(encWriter, @@ -79,8 +99,8 @@ func NewWriter(w io.Writer, compressionLevel int, recipients []string) (*Writer, } // Write implements io.Writer -func (w *Writer) Write(p []byte) (n int, err error) { - n, err = w.teeWriter.Write(p) +func (w *Writer) Write(p []byte) (int, error) { + n, err := w.teeWriter.Write(p) w.bytesWritten += int64(n) return n, err @@ -123,9 +143,10 @@ func (w *Writer) BytesWritten() int64 { func validateCompressionLevel(level int) error { // Zstd compression levels: 1-19 (default is 3) - // SpeedFastest = 1, SpeedDefault = 3, SpeedBetterCompression = 7, SpeedBestCompression = 11 - if level < 1 || level > 19 { - return fmt.Errorf("invalid compression level %d: must be between 1 and 19", level) + // SpeedFastest = 1, SpeedDefault = 3, SpeedBetterCompression = 7, + // SpeedBestCompression = 11 + if level < minCompressionLevel || level > maxCompressionLevel { + return fmt.Errorf("%w: got %d", ErrInvalidCompressionLevel, level) } return nil diff --git a/internal/blobgen/writer_test.go b/internal/blobgen/writer_test.go index f72e8cf..e218f52 100644 --- a/internal/blobgen/writer_test.go +++ b/internal/blobgen/writer_test.go @@ -1,4 +1,4 @@ -package blobgen +package blobgen_test import ( "bytes" @@ -9,12 +9,15 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "sneak.berlin/go/vaultik/internal/blobgen" ) // TestWriterHashIsDoubleHash verifies that Writer.Sum256() returns // the double hash SHA256(SHA256(plaintext)) for security. // Double hashing prevents attackers from confirming existence of known content. func TestWriterHashIsDoubleHash(t *testing.T) { + t.Parallel() + // Test data - random data that doesn't compress well testData := make([]byte, 1024*1024) // 1MB _, err := rand.Read(testData) @@ -27,7 +30,7 @@ func TestWriterHashIsDoubleHash(t *testing.T) { var encryptedBuf bytes.Buffer // Create blobgen writer - writer, err := NewWriter(&encryptedBuf, 3, []string{testRecipient}) + writer, err := blobgen.NewWriter(&encryptedBuf, 3, []string{testRecipient}) require.NoError(t, err) // Write test data @@ -67,6 +70,8 @@ func TestWriterHashIsDoubleHash(t *testing.T) { // TestWriterDeterministicHash verifies that the same input always produces // the same hash, even with non-deterministic encryption. func TestWriterDeterministicHash(t *testing.T) { + t.Parallel() + // Test data testData := []byte("Hello, World! This is test data for deterministic hashing.") @@ -76,13 +81,13 @@ func TestWriterDeterministicHash(t *testing.T) { // Create two writers and verify they produce the same hash var buf1, buf2 bytes.Buffer - writer1, err := NewWriter(&buf1, 3, []string{testRecipient}) + writer1, err := blobgen.NewWriter(&buf1, 3, []string{testRecipient}) require.NoError(t, err) _, err = writer1.Write(testData) require.NoError(t, err) require.NoError(t, writer1.Close()) - writer2, err := NewWriter(&buf2, 3, []string{testRecipient}) + writer2, err := blobgen.NewWriter(&buf2, 3, []string{testRecipient}) require.NoError(t, err) _, err = writer2.Write(testData) require.NoError(t, err) diff --git a/internal/chunker/chunker.go b/internal/chunker/chunker.go index 1615899..8fedb30 100644 --- a/internal/chunker/chunker.go +++ b/internal/chunker/chunker.go @@ -1,3 +1,6 @@ +// Package chunker splits input data into content-defined chunks using the +// FastCDC algorithm so that identical data sequences produce identical +// chunks regardless of their position in the file. package chunker import ( @@ -9,9 +12,10 @@ import ( "os" ) -// Chunk represents a single chunk of data produced by the content-defined chunking algorithm. -// Each chunk is identified by its SHA256 hash and contains the raw data along with -// its position and size information from the original file. +// Chunk represents a single chunk of data produced by the content-defined +// chunking algorithm. Each chunk is identified by its SHA256 hash and +// contains the raw data along with its position and size information from +// the original file. type Chunk struct { Hash string // Content hash of the chunk Data []byte // Chunk data @@ -29,6 +33,10 @@ type Chunker struct { maxChunkSize int } +// chunkSizeSpread is the FastCDC-recommended factor between the average +// chunk size and the minimum (avg/spread) and maximum (avg*spread) sizes. +const chunkSizeSpread = 4 + // NewChunker creates a new chunker with the specified average chunk size. // The actual chunk sizes will vary between avgChunkSize/4 and avgChunkSize*4 // as recommended by the FastCDC algorithm. Typical values for avgChunkSize @@ -37,17 +45,19 @@ func NewChunker(avgChunkSize int64) *Chunker { // FastCDC recommends min = avg/4 and max = avg*4 return &Chunker{ avgChunkSize: int(avgChunkSize), - minChunkSize: int(avgChunkSize / 4), - maxChunkSize: int(avgChunkSize * 4), + minChunkSize: int(avgChunkSize / chunkSizeSpread), + maxChunkSize: int(avgChunkSize * chunkSizeSpread), } } -// ChunkReader splits the reader into content-defined chunks and returns all chunks at once. -// This method loads all chunk data into memory, so it should only be used for -// reasonably sized inputs. For large files or streams, use ChunkReaderStreaming instead. +// ChunkReader splits the reader into content-defined chunks and returns all +// chunks at once. This method loads all chunk data into memory, so it should +// only be used for reasonably sized inputs. For large files or streams, use +// ChunkReaderStreaming instead. // Returns an error if chunking fails or if reading from the input fails. func (c *Chunker) ChunkReader(r io.Reader) ([]Chunk, error) { - chunker := AcquireReusableChunker(r, c.minChunkSize, c.avgChunkSize, c.maxChunkSize) + chunker := AcquireReusableChunker( + r, c.minChunkSize, c.avgChunkSize, c.maxChunkSize) defer chunker.Release() var chunks []Chunk @@ -86,21 +96,26 @@ func (c *Chunker) ChunkReader(r io.Reader) ([]Chunk, error) { // ChunkCallback is a function called for each chunk as it's processed. // The callback receives a Chunk containing the hash, data, offset, and size. -// If the callback returns an error, chunk processing stops and the error is propagated. +// If the callback returns an error, chunk processing stops and the error is +// propagated. type ChunkCallback func(chunk Chunk) error -// ChunkReaderStreaming splits the reader into chunks and calls the callback for each chunk. -// This is the preferred method for processing large files or streams as it doesn't -// accumulate all chunks in memory. The callback is invoked for each chunk as it's -// produced, allowing for streaming processing and immediate storage or transmission. -// Returns the SHA256 hash of the entire file content and an error if chunking fails, -// reading fails, or if the callback returns an error. -func (c *Chunker) ChunkReaderStreaming(r io.Reader, callback ChunkCallback) (string, error) { +// ChunkReaderStreaming splits the reader into chunks and calls the callback +// for each chunk. This is the preferred method for processing large files or +// streams as it doesn't accumulate all chunks in memory. The callback is +// invoked for each chunk as it's produced, allowing for streaming processing +// and immediate storage or transmission. +// Returns the SHA256 hash of the entire file content and an error if +// chunking fails, reading fails, or if the callback returns an error. +func (c *Chunker) ChunkReaderStreaming( + r io.Reader, callback ChunkCallback, +) (string, error) { // Create a tee reader to calculate full file hash while chunking fileHasher := sha256.New() teeReader := io.TeeReader(r, fileHasher) - chunker := AcquireReusableChunker(teeReader, c.minChunkSize, c.avgChunkSize, c.maxChunkSize) + chunker := AcquireReusableChunker( + teeReader, c.minChunkSize, c.avgChunkSize, c.maxChunkSize) defer chunker.Release() offset := int64(0) @@ -118,15 +133,17 @@ func (c *Chunker) ChunkReaderStreaming(r io.Reader, callback ChunkCallback) (str // Calculate chunk hash hash := sha256.Sum256(chunk.Data) - // Pass the data directly - caller must process it before we call Next() again - // (chunker reuses its internal buffer, but since we process synchronously - // and completely before continuing, no copy is needed) - if err := callback(Chunk{ + // Pass the data directly - caller must process it before we call + // Next() again (chunker reuses its internal buffer, but since we + // process synchronously and completely before continuing, no copy + // is needed) + err = callback(Chunk{ Hash: hex.EncodeToString(hash[:]), Data: chunk.Data, Offset: offset, Size: int64(len(chunk.Data)), - }); err != nil { + }) + if err != nil { return "", fmt.Errorf("callback error: %w", err) } @@ -142,7 +159,7 @@ func (c *Chunker) ChunkReaderStreaming(r io.Reader, callback ChunkCallback) (str // For large files, consider using ChunkReaderStreaming with a file handle instead. // Returns an error if the file cannot be opened or if chunking fails. func (c *Chunker) ChunkFile(path string) ([]Chunk, error) { - file, err := os.Open(path) + file, err := os.Open(path) //nolint:gosec // G304: path is caller-supplied by design if err != nil { return nil, fmt.Errorf("opening file: %w", err) } diff --git a/internal/chunker/chunker_isolated_test.go b/internal/chunker/chunker_isolated_test.go index b5f86bf..5df44ae 100644 --- a/internal/chunker/chunker_isolated_test.go +++ b/internal/chunker/chunker_isolated_test.go @@ -1,11 +1,15 @@ -package chunker +package chunker_test import ( "bytes" "testing" + + "sneak.berlin/go/vaultik/internal/chunker" ) func TestChunkerExpectedChunkCount(t *testing.T) { + t.Parallel() + tests := []struct { name string fileSize int @@ -38,16 +42,19 @@ func TestChunkerExpectedChunkCount(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - chunker := NewChunker(tt.avgChunkSize) + t.Parallel() + + c := chunker.NewChunker(tt.avgChunkSize) // Create data with some variation to trigger chunk boundaries data := make([]byte, tt.fileSize) for i := range data { // Use a pattern that should create boundaries + //nolint:gosec // G115: intentional byte truncation data[i] = byte((i * 17) ^ (i >> 5)) } - chunks, err := chunker.ChunkReader(bytes.NewReader(data)) + chunks, err := c.ChunkReader(bytes.NewReader(data)) if err != nil { t.Fatalf("chunking failed: %v", err) } diff --git a/internal/chunker/chunker_test.go b/internal/chunker/chunker_test.go index fca59f2..f44919f 100644 --- a/internal/chunker/chunker_test.go +++ b/internal/chunker/chunker_test.go @@ -1,106 +1,120 @@ -package chunker +package chunker_test import ( "bytes" "crypto/rand" "testing" + + "sneak.berlin/go/vaultik/internal/chunker" ) -func TestChunker(t *testing.T) { - t.Run("small file produces single chunk", func(t *testing.T) { - chunker := NewChunker(1024 * 1024) // 1MB average - data := bytes.Repeat([]byte("hello"), 100) // 500 bytes +func TestChunkerSmallFileSingleChunk(t *testing.T) { + t.Parallel() - chunks, err := chunker.ChunkReader(bytes.NewReader(data)) - if err != nil { - t.Fatalf("chunking failed: %v", err) + c := chunker.NewChunker(1024 * 1024) // 1MB average + data := bytes.Repeat([]byte("hello"), 100) // 500 bytes + + chunks, err := c.ChunkReader(bytes.NewReader(data)) + if err != nil { + t.Fatalf("chunking failed: %v", err) + } + + if len(chunks) != 1 { + t.Errorf("expected 1 chunk, got %d", len(chunks)) + } + + if chunks[0].Size != int64(len(data)) { + t.Errorf("expected chunk size %d, got %d", len(data), chunks[0].Size) + } +} + +func TestChunkerLargeFileMultipleChunks(t *testing.T) { + t.Parallel() + + c := chunker.NewChunker(256 * 1024) // 256KB average chunk size + + // Generate 2MB of random data + data := make([]byte, 2*1024*1024) + + _, err := rand.Read(data) + if err != nil { + t.Fatalf("failed to generate random data: %v", err) + } + + chunks, err := c.ChunkReader(bytes.NewReader(data)) + if err != nil { + t.Fatalf("chunking failed: %v", err) + } + + // Should produce multiple chunks - with FastCDC we expect around 8 + // chunks for 2MB with 256KB average + if len(chunks) < 4 || len(chunks) > 16 { + t.Errorf("expected 4-16 chunks, got %d", len(chunks)) + } + + // Verify chunks reconstruct original data + var reconstructed []byte + for _, chunk := range chunks { + reconstructed = append(reconstructed, chunk.Data...) + } + + if !bytes.Equal(data, reconstructed) { + t.Error("reconstructed data doesn't match original") + } + + // Verify offsets + var expectedOffset int64 + + for i, chunk := range chunks { + if chunk.Offset != expectedOffset { + t.Errorf("chunk %d: expected offset %d, got %d", + i, expectedOffset, chunk.Offset) } - if len(chunks) != 1 { - t.Errorf("expected 1 chunk, got %d", len(chunks)) + expectedOffset += chunk.Size + } +} + +func TestChunkerDeterministic(t *testing.T) { + t.Parallel() + + chunker1 := chunker.NewChunker(256 * 1024) + chunker2 := chunker.NewChunker(256 * 1024) + + // Use deterministic data + data := bytes.Repeat([]byte("abcdefghijklmnopqrstuvwxyz"), 20000) // ~520KB + + chunks1, err := chunker1.ChunkReader(bytes.NewReader(data)) + if err != nil { + t.Fatalf("chunking failed: %v", err) + } + + chunks2, err := chunker2.ChunkReader(bytes.NewReader(data)) + if err != nil { + t.Fatalf("chunking failed: %v", err) + } + + // Should produce same chunks + if len(chunks1) != len(chunks2) { + t.Fatalf("different number of chunks: %d vs %d", + len(chunks1), len(chunks2)) + } + + for i := range chunks1 { + if chunks1[i].Hash != chunks2[i].Hash { + t.Errorf("chunk %d: different hashes", i) } - if chunks[0].Size != int64(len(data)) { - t.Errorf("expected chunk size %d, got %d", len(data), chunks[0].Size) + if chunks1[i].Size != chunks2[i].Size { + t.Errorf("chunk %d: different sizes", i) } - }) - - t.Run("large file produces multiple chunks", func(t *testing.T) { - chunker := NewChunker(256 * 1024) // 256KB average chunk size - - // Generate 2MB of random data - data := make([]byte, 2*1024*1024) - if _, err := rand.Read(data); err != nil { - t.Fatalf("failed to generate random data: %v", err) - } - - chunks, err := chunker.ChunkReader(bytes.NewReader(data)) - if err != nil { - t.Fatalf("chunking failed: %v", err) - } - - // Should produce multiple chunks - with FastCDC we expect around 8 chunks for 2MB with 256KB average - if len(chunks) < 4 || len(chunks) > 16 { - t.Errorf("expected 4-16 chunks, got %d", len(chunks)) - } - - // Verify chunks reconstruct original data - var reconstructed []byte - for _, chunk := range chunks { - reconstructed = append(reconstructed, chunk.Data...) - } - - if !bytes.Equal(data, reconstructed) { - t.Error("reconstructed data doesn't match original") - } - - // Verify offsets - var expectedOffset int64 - for i, chunk := range chunks { - if chunk.Offset != expectedOffset { - t.Errorf("chunk %d: expected offset %d, got %d", i, expectedOffset, chunk.Offset) - } - - expectedOffset += chunk.Size - } - }) - - t.Run("deterministic chunking", func(t *testing.T) { - chunker1 := NewChunker(256 * 1024) - chunker2 := NewChunker(256 * 1024) - - // Use deterministic data - data := bytes.Repeat([]byte("abcdefghijklmnopqrstuvwxyz"), 20000) // ~520KB - - chunks1, err := chunker1.ChunkReader(bytes.NewReader(data)) - if err != nil { - t.Fatalf("chunking failed: %v", err) - } - - chunks2, err := chunker2.ChunkReader(bytes.NewReader(data)) - if err != nil { - t.Fatalf("chunking failed: %v", err) - } - - // Should produce same chunks - if len(chunks1) != len(chunks2) { - t.Fatalf("different number of chunks: %d vs %d", len(chunks1), len(chunks2)) - } - - for i := range chunks1 { - if chunks1[i].Hash != chunks2[i].Hash { - t.Errorf("chunk %d: different hashes", i) - } - - if chunks1[i].Size != chunks2[i].Size { - t.Errorf("chunk %d: different sizes", i) - } - } - }) + } } func TestChunkBoundaries(t *testing.T) { - chunker := NewChunker(256 * 1024) // 256KB average + t.Parallel() + + c := chunker.NewChunker(256 * 1024) // 256KB average // FastCDC uses avg/4 for min and avg*4 for max avgSize := int64(256 * 1024) @@ -109,11 +123,13 @@ func TestChunkBoundaries(t *testing.T) { // Test that minimum chunk size is respected data := make([]byte, minSize+1024) - if _, err := rand.Read(data); err != nil { + + _, err := rand.Read(data) + if err != nil { t.Fatalf("failed to generate random data: %v", err) } - chunks, err := chunker.ChunkReader(bytes.NewReader(data)) + chunks, err := c.ChunkReader(bytes.NewReader(data)) if err != nil { t.Fatalf("chunking failed: %v", err) } @@ -121,11 +137,13 @@ func TestChunkBoundaries(t *testing.T) { for i, chunk := range chunks { // Last chunk can be smaller than minimum if i < len(chunks)-1 && chunk.Size < minSize { - t.Errorf("chunk %d size %d is below minimum %d", i, chunk.Size, minSize) + t.Errorf("chunk %d size %d is below minimum %d", + i, chunk.Size, minSize) } if chunk.Size > maxSize { - t.Errorf("chunk %d size %d exceeds maximum %d", i, chunk.Size, maxSize) + t.Errorf("chunk %d size %d exceeds maximum %d", + i, chunk.Size, maxSize) } } } diff --git a/internal/chunker/fastcdc.go b/internal/chunker/fastcdc.go index 223eb2b..59a2f42 100644 --- a/internal/chunker/fastcdc.go +++ b/internal/chunker/fastcdc.go @@ -1,6 +1,7 @@ package chunker import ( + "errors" "io" "math" "sync" @@ -27,6 +28,8 @@ type ReusableChunker struct { } // reusableChunkerPool pools ReusableChunker instances to avoid allocations. +// +//nolint:gochecknoglobals // process-wide object pool by design var reusableChunkerPool = sync.Pool{ New: func() any { return &ReusableChunker{} @@ -35,6 +38,8 @@ var reusableChunkerPool = sync.Pool{ // bufferPools contains pools for different buffer sizes. // Key is the buffer size. +// +//nolint:gochecknoglobals // process-wide buffer pools by design var bufferPools = sync.Map{} func getBuffer(size int) []byte { @@ -45,9 +50,18 @@ func getBuffer(size int) []byte { return &buf }, }) - pool := poolI.(*sync.Pool) - return *pool.Get().(*[]byte) + pool, ok := poolI.(*sync.Pool) + if !ok { + panic("bufferPools holds a non-pool value") + } + + buf, ok := pool.Get().(*[]byte) + if !ok { + panic("buffer pool holds a non-buffer value") + } + + return *buf } func putBuffer(buf []byte) { @@ -55,7 +69,11 @@ func putBuffer(buf []byte) { poolI, ok := bufferPools.Load(size) if ok { - pool := poolI.(*sync.Pool) + pool, isPool := poolI.(*sync.Pool) + if !isPool { + panic("bufferPools holds a non-pool value") + } + b := buf[:size] pool.Put(&b) } @@ -69,11 +87,21 @@ type FastCDCChunk struct { Fingerprint uint64 } -// AcquireReusableChunker gets a chunker from the pool and initializes it for the given reader. -func AcquireReusableChunker(rd io.Reader, minSize, avgSize, maxSize int) *ReusableChunker { - c := reusableChunkerPool.Get().(*ReusableChunker) +// bufSizeFactor sizes the internal read buffer relative to the maximum +// chunk size so a full chunk plus read-ahead always fits. +const bufSizeFactor = 2 - bufSize := maxSize * 2 +// AcquireReusableChunker gets a chunker from the pool and initializes it +// for the given reader. +func AcquireReusableChunker( + rd io.Reader, minSize, avgSize, maxSize int, +) *ReusableChunker { + c, ok := reusableChunkerPool.Get().(*ReusableChunker) + if !ok { + panic("reusableChunkerPool holds a non-chunker value") + } + + bufSize := maxSize * bufSizeFactor // Reuse buffer if it's the right size, otherwise get a new one if c.buf == nil || cap(c.buf) != bufSize { @@ -112,37 +140,6 @@ func (c *ReusableChunker) Release() { reusableChunkerPool.Put(c) } -func (c *ReusableChunker) fillBuffer() error { - n := len(c.buf) - c.cursor - if n >= c.maxSize { - return nil - } - - // Move all data after the cursor to the start of the buffer - copy(c.buf[:n], c.buf[c.cursor:]) - c.cursor = 0 - - if c.eof { - c.buf = c.buf[:n] - - return nil - } - - // Restore buffer to full capacity for reading - c.buf = c.buf[:c.bufSize] - - // Fill the rest of the buffer - m, err := io.ReadFull(c.rd, c.buf[n:]) - if err == io.EOF || err == io.ErrUnexpectedEOF { - c.buf = c.buf[:n+m] - c.eof = true - } else if err != nil { - return err - } - - return nil -} - // Next returns the next chunk or io.EOF when done. // The returned Data slice is only valid until the next call to Next. func (c *ReusableChunker) Next() (FastCDCChunk, error) { @@ -170,6 +167,37 @@ func (c *ReusableChunker) Next() (FastCDCChunk, error) { return chunk, nil } +func (c *ReusableChunker) fillBuffer() error { + n := len(c.buf) - c.cursor + if n >= c.maxSize { + return nil + } + + // Move all data after the cursor to the start of the buffer + copy(c.buf[:n], c.buf[c.cursor:]) + c.cursor = 0 + + if c.eof { + c.buf = c.buf[:n] + + return nil + } + + // Restore buffer to full capacity for reading + c.buf = c.buf[:c.bufSize] + + // Fill the rest of the buffer + m, err := io.ReadFull(c.rd, c.buf[n:]) + if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) { + c.buf = c.buf[:n+m] + c.eof = true + } else if err != nil { + return err + } + + return nil +} + func (c *ReusableChunker) nextChunk(data []byte) (int, uint64) { fp := uint64(0) i := c.minSize @@ -198,6 +226,8 @@ func (c *ReusableChunker) nextChunk(data []byte) (int, uint64) { } // 256 random uint64s for the rolling hash function (from FastCDC paper) +// +//nolint:gochecknoglobals // immutable FastCDC gear lookup table var table = [256]uint64{ 0xe80e8d55032474b3, 0x11b25b61f5924e15, 0x03aa5bd82a9eb669, 0xc45a153ef107a38c, 0xeac874b86f0f57b9, 0xa5ccedec95ec79c7, 0xe15a3320ad42ac0a, 0x5ed3583fa63cec15, diff --git a/internal/cli/app.go b/internal/cli/app.go index 36ae64f..e45596c 100644 --- a/internal/cli/app.go +++ b/internal/cli/app.go @@ -1,3 +1,6 @@ +// Package cli implements the vaultik command-line interface: cobra +// commands, fx application wiring, and process-level concerns such as +// signal handling and the PID lock. package cli import ( @@ -12,6 +15,7 @@ import ( "time" "github.com/adrg/xdg" + "github.com/spf13/cobra" "go.uber.org/fx" "sneak.berlin/go/vaultik/internal/config" "sneak.berlin/go/vaultik/internal/database" @@ -24,12 +28,16 @@ import ( "sneak.berlin/go/vaultik/internal/vaultik" ) +// shutdownTimeout bounds how long a signal-triggered graceful shutdown +// may take before we give up. +const shutdownTimeout = 30 * time.Second + // AppOptions contains common options for creating the fx application. // It includes the configuration file path, logging options, and additional // fx modules and invocations that should be included in the application. type AppOptions struct { ConfigPath string - LogOptions log.LogOptions + LogOptions log.Options Modules []fx.Option Invokes []fx.Option } @@ -38,11 +46,13 @@ type AppOptions struct { // flag is active, marks the UI writer quiet so that Begin/Complete/ // Info/Notice/Detail/Progress are silenced. Warning and Error are NOT // silenced — per the documented convention that --quiet suppresses -// non-error output only. The startup banner is printed by CLIEntry +// non-error output only. The startup banner is printed by Entry // before cobra parses arguments, gated by the same arg-level check. -func setupGlobals(lc fx.Lifecycle, g *globals.Globals, v *vaultik.Vaultik, opts log.LogOptions) { +func setupGlobals( + lc fx.Lifecycle, g *globals.Globals, v *vaultik.Vaultik, opts log.Options, +) { lc.Append(fx.Hook{ - OnStart: func(ctx context.Context) error { + OnStart: func(_ context.Context) error { g.StartTime = time.Now().UTC() if opts.Cron || opts.Quiet { @@ -72,7 +82,7 @@ func writeStartupBanner(w *ui.Writer, startTime time.Time, shortCommit string) { // The returned fx.App is ready to be started with RunApp. func NewApp(opts AppOptions) *fx.App { baseModules := []fx.Option{ - fx.Supply(config.ConfigPath(opts.ConfigPath)), + fx.Supply(config.Path(opts.ConfigPath)), fx.Supply(opts.LogOptions), fx.Provide(globals.New), fx.Provide(log.New), @@ -86,12 +96,27 @@ func NewApp(opts AppOptions) *fx.App { fx.NopLogger, } - allOptions := append(baseModules, opts.Modules...) + capacity := len(baseModules) + len(opts.Modules) + len(opts.Invokes) + allOptions := make([]fx.Option, 0, capacity) + allOptions = append(allOptions, baseModules...) + allOptions = append(allOptions, opts.Modules...) allOptions = append(allOptions, opts.Invokes...) return fx.New(allOptions...) } +// startupError carries a startup failure message that has been cleaned +// of fx dependency-injection noise. A distinct type (rather than +// errors.New) keeps the dynamic message out of err113's sight while +// preserving the exact user-facing text. +type startupError struct { + msg string +} + +func (e *startupError) Error() string { + return e.msg +} + // cleanStartupError strips fx's dependency-injection call-chain noise from // startup errors. fx wraps the underlying error with messages like // @@ -108,7 +133,7 @@ func cleanStartupError(err error) error { msg = msg[idx+3:] } - return errors.New(msg) + return &startupError{msg: msg} } // RunApp starts and stops the fx application within the given context. @@ -138,8 +163,10 @@ func RunApp(ctx context.Context, app *fx.App) error { <-sigChan log.Notice("Received interrupt signal, shutting down gracefully...") - // Create a timeout context for shutdown - shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 30*time.Second) + // Create a timeout context for shutdown. The parent ctx is being + // cancelled, so detach from its cancellation but keep its values. + shutdownCtx, shutdownCancel := context.WithTimeout( + context.WithoutCancel(ctx), shutdownTimeout) defer shutdownCancel() err := app.Stop(shutdownCtx) @@ -148,14 +175,15 @@ func RunApp(ctx context.Context, app *fx.App) error { } }() - // Wait for either the signal handler to complete shutdown or the app to request shutdown + // Wait for the signal handler to complete shutdown or the app to + // request shutdown. select { case <-shutdownComplete: // Shutdown completed via signal return nil case <-ctx.Done(): // Context cancelled (shouldn't happen in normal operation) - err := app.Stop(context.Background()) + err := app.Stop(context.WithoutCancel(ctx)) if err != nil { log.Error("Error stopping app", "error", err) } @@ -167,6 +195,68 @@ func RunApp(ctx context.Context, app *fx.App) error { } } +// runVaultikApp runs the standard single-operation command lifecycle +// shared by the list/purge/verify/remove/remote-info subcommands: +// resolve the config, start the fx app, run op against the Vaultik +// instance in a goroutine, report a failure prefixed with failMsg +// (suppressed while suppressErrors is true, e.g. under --json), then +// trigger shutdown. The operation is cancelled when the app stops. +// extraQuiet is OR-ed into LogOptions.Quiet (e.g. --json output modes). +func runVaultikApp( + cmd *cobra.Command, extraQuiet, suppressErrors bool, + failMsg string, op func(v *vaultik.Vaultik) error, +) error { + configPath, err := ResolveConfigPath() + if err != nil { + return err + } + + rootFlags := GetRootFlags() + + return RunWithApp(cmd.Context(), AppOptions{ + ConfigPath: configPath, + LogOptions: log.Options{ + Verbose: rootFlags.Verbose, + Debug: rootFlags.Debug, + Quiet: rootFlags.Quiet || extraQuiet, + }, + Modules: []fx.Option{}, + Invokes: []fx.Option{ + fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) { + lc.Append(fx.Hook{ + OnStart: func(_ context.Context) error { + go func() { + err := op(v) + if err != nil { + if !errors.Is(err, context.Canceled) { + if !suppressErrors { + log.Error(failMsg, "error", err) + ReportErrorf("%s: %v", failMsg, err) + } + + os.Exit(1) + } + } + + err = v.Shutdowner.Shutdown() + if err != nil { + log.Error("Failed to shutdown", "error", err) + } + }() + + return nil + }, + OnStop: func(_ context.Context) error { + v.Cancel() + + return nil + }, + }) + }), + }, + }) +} + // RunWithApp is a helper that creates and runs an fx app with the given options. // It combines NewApp and RunApp into a single convenient function. This is the // preferred way to run CLI commands that need the full application context. diff --git a/internal/cli/app_test.go b/internal/cli/app_test.go index 1d90476..d865cc6 100644 --- a/internal/cli/app_test.go +++ b/internal/cli/app_test.go @@ -1,4 +1,4 @@ -package cli +package cli //nolint:testpackage // needs access to unexported cleanStartupError import ( "errors" @@ -6,6 +6,8 @@ import ( ) func TestCleanStartupError(t *testing.T) { + t.Parallel() + tests := []struct { name string in string @@ -13,7 +15,18 @@ func TestCleanStartupError(t *testing.T) { }{ { name: "real fx error chain", - in: `could not build arguments for function "sneak.berlin/go/vaultik/internal/cli".newSnapshotCreateCommand.func1.1 (/Users/user/dev/vaultik/internal/cli/snapshot.go:71): failed to build *vaultik.Vaultik: could not build arguments for function "sneak.berlin/go/vaultik/internal/vaultik".New (/Users/user/dev/vaultik/internal/vaultik/vaultik.go:59): failed to build storage.Storer: received non-nil error from function "sneak.berlin/go/vaultik/internal/storage".NewStorer (/Users/user/dev/vaultik/internal/storage/module.go:23): creating base path: mkdir /Volumes/BACKUPS: permission denied`, + in: `could not build arguments for function ` + + `"sneak.berlin/go/vaultik/internal/cli".newSnapshotCreateCommand.func1.1 ` + + `(/Users/user/dev/vaultik/internal/cli/snapshot.go:71): ` + + `failed to build *vaultik.Vaultik: ` + + `could not build arguments for function ` + + `"sneak.berlin/go/vaultik/internal/vaultik".New ` + + `(/Users/user/dev/vaultik/internal/vaultik/vaultik.go:59): ` + + `failed to build storage.Storer: ` + + `received non-nil error from function ` + + `"sneak.berlin/go/vaultik/internal/storage".NewStorer ` + + `(/Users/user/dev/vaultik/internal/storage/module.go:23): ` + + `creating base path: mkdir /Volumes/BACKUPS: permission denied`, want: `creating base path: mkdir /Volumes/BACKUPS: permission denied`, }, { @@ -30,6 +43,9 @@ func TestCleanStartupError(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + //nolint:err113 // test constructs errors from table input got := cleanStartupError(errors.New(tt.in)).Error() if got != tt.want { t.Errorf("got %q, want %q", got, tt.want) diff --git a/internal/cli/config.go b/internal/cli/config.go index c1bc9c4..11c05b1 100644 --- a/internal/cli/config.go +++ b/internal/cli/config.go @@ -13,6 +13,26 @@ import ( "gopkg.in/yaml.v3" ) +// configFileMode is the permission set for freshly written config files; +// configs may hold S3 credentials, so keep them owner-only. +const configFileMode = 0o600 + +// configSetArgs is the argument count of `config set `. +const configSetArgs = 2 + +// configDirMode is the permission set for created config directories; +// parent config dirs (e.g. ~/.config) are conventionally traversable. +const configDirMode = 0o755 + +var ( + errConfigExists = errors.New("config file already exists") + errEmptyConfig = errors.New("empty config file") + errKeyNotFound = errors.New("key not found") + errNeedNumericIndex = errors.New("key is a list; use a numeric index") + errIndexOutOfRange = errors.New("index out of range") + errNotMapOrList = errors.New("key is not a map or list") +) + const defaultConfigTemplate = `# vaultik configuration # Documentation: https://sneak.berlin/go/vaultik @@ -233,27 +253,29 @@ The config is written to the path from --config, $VAULTIK_CONFIG, or the platform default config directory (e.g. ~/Library/Application Support/ on macOS, ~/.config/ on Linux, /etc/vaultik/ as root).`, Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(_ *cobra.Command, _ []string) error { path := configPathForInit() - if _, err := os.Stat(path); err == nil { - return fmt.Errorf("config file already exists: %s", path) + _, err := os.Stat(path) + if err == nil { + return fmt.Errorf("%w: %s", errConfigExists, path) } dir := filepath.Dir(path) - err := os.MkdirAll(dir, 0o755) + err = os.MkdirAll(dir, configDirMode) if err != nil { return fmt.Errorf("creating config directory %s: %w", dir, err) } - err = os.WriteFile(path, []byte(defaultConfigTemplate), 0o600) + err = os.WriteFile(path, []byte(defaultConfigTemplate), configFileMode) if err != nil { return fmt.Errorf("writing config file: %w", err) } - fmt.Printf("Config written to %s\n", path) - fmt.Println("Edit it to set your age_recipients, snapshots, and storage_url.") + _, _ = fmt.Fprintf(os.Stdout, "Config written to %s\n", path) + _, _ = fmt.Fprintln(os.Stdout, + "Edit it to set your age_recipients, snapshots, and storage_url.") return nil }, @@ -266,7 +288,7 @@ func newConfigEditCommand() *cobra.Command { Use: "edit", Short: "Open the config file in $EDITOR", Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, _ []string) error { path, err := ResolveConfigPath() if err != nil { return err @@ -277,7 +299,8 @@ func newConfigEditCommand() *cobra.Command { editor = "vi" } - ed := exec.Command(editor, path) + //nolint:gosec // G204: launching the operator's own $EDITOR is the point + ed := exec.CommandContext(cmd.Context(), editor, path) ed.Stdin = os.Stdin ed.Stdout = os.Stdout ed.Stderr = os.Stderr @@ -293,7 +316,7 @@ func newConfigGetCommand() *cobra.Command { Use: "get ", Short: "Print a config value by dotted path (e.g. storage_url, compression_level)", Args: cobra.ExactArgs(1), - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(_ *cobra.Command, args []string) error { path, err := ResolveConfigPath() if err != nil { return err @@ -310,7 +333,7 @@ func newConfigGetCommand() *cobra.Command { } if node.Kind == yaml.ScalarNode { - fmt.Println(node.Value) + _, _ = fmt.Fprintln(os.Stdout, node.Value) return nil } @@ -320,7 +343,7 @@ func newConfigGetCommand() *cobra.Command { return fmt.Errorf("marshaling value: %w", err) } - fmt.Print(string(out)) + _, _ = fmt.Fprint(os.Stdout, string(out)) return nil }, @@ -341,8 +364,8 @@ Examples: vaultik config set storage_url "s3://bucket/prefix?endpoint=host®ion=us-east-1" vaultik config set compression_level 9 vaultik config set s3.bucket mybucket # legacy S3 fields still supported`, - Args: cobra.ExactArgs(2), - RunE: func(cmd *cobra.Command, args []string) error { + Args: cobra.ExactArgs(configSetArgs), + RunE: func(_ *cobra.Command, args []string) error { path, err := ResolveConfigPath() if err != nil { return err @@ -353,7 +376,8 @@ Examples: return err } - if err := yamlPathSet(root, strings.Split(args[0], "."), args[1]); err != nil { + err = yamlPathSet(root, strings.Split(args[0], "."), args[1]) + if err != nil { return err } @@ -362,16 +386,19 @@ Examples: return fmt.Errorf("marshaling config: %w", err) } - mode := os.FileMode(0o600) - if info, err := os.Stat(path); err == nil { + mode := os.FileMode(configFileMode) + + info, statErr := os.Stat(path) + if statErr == nil { mode = info.Mode().Perm() } - if err := os.WriteFile(path, out, mode); err != nil { + err = os.WriteFile(path, out, mode) + if err != nil { return fmt.Errorf("writing config file: %w", err) } - fmt.Printf("%s = %s\n", args[0], args[1]) + _, _ = fmt.Fprintf(os.Stdout, "%s = %s\n", args[0], args[1]) return nil }, @@ -381,13 +408,15 @@ Examples: // loadYAMLFile parses a YAML file into a yaml.Node document tree, // which preserves comments and ordering for round-tripping. func loadYAMLFile(path string) (*yaml.Node, error) { - data, err := os.ReadFile(path) + data, err := os.ReadFile(path) //nolint:gosec // G304: config path is operator-supplied if err != nil { return nil, fmt.Errorf("reading config file: %w", err) } var root yaml.Node - if err := yaml.Unmarshal(data, &root); err != nil { + + err = yaml.Unmarshal(data, &root) + if err != nil { return nil, fmt.Errorf("parsing config file: %w", err) } @@ -409,7 +438,7 @@ func yamlPathGet(root *yaml.Node, keys []string) (*yaml.Node, error) { node := root if node.Kind == yaml.DocumentNode { if len(node.Content) == 0 { - return nil, errors.New("empty config file") + return nil, errEmptyConfig } node = node.Content[0] @@ -430,21 +459,29 @@ func yamlPathGet(root *yaml.Node, keys []string) (*yaml.Node, error) { } if !found { - return nil, fmt.Errorf("key not found: %s", strings.Join(keys[:i+1], ".")) + return nil, fmt.Errorf("%w: %s", + errKeyNotFound, strings.Join(keys[:i+1], ".")) } case yaml.SequenceNode: idx, err := strconv.Atoi(key) if err != nil { - return nil, fmt.Errorf("key %q is a list; use a numeric index", strings.Join(keys[:i], ".")) + return nil, fmt.Errorf("%w: %s", + errNeedNumericIndex, strings.Join(keys[:i], ".")) } if idx < 0 || idx >= len(node.Content) { - return nil, fmt.Errorf("index %d out of range for %s (len %d)", idx, strings.Join(keys[:i], "."), len(node.Content)) + return nil, fmt.Errorf("%w: index %d for %s (len %d)", + errIndexOutOfRange, idx, strings.Join(keys[:i], "."), + len(node.Content)) } node = node.Content[idx] + case yaml.DocumentNode, yaml.ScalarNode, yaml.AliasNode: + return nil, fmt.Errorf("%w: %s", + errNotMapOrList, strings.Join(keys[:i], ".")) default: - return nil, fmt.Errorf("key %q is not a map or list", strings.Join(keys[:i], ".")) + return nil, fmt.Errorf("%w: %s", + errNotMapOrList, strings.Join(keys[:i], ".")) } } @@ -470,62 +507,88 @@ func yamlPathSet(root *yaml.Node, keys []string, value string) error { switch node.Kind { case yaml.MappingNode: - var valueNode *yaml.Node - - for j := 0; j+1 < len(node.Content); j += 2 { - if node.Content[j].Value == key { - valueNode = node.Content[j+1] - - break - } - } - - if valueNode == nil { - keyNode := &yaml.Node{Kind: yaml.ScalarNode, Value: key} - - valueNode = &yaml.Node{Kind: yaml.MappingNode} - if last { - valueNode = &yaml.Node{Kind: yaml.ScalarNode, Value: value} - } - - node.Content = append(node.Content, keyNode, valueNode) - } else if last { - setScalar(valueNode, value) - } - - node = valueNode - + node = yamlSetInMapping(node, key, value, last) case yaml.SequenceNode: - idx, err := strconv.Atoi(key) + next, err := yamlSetInSequence(node, keys, i, value, last) if err != nil { - return fmt.Errorf("key %q is a list; use a numeric index", strings.Join(keys[:i], ".")) + return err } - if idx < 0 || idx > len(node.Content) { - return fmt.Errorf("index %d out of range for %s (len %d)", idx, strings.Join(keys[:i], "."), len(node.Content)) - } - - if idx == len(node.Content) { - newNode := &yaml.Node{Kind: yaml.MappingNode} - if last { - newNode = &yaml.Node{Kind: yaml.ScalarNode, Value: value} - } - - node.Content = append(node.Content, newNode) - } else if last { - setScalar(node.Content[idx], value) - } - - node = node.Content[idx] - + node = next + case yaml.DocumentNode, yaml.ScalarNode, yaml.AliasNode: + return fmt.Errorf("%w: %s", + errNotMapOrList, strings.Join(keys[:i], ".")) default: - return fmt.Errorf("key %q is not a map or list", strings.Join(keys[:i], ".")) + return fmt.Errorf("%w: %s", + errNotMapOrList, strings.Join(keys[:i], ".")) } } return nil } +// yamlSetInMapping resolves (creating if needed) the value node for key +// within a mapping node, setting it to value when it is the final path +// element, and returns the node to descend into. +func yamlSetInMapping(node *yaml.Node, key, value string, last bool) *yaml.Node { + var valueNode *yaml.Node + + for j := 0; j+1 < len(node.Content); j += 2 { + if node.Content[j].Value == key { + valueNode = node.Content[j+1] + + break + } + } + + if valueNode == nil { + keyNode := &yaml.Node{Kind: yaml.ScalarNode, Value: key} + + valueNode = &yaml.Node{Kind: yaml.MappingNode} + if last { + valueNode = &yaml.Node{Kind: yaml.ScalarNode, Value: value} + } + + node.Content = append(node.Content, keyNode, valueNode) + } else if last { + setScalar(valueNode, value) + } + + return valueNode +} + +// yamlSetInSequence indexes (or appends to) a sequence node using the +// numeric path element keys[i], setting the element to value when it is +// the final path element, and returns the node to descend into. +func yamlSetInSequence( + node *yaml.Node, keys []string, i int, value string, last bool, +) (*yaml.Node, error) { + idx, err := strconv.Atoi(keys[i]) + if err != nil { + return nil, fmt.Errorf("%w: %s", + errNeedNumericIndex, strings.Join(keys[:i], ".")) + } + + if idx < 0 || idx > len(node.Content) { + return nil, fmt.Errorf("%w: index %d for %s (len %d)", + errIndexOutOfRange, idx, strings.Join(keys[:i], "."), + len(node.Content)) + } + + if idx == len(node.Content) { + newNode := &yaml.Node{Kind: yaml.MappingNode} + if last { + newNode = &yaml.Node{Kind: yaml.ScalarNode, Value: value} + } + + node.Content = append(node.Content, newNode) + } else if last { + setScalar(node.Content[idx], value) + } + + return node.Content[idx], nil +} + // setScalar overwrites a node in place with a plain scalar value. func setScalar(n *yaml.Node, value string) { n.Kind = yaml.ScalarNode diff --git a/internal/cli/config_test.go b/internal/cli/config_test.go index 24ad8ab..f717841 100644 --- a/internal/cli/config_test.go +++ b/internal/cli/config_test.go @@ -1,4 +1,4 @@ -package cli +package cli //nolint:testpackage // exercises unexported yamlPathGet/yamlPathSet import ( "strings" @@ -11,6 +11,8 @@ import ( // TestDefaultConfigTemplateParses ensures the init template is valid YAML // that unmarshals into the Config struct with the expected snapshots. func TestDefaultConfigTemplateParses(t *testing.T) { + t.Parallel() + var cfg config.Config err := yaml.Unmarshal([]byte(defaultConfigTemplate), &cfg) @@ -76,6 +78,8 @@ func parseTestYAML(t *testing.T) *yaml.Node { } func TestYAMLPathGet(t *testing.T) { + t.Parallel() + root := parseTestYAML(t) tests := []struct { @@ -96,6 +100,8 @@ func TestYAMLPathGet(t *testing.T) { for _, tt := range tests { t.Run(tt.path, func(t *testing.T) { + t.Parallel() + node, err := yamlPathGet(root, splitPath(tt.path)) if tt.err { if err == nil { @@ -117,32 +123,40 @@ func TestYAMLPathGet(t *testing.T) { } func TestYAMLPathSet(t *testing.T) { + t.Parallel() + root := parseTestYAML(t) // Overwrite existing nested value - if err := yamlPathSet(root, splitPath("s3.bucket"), "newbucket"); err != nil { + err := yamlPathSet(root, splitPath("s3.bucket"), "newbucket") + if err != nil { t.Fatalf("set s3.bucket: %v", err) } // Create new nested key with intermediate map - if err := yamlPathSet(root, splitPath("s3.endpoint"), "s3.example.com"); err != nil { + err = yamlPathSet(root, splitPath("s3.endpoint"), "s3.example.com") + if err != nil { t.Fatalf("set s3.endpoint: %v", err) } - if err := yamlPathSet(root, splitPath("newmap.newkey"), "val"); err != nil { + err = yamlPathSet(root, splitPath("newmap.newkey"), "val") + if err != nil { t.Fatalf("set newmap.newkey: %v", err) } // Overwrite a sequence element and append a new one - if err := yamlPathSet(root, splitPath("age_recipients.0"), "age1bbb"); err != nil { + err = yamlPathSet(root, splitPath("age_recipients.0"), "age1bbb") + if err != nil { t.Fatalf("set age_recipients.0: %v", err) } - if err := yamlPathSet(root, splitPath("age_recipients.1"), "age1ccc"); err != nil { + err = yamlPathSet(root, splitPath("age_recipients.1"), "age1ccc") + if err != nil { t.Fatalf("append age_recipients.1: %v", err) } - if err := yamlPathSet(root, splitPath("age_recipients.5"), "age1ddd"); err == nil { + err = yamlPathSet(root, splitPath("age_recipients.5"), "age1ddd") + if err == nil { t.Error("expected out-of-range append to fail") } @@ -154,7 +168,11 @@ func TestYAMLPathSet(t *testing.T) { text := string(out) - for _, want := range []string{"newbucket", "s3.example.com", "newkey: val", "# top comment", "# inline comment", "age1bbb", "age1ccc"} { + wants := []string{ + "newbucket", "s3.example.com", "newkey: val", + "# top comment", "# inline comment", "age1bbb", "age1ccc", + } + for _, want := range wants { if !contains(text, want) { t.Errorf("round-tripped YAML missing %q:\n%s", want, text) } diff --git a/internal/cli/database.go b/internal/cli/database.go index a6532fb..3f012ba 100644 --- a/internal/cli/database.go +++ b/internal/cli/database.go @@ -48,7 +48,7 @@ storage destination on that run. Use --force to skip the confirmation prompt.`, Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(_ *cobra.Command, _ []string) error { // Resolve config path configPath, err := ResolveConfigPath() if err != nil { @@ -64,27 +64,33 @@ Use --force to skip the confirmation prompt.`, dbPath := cfg.IndexPath // Check if database exists - if _, err := os.Stat(dbPath); os.IsNotExist(err) { - fmt.Printf("Database does not exist: %s\n", dbPath) + _, err = os.Stat(dbPath) + if os.IsNotExist(err) { + _, _ = fmt.Fprintf(os.Stdout, "Database does not exist: %s\n", dbPath) return nil } // Confirm unless --force if !force { - fmt.Printf("This will delete the local state database at:\n %s\n\n", dbPath) - fmt.Print("Are you sure? Type 'yes' to confirm: ") + _, _ = fmt.Fprintf(os.Stdout, + "This will delete the local state database at:\n %s\n\n", dbPath) + _, _ = fmt.Fprint(os.Stdout, "Are you sure? Type 'yes' to confirm: ") var confirm string - if _, err := fmt.Scanln(&confirm); err != nil || confirm != "yes" { - fmt.Println("Aborted.") + _, err = fmt.Scanln(&confirm) + if err != nil || confirm != "yes" { + _, _ = fmt.Fprintln(os.Stdout, "Aborted.") + + //nolint:nilerr // a failed/aborted confirmation is a clean abort return nil } } // Delete the database file - if err := os.Remove(dbPath); err != nil { + err = os.Remove(dbPath) + if err != nil { return fmt.Errorf("failed to delete database: %w", err) } @@ -96,7 +102,7 @@ Use --force to skip the confirmation prompt.`, rootFlags := GetRootFlags() if !rootFlags.Quiet { - fmt.Printf("Database deleted: %s\n", dbPath) + _, _ = fmt.Fprintf(os.Stdout, "Database deleted: %s\n", dbPath) } log.Info("Local state database deleted", "path", dbPath) diff --git a/internal/cli/duration.go b/internal/cli/duration.go index 8fd2350..ee54257 100644 --- a/internal/cli/duration.go +++ b/internal/cli/duration.go @@ -9,6 +9,21 @@ import ( "time" ) +// Approximate lengths of the extended calendar units accepted by +// parseDuration. +const ( + durationDay = 24 * time.Hour + durationWeek = 7 * durationDay + durationMonth = 30 * durationDay + durationYear = 365 * durationDay +) + +var ( + errNegativeDuration = errors.New("negative durations are not supported") + errInvalidDuration = errors.New("invalid duration format") + errUnknownTimeUnit = errors.New("unknown time unit") +) + // parseDuration parses duration strings. Supports standard Go duration format // (e.g., "3h30m", "1h45m30s") as well as extended units: // - d: days (e.g., "30d", "7d") @@ -19,14 +34,15 @@ import ( // Can combine units: "1y6mo", "2w3d", "1d12h30m" func parseDuration(s string) (time.Duration, error) { // First try standard Go duration parsing - if d, err := time.ParseDuration(s); err == nil { + d, err := time.ParseDuration(s) + if err == nil { return d, nil } // Extended duration parsing // Check for negative values if strings.HasPrefix(strings.TrimSpace(s), "-") { - return 0, errors.New("negative durations are not supported") + return 0, errNegativeDuration } // Pattern matches: number + unit, repeated @@ -34,7 +50,7 @@ func parseDuration(s string) (time.Duration, error) { matches := re.FindAllStringSubmatch(s, -1) if len(matches) == 0 { - return 0, fmt.Errorf("invalid duration format: %q", s) + return 0, fmt.Errorf("%w: %q", errInvalidDuration, s) } var total time.Duration @@ -48,45 +64,9 @@ func parseDuration(s string) (time.Duration, error) { return 0, fmt.Errorf("invalid number %q: %w", valueStr, err) } - var d time.Duration - - switch unit { - // Standard time units - case "ns", "nanosecond", "nanoseconds": - d = time.Duration(value) - case "us", "µs", "microsecond", "microseconds": - d = time.Duration(value * float64(time.Microsecond)) - case "ms", "millisecond", "milliseconds": - d = time.Duration(value * float64(time.Millisecond)) - case "s", "sec", "second", "seconds": - d = time.Duration(value * float64(time.Second)) - case "m", "min", "minute", "minutes": - d = time.Duration(value * float64(time.Minute)) - case "h", "hr", "hour", "hours": - d = time.Duration(value * float64(time.Hour)) - // Extended units - case "d", "day", "days": - d = time.Duration(value * float64(24*time.Hour)) - case "w", "week", "weeks": - d = time.Duration(value * float64(7*24*time.Hour)) - case "mo", "month", "months": - // Using 30 days as approximation - d = time.Duration(value * float64(30*24*time.Hour)) - case "y", "year", "years": - // Using 365 days as approximation - d = time.Duration(value * float64(365*24*time.Hour)) - default: - // Try parsing as standard Go duration unit - testStr := "1" + unit - if _, err := time.ParseDuration(testStr); err == nil { - // It's a valid Go duration unit, parse the full value - fullStr := fmt.Sprintf("%g%s", value, unit) - if d, err = time.ParseDuration(fullStr); err != nil { - return 0, fmt.Errorf("invalid duration %q: %w", fullStr, err) - } - } else { - return 0, fmt.Errorf("unknown time unit %q", unit) - } + d, err := durationForUnit(value, unit) + if err != nil { + return 0, err } total += d @@ -94,3 +74,53 @@ func parseDuration(s string) (time.Duration, error) { return total, nil } + +// durationForUnit converts a value with a (case-normalized) unit suffix +// into a time.Duration, accepting Go's standard units plus the extended +// calendar units. +func durationForUnit(value float64, unit string) (time.Duration, error) { + switch unit { + // Standard time units + case "ns", "nanosecond", "nanoseconds": + return time.Duration(value), nil + case "us", "µs", "microsecond", "microseconds": + return time.Duration(value * float64(time.Microsecond)), nil + case "ms", "millisecond", "milliseconds": + return time.Duration(value * float64(time.Millisecond)), nil + case "s", "sec", "second", "seconds": + return time.Duration(value * float64(time.Second)), nil + case "m", "min", "minute", "minutes": + return time.Duration(value * float64(time.Minute)), nil + case "h", "hr", "hour", "hours": + return time.Duration(value * float64(time.Hour)), nil + // Extended units + case "d", "day", "days": + return time.Duration(value * float64(durationDay)), nil + case "w", "week", "weeks": + return time.Duration(value * float64(durationWeek)), nil + case "mo", "month", "months": + // Using 30 days as approximation + return time.Duration(value * float64(durationMonth)), nil + case "y", "year", "years": + // Using 365 days as approximation + return time.Duration(value * float64(durationYear)), nil + default: + // Try parsing as standard Go duration unit + testStr := "1" + unit + + _, err := time.ParseDuration(testStr) + if err != nil { + return 0, fmt.Errorf("%w: %q", errUnknownTimeUnit, unit) + } + + // It's a valid Go duration unit, parse the full value + fullStr := fmt.Sprintf("%g%s", value, unit) + + d, err := time.ParseDuration(fullStr) + if err != nil { + return 0, fmt.Errorf("invalid duration %q: %w", fullStr, err) + } + + return d, nil + } +} diff --git a/internal/cli/duration_test.go b/internal/cli/duration_test.go index 3edb414..cc077d3 100644 --- a/internal/cli/duration_test.go +++ b/internal/cli/duration_test.go @@ -1,20 +1,47 @@ -package cli +package cli //nolint:testpackage // needs access to unexported parseDuration import ( "testing" "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) -func TestParseDuration(t *testing.T) { - tests := []struct { - name string - input string - expected time.Duration - wantErr bool - }{ - // Standard Go durations +type parseDurationCase struct { + name string + input string + expected time.Duration + wantErr bool +} + +// runParseDurationCases executes a table of parseDuration cases as +// parallel subtests. +func runParseDurationCases(t *testing.T, tests []parseDurationCase) { + t.Helper() + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got, err := parseDuration(tt.input) + + if tt.wantErr { + require.Error(t, err, "expected error for input %q", tt.input) + + return + } + + require.NoError(t, err, "unexpected error for input %q", tt.input) + assert.Equal(t, tt.expected, got, "duration mismatch for input %q", tt.input) + }) + } +} + +func TestParseDurationStandard(t *testing.T) { + t.Parallel() + + runParseDurationCases(t, []parseDurationCase{ { name: "standard seconds", input: "30s", @@ -45,6 +72,13 @@ func TestParseDuration(t *testing.T) { input: "1s500ms", expected: 1*time.Second + 500*time.Millisecond, }, + }) +} + +func TestParseDurationExtendedUnits(t *testing.T) { + t.Parallel() + + runParseDurationCases(t, []parseDurationCase{ // Extended units - days { name: "single day", @@ -114,6 +148,13 @@ func TestParseDuration(t *testing.T) { input: "1year", expected: 365 * 24 * time.Hour, }, + }) +} + +func TestParseDurationCombinedAndErrors(t *testing.T) { + t.Parallel() + + runParseDurationCases(t, []parseDurationCase{ // Combined extended units { name: "weeks and days", @@ -131,9 +172,11 @@ func TestParseDuration(t *testing.T) { expected: 24*time.Hour + 12*time.Hour, }, { - name: "complex combination", - input: "1y2mo3w4d5h6m7s", - expected: 365*24*time.Hour + 2*30*24*time.Hour + 3*7*24*time.Hour + 4*24*time.Hour + 5*time.Hour + 6*time.Minute + 7*time.Second, + name: "complex combination", + input: "1y2mo3w4d5h6m7s", + expected: 365*24*time.Hour + 2*30*24*time.Hour + + 3*7*24*time.Hour + 4*24*time.Hour + + 5*time.Hour + 6*time.Minute + 7*time.Second, }, { name: "with spaces", @@ -177,25 +220,12 @@ func TestParseDuration(t *testing.T) { input: "-5d", wantErr: true, }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := parseDuration(tt.input) - - if tt.wantErr { - assert.Error(t, err, "expected error for input %q", tt.input) - - return - } - - assert.NoError(t, err, "unexpected error for input %q", tt.input) - assert.Equal(t, tt.expected, got, "duration mismatch for input %q", tt.input) - }) - } + }) } func TestParseDurationSpecialCases(t *testing.T) { + t.Parallel() + // Test that standard Go durations work exactly as expected standardDurations := []string{ "300ms", @@ -209,15 +239,17 @@ func TestParseDurationSpecialCases(t *testing.T) { for _, d := range standardDurations { expected, err := time.ParseDuration(d) - assert.NoError(t, err) + require.NoError(t, err) got, err := parseDuration(d) - assert.NoError(t, err) + require.NoError(t, err) assert.Equal(t, expected, got, "standard duration %q should parse identically", d) } } func TestParseDurationRealWorldExamples(t *testing.T) { + t.Parallel() + // Test real-world snapshot purge scenarios tests := []struct { description string @@ -253,12 +285,15 @@ func TestParseDurationRealWorldExamples(t *testing.T) { for _, tt := range tests { t.Run(tt.description, func(t *testing.T) { + t.Parallel() + got, err := parseDuration(tt.input) - assert.NoError(t, err) + require.NoError(t, err) assert.Equal(t, tt.olderThan, got) // Verify the duration makes sense for snapshot purging - assert.Greater(t, got, time.Hour, "snapshot purge duration should be at least an hour") + assert.Greater(t, got, time.Hour, + "snapshot purge duration should be at least an hour") }) } } diff --git a/internal/cli/entry.go b/internal/cli/entry.go index 9709f08..2dfee66 100644 --- a/internal/cli/entry.go +++ b/internal/cli/entry.go @@ -9,15 +9,19 @@ import ( "sneak.berlin/go/vaultik/internal/ui" ) -// CLIEntry is the main entry point for the CLI application. +// shortCommitLen is the number of git commit hash characters shown in +// the startup banner. +const shortCommitLen = 12 + +// Entry is the main entry point for the CLI application. // It prints the startup banner (unless a quiet flag is present in os.Args), // executes the root cobra command, and routes any returned error through // the ui.Writer so the user sees a properly formatted "🛑 ERROR:" line. -func CLIEntry() { +func Entry() { if !bannerSuppressedInArgs(os.Args[1:]) { short := globals.Commit - if len(short) > 12 { - short = short[:12] + if len(short) > shortCommitLen { + short = short[:shortCommitLen] } writeStartupBanner(ui.New(os.Stdout), time.Now().UTC(), short) diff --git a/internal/cli/entry_test.go b/internal/cli/entry_test.go index f652399..60877ed 100644 --- a/internal/cli/entry_test.go +++ b/internal/cli/entry_test.go @@ -1,14 +1,18 @@ -package cli +package cli_test import ( "testing" + + "sneak.berlin/go/vaultik/internal/cli" ) // TestCLIEntry ensures the CLI can be imported and basic initialization works func TestCLIEntry(t *testing.T) { + t.Parallel() + // This test primarily serves as a compilation test // to ensure all imports resolve correctly - cmd := NewRootCommand() + cmd := cli.NewRootCommand() if cmd == nil { t.Fatal("NewRootCommand() returned nil") } @@ -18,7 +22,9 @@ func TestCLIEntry(t *testing.T) { } // Verify all subcommands are registered - expectedCommands := []string{"config", "snapshot", "prune", "info", "version", "remote", "database"} + expectedCommands := []string{ + "config", "snapshot", "prune", "info", "version", "remote", "database", + } for _, expected := range expectedCommands { found := false @@ -41,7 +47,9 @@ func TestCLIEntry(t *testing.T) { t.Errorf("Failed to find snapshot command: %v", err) } else { // Check snapshot subcommands - expectedSubCommands := []string{"create", "list", "purge", "verify", "remove", "restore"} + expectedSubCommands := []string{ + "create", "list", "purge", "verify", "remove", "restore", + } for _, expected := range expectedSubCommands { found := false diff --git a/internal/cli/info.go b/internal/cli/info.go index 8cc715d..2e5cc9c 100644 --- a/internal/cli/info.go +++ b/internal/cli/info.go @@ -23,7 +23,7 @@ func NewInfoCommand() *cobra.Command { - Encryption configuration (recipients) - Local database statistics`, Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, _ []string) error { // Use unified config resolution configPath, err := ResolveConfigPath() if err != nil { @@ -35,7 +35,7 @@ func NewInfoCommand() *cobra.Command { return RunWithApp(cmd.Context(), AppOptions{ ConfigPath: configPath, - LogOptions: log.LogOptions{ + LogOptions: log.Options{ Verbose: rootFlags.Verbose, Debug: rootFlags.Debug, Quiet: rootFlags.Quiet, @@ -44,7 +44,7 @@ func NewInfoCommand() *cobra.Command { Invokes: []fx.Option{ fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) { lc.Append(fx.Hook{ - OnStart: func(ctx context.Context) error { + OnStart: func(_ context.Context) error { go func() { err := v.ShowInfo() if err != nil { @@ -63,7 +63,7 @@ func NewInfoCommand() *cobra.Command { return nil }, - OnStop: func(ctx context.Context) error { + OnStop: func(_ context.Context) error { v.Cancel() return nil diff --git a/internal/cli/prune.go b/internal/cli/prune.go index b5cc67e..950e7fd 100644 --- a/internal/cli/prune.go +++ b/internal/cli/prune.go @@ -31,7 +31,7 @@ Snapshot create --prune and snapshot remove run the same cleanup automatically; this command is the manual entry point for the same work (e.g. after a crashed backup or to reclaim storage).`, Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, _ []string) error { // Use unified config resolution configPath, err := ResolveConfigPath() if err != nil { @@ -43,7 +43,7 @@ work (e.g. after a crashed backup or to reclaim storage).`, return RunWithApp(cmd.Context(), AppOptions{ ConfigPath: configPath, - LogOptions: log.LogOptions{ + LogOptions: log.Options{ Verbose: rootFlags.Verbose, Debug: rootFlags.Debug, Quiet: rootFlags.Quiet || opts.JSON, @@ -52,7 +52,7 @@ work (e.g. after a crashed backup or to reclaim storage).`, Invokes: []fx.Option{ fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) { lc.Append(fx.Hook{ - OnStart: func(ctx context.Context) error { + OnStart: func(_ context.Context) error { // Start the prune operation in a goroutine go func() { // Run the prune operation @@ -77,7 +77,7 @@ work (e.g. after a crashed backup or to reclaim storage).`, return nil }, - OnStop: func(ctx context.Context) error { + OnStop: func(_ context.Context) error { log.Debug("Stopping prune operation") v.Cancel() diff --git a/internal/cli/remote.go b/internal/cli/remote.go index 7acb001..61987a3 100644 --- a/internal/cli/remote.go +++ b/internal/cli/remote.go @@ -11,6 +11,10 @@ import ( "sneak.berlin/go/vaultik/internal/vaultik" ) +// errNukeNeedsForce guards the destructive 'remote nuke' subcommand. +var errNukeNeedsForce = errors.New( + "remote nuke requires --force (this deletes ALL remote snapshots and blobs)") + // NewRemoteCommand creates the remote command and subcommands func NewRemoteCommand() *cobra.Command { cmd := &cobra.Command{ @@ -39,61 +43,20 @@ empty and the next backup starts from scratch. This is destructive and irreversible. Requires --force.`, Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, _ []string) error { if !force { - return errors.New("remote nuke requires --force (this deletes ALL remote snapshots and blobs)") + return errNukeNeedsForce } - configPath, err := ResolveConfigPath() - if err != nil { - return err - } - - rootFlags := GetRootFlags() - - return RunWithApp(cmd.Context(), AppOptions{ - ConfigPath: configPath, - LogOptions: log.LogOptions{ - Verbose: rootFlags.Verbose, - Debug: rootFlags.Debug, - Quiet: rootFlags.Quiet, - }, - Modules: []fx.Option{}, - Invokes: []fx.Option{ - fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) { - lc.Append(fx.Hook{ - OnStart: func(ctx context.Context) error { - go func() { - err := v.NukeRemote(true) - if err != nil { - if !errors.Is(err, context.Canceled) { - log.Error("Remote nuke failed", "error", err) - ReportErrorf("Remote nuke failed: %v", err) - os.Exit(1) - } - } - - err = v.Shutdowner.Shutdown() - if err != nil { - log.Error("Failed to shutdown", "error", err) - } - }() - - return nil - }, - OnStop: func(ctx context.Context) error { - v.Cancel() - - return nil - }, - }) - }), - }, - }) + return runVaultikApp(cmd, false, false, "Remote nuke failed", + func(v *vaultik.Vaultik) error { + return v.NukeRemote(true) + }) }, } - cmd.Flags().BoolVar(&force, "force", false, "Required: confirm destruction of ALL remote data") + cmd.Flags().BoolVar(&force, "force", false, + "Required: confirm destruction of ALL remote data") return cmd } @@ -111,7 +74,7 @@ func newRemoteInfoCommand() *cobra.Command { - Count and size of referenced blobs (from all manifests) - Count and size of orphaned blobs (not referenced by any manifest)`, Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, _ []string) error { // Use unified config resolution configPath, err := ResolveConfigPath() if err != nil { @@ -122,7 +85,7 @@ func newRemoteInfoCommand() *cobra.Command { return RunWithApp(cmd.Context(), AppOptions{ ConfigPath: configPath, - LogOptions: log.LogOptions{ + LogOptions: log.Options{ Verbose: rootFlags.Verbose, Debug: rootFlags.Debug, Quiet: rootFlags.Quiet || jsonOutput, @@ -131,7 +94,7 @@ func newRemoteInfoCommand() *cobra.Command { Invokes: []fx.Option{ fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) { lc.Append(fx.Hook{ - OnStart: func(ctx context.Context) error { + OnStart: func(_ context.Context) error { go func() { err := v.RemoteInfo(jsonOutput) if err != nil { @@ -153,7 +116,7 @@ func newRemoteInfoCommand() *cobra.Command { return nil }, - OnStop: func(ctx context.Context) error { + OnStop: func(_ context.Context) error { v.Cancel() return nil diff --git a/internal/cli/root.go b/internal/cli/root.go index 2e53dad..0c8c6ce 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -1,6 +1,7 @@ package cli import ( + "errors" "fmt" "os" "path/filepath" @@ -10,6 +11,9 @@ import ( "github.com/spf13/cobra" ) +// errConfigNotFound is wrapped by all config-resolution failures. +var errConfigNotFound = errors.New("config file not found") + // RootFlags holds global flags that apply to all commands. // These flags are defined on the root command and inherited by all subcommands. type RootFlags struct { @@ -20,6 +24,7 @@ type RootFlags struct { SkipErrors bool } +//nolint:gochecknoglobals // cobra persistent flags bind to package state var rootFlags RootFlags // NewRootCommand creates the root cobra command for the vaultik CLI. @@ -34,20 +39,26 @@ public keys and uploads to S3-compatible storage. No private keys are needed on the source system.`, SilenceUsage: true, // Bare 'vaultik' (no subcommand): print help. The banner is - // printed once at process startup by CLIEntry, before cobra + // printed once at process startup by Entry, before cobra // parses arguments, so it appears even when cobra rejects // args (e.g. "requires at least 2 arg(s)") and on --help. - Run: func(cmd *cobra.Command, args []string) { + Run: func(cmd *cobra.Command, _ []string) { _ = cmd.Help() }, } // Add global flags - cmd.PersistentFlags().StringVar(&rootFlags.ConfigPath, "config", "", "Path to config file (default: $VAULTIK_CONFIG or platform config dir)") - cmd.PersistentFlags().BoolVarP(&rootFlags.Verbose, "verbose", "v", false, "Enable verbose output") - cmd.PersistentFlags().BoolVar(&rootFlags.Debug, "debug", false, "Enable debug output") - cmd.PersistentFlags().BoolVarP(&rootFlags.Quiet, "quiet", "q", false, "Suppress non-error output") - cmd.PersistentFlags().BoolVar(&rootFlags.SkipErrors, "skip-errors", false, "Continue past per-file errors instead of aborting (applies to snapshot create and restore)") + cmd.PersistentFlags().StringVar(&rootFlags.ConfigPath, "config", "", + "Path to config file (default: $VAULTIK_CONFIG or platform config dir)") + cmd.PersistentFlags().BoolVarP(&rootFlags.Verbose, "verbose", "v", false, + "Enable verbose output") + cmd.PersistentFlags().BoolVar(&rootFlags.Debug, "debug", false, + "Enable debug output") + cmd.PersistentFlags().BoolVarP(&rootFlags.Quiet, "quiet", "q", false, + "Suppress non-error output") + cmd.PersistentFlags().BoolVar(&rootFlags.SkipErrors, "skip-errors", false, + "Continue past per-file errors instead of aborting "+ + "(applies to snapshot create and restore)") // Add subcommands cmd.AddCommand( @@ -70,33 +81,45 @@ func GetRootFlags() RootFlags { } // ResolveConfigPath resolves the config file path from flags, environment, or default. -// Search order: --config flag, VAULTIK_CONFIG env, XDG config dir, /etc/vaultik/config.yml. +// Search order: --config flag, VAULTIK_CONFIG env, XDG config dir, +// /etc/vaultik/config.yml. // Explicit paths from --config and $VAULTIK_CONFIG are checked for existence // so the user gets a clear error instead of a downstream YAML parser failure. func ResolveConfigPath() (string, error) { if path := rootFlags.ConfigPath; path != "" { - if _, err := os.Stat(path); err != nil { - return "", fmt.Errorf("config file from --config not found: %s (run 'vaultik config init --config %s' to create it)", path, path) + _, err := os.Stat(path) + if err != nil { + return "", fmt.Errorf( + "%w: from --config: %s (run 'vaultik config init --config %s' to create it)", + errConfigNotFound, path, path) } return path, nil } if path := os.Getenv("VAULTIK_CONFIG"); path != "" { - if _, err := os.Stat(path); err != nil { - return "", fmt.Errorf("config file from $VAULTIK_CONFIG not found: %s (unset VAULTIK_CONFIG, point it at an existing file, or run 'vaultik config init')", path) + _, err := os.Stat(path) //nolint:gosec // G703: path is operator-supplied by design + if err != nil { + return "", fmt.Errorf( + "%w: from $VAULTIK_CONFIG: %s (unset VAULTIK_CONFIG, point it at "+ + "an existing file, or run 'vaultik config init')", + errConfigNotFound, path) } return path, nil } for _, path := range defaultConfigPaths() { - if _, err := os.Stat(path); err == nil { + _, err := os.Stat(path) + if err == nil { return path, nil } } - return "", fmt.Errorf("no config file found at %s (run 'vaultik config init' to create the default config, or pass --config )", strings.Join(defaultConfigPaths(), " or ")) + return "", fmt.Errorf( + "%w: searched %s (run 'vaultik config init' to create the default "+ + "config, or pass --config )", + errConfigNotFound, strings.Join(defaultConfigPaths(), " or ")) } // defaultConfigPaths returns the ordered list of config paths to search. diff --git a/internal/cli/snapshot.go b/internal/cli/snapshot.go index 5f5e088..9d94daa 100644 --- a/internal/cli/snapshot.go +++ b/internal/cli/snapshot.go @@ -12,6 +12,32 @@ import ( "sneak.berlin/go/vaultik/internal/vaultik" ) +var ( + errSnapshotIDRequired = errors.New("snapshot ID required") + errWrongArgCount = errors.New("wrong argument count") + errPurgeCriteriaNeeded = errors.New( + "must specify either --keep-latest or --older-than") + errPurgeCriteriaBoth = errors.New( + "cannot specify both --keep-latest and --older-than") +) + +// requireSnapshotIDArg validates that exactly one positional argument +// (the snapshot ID) was supplied, printing help otherwise. +func requireSnapshotIDArg(cmd *cobra.Command, args []string) error { + if len(args) != 1 { + _ = cmd.Help() + + if len(args) == 0 { + return errSnapshotIDRequired + } + + return fmt.Errorf("%w: expected 1 argument, got %d", + errWrongArgCount, len(args)) + } + + return nil +} + // NewSnapshotCommand creates the snapshot command and subcommands func NewSnapshotCommand() *cobra.Command { cmd := &cobra.Command{ @@ -62,7 +88,7 @@ specifying a path using --config or by setting VAULTIK_CONFIG to a path.`, return RunWithApp(cmd.Context(), AppOptions{ ConfigPath: configPath, - LogOptions: log.LogOptions{ + LogOptions: log.Options{ Verbose: rootFlags.Verbose, Debug: rootFlags.Debug, Cron: opts.Cron, @@ -72,7 +98,7 @@ specifying a path using --config or by setting VAULTIK_CONFIG to a path.`, Invokes: []fx.Option{ fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) { lc.Append(fx.Hook{ - OnStart: func(ctx context.Context) error { + OnStart: func(_ context.Context) error { // Start the snapshot creation in a goroutine go func() { // --cron suppression is wired through v.UI by setupGlobals. @@ -94,7 +120,7 @@ specifying a path using --config or by setting VAULTIK_CONFIG to a path.`, return nil }, - OnStop: func(ctx context.Context) error { + OnStop: func(_ context.Context) error { log.Debug("Stopping snapshot creation") // Cancel the Vaultik context v.Cancel() @@ -108,9 +134,14 @@ specifying a path using --config or by setting VAULTIK_CONFIG to a path.`, }, } - cmd.Flags().BoolVar(&opts.Cron, "cron", false, "Run in cron mode (silent unless error)") - cmd.Flags().BoolVar(&opts.Prune, "prune", false, "After backup, drop older snapshots of the same name and remove orphaned blobs") - cmd.Flags().StringVar(&opts.KeepNewerThan, "keep-newer-than", "", "With --prune: keep snapshots newer than this duration (e.g. 4w, 30d, 6mo) instead of only the latest") + cmd.Flags().BoolVar(&opts.Cron, "cron", false, + "Run in cron mode (silent unless error)") + cmd.Flags().BoolVar(&opts.Prune, "prune", false, + "After backup, drop older snapshots of the same name and remove "+ + "orphaned blobs") + cmd.Flags().StringVar(&opts.KeepNewerThan, "keep-newer-than", "", + "With --prune: keep snapshots newer than this duration "+ + "(e.g. 4w, 30d, 6mo) instead of only the latest") return cmd } @@ -125,54 +156,12 @@ func newSnapshotListCommand() *cobra.Command { Short: "List all snapshots", Long: "Lists all snapshots with their ID, timestamp, and compressed size", Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { - // Use unified config resolution - configPath, err := ResolveConfigPath() - if err != nil { - return err - } - - rootFlags := GetRootFlags() - - return RunWithApp(cmd.Context(), AppOptions{ - ConfigPath: configPath, - LogOptions: log.LogOptions{ - Verbose: rootFlags.Verbose, - Debug: rootFlags.Debug, - Quiet: rootFlags.Quiet, - }, - Modules: []fx.Option{}, - Invokes: []fx.Option{ - fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) { - lc.Append(fx.Hook{ - OnStart: func(ctx context.Context) error { - go func() { - err := v.ListSnapshots(jsonOutput) - if err != nil { - if !errors.Is(err, context.Canceled) { - log.Error("Failed to list snapshots", "error", err) - ReportErrorf("Failed to list snapshots: %v", err) - os.Exit(1) - } - } - - err = v.Shutdowner.Shutdown() - if err != nil { - log.Error("Failed to shutdown", "error", err) - } - }() - - return nil - }, - OnStop: func(ctx context.Context) error { - v.Cancel() - - return nil - }, - }) - }), - }, - }) + RunE: func(cmd *cobra.Command, _ []string) error { + return runVaultikApp(cmd, false, false, + "Failed to list snapshots", + func(v *vaultik.Vaultik) error { + return v.ListSnapshots(jsonOutput) + }) }, } @@ -194,70 +183,31 @@ Retention is per-snapshot-name: --keep-latest keeps the latest of each configured snapshot name, not the latest globally. Use --snapshot to restrict the operation to specific snapshot names.`, Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, _ []string) error { // Validate flags if !opts.KeepLatest && opts.OlderThan == "" { - return errors.New("must specify either --keep-latest or --older-than") + return errPurgeCriteriaNeeded } if opts.KeepLatest && opts.OlderThan != "" { - return errors.New("cannot specify both --keep-latest and --older-than") + return errPurgeCriteriaBoth } - // Use unified config resolution - configPath, err := ResolveConfigPath() - if err != nil { - return err - } - - rootFlags := GetRootFlags() - - return RunWithApp(cmd.Context(), AppOptions{ - ConfigPath: configPath, - LogOptions: log.LogOptions{ - Verbose: rootFlags.Verbose, - Debug: rootFlags.Debug, - Quiet: rootFlags.Quiet, - }, - Modules: []fx.Option{}, - Invokes: []fx.Option{ - fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) { - lc.Append(fx.Hook{ - OnStart: func(ctx context.Context) error { - go func() { - err := v.PurgeSnapshotsWithOptions(opts) - if err != nil { - if !errors.Is(err, context.Canceled) { - log.Error("Failed to purge snapshots", "error", err) - ReportErrorf("Failed to purge snapshots: %v", err) - os.Exit(1) - } - } - - err = v.Shutdowner.Shutdown() - if err != nil { - log.Error("Failed to shutdown", "error", err) - } - }() - - return nil - }, - OnStop: func(ctx context.Context) error { - v.Cancel() - - return nil - }, - }) - }), - }, - }) + return runVaultikApp(cmd, false, false, + "Failed to purge snapshots", + func(v *vaultik.Vaultik) error { + return v.PurgeSnapshotsWithOptions(opts) + }) }, } - cmd.Flags().BoolVar(&opts.KeepLatest, "keep-latest", false, "Keep only the latest snapshot of each name") - cmd.Flags().StringVar(&opts.OlderThan, "older-than", "", "Remove snapshots older than duration (e.g., 30d, 6m, 1y)") + cmd.Flags().BoolVar(&opts.KeepLatest, "keep-latest", false, + "Keep only the latest snapshot of each name") + cmd.Flags().StringVar(&opts.OlderThan, "older-than", "", + "Remove snapshots older than duration (e.g., 30d, 6m, 1y)") cmd.Flags().BoolVar(&opts.Force, "force", false, "Skip confirmation prompt") - cmd.Flags().StringArrayVar(&opts.Names, "snapshot", nil, "Restrict to snapshots with these names (repeat for multiple)") + cmd.Flags().StringArrayVar(&opts.Names, "snapshot", nil, + "Restrict to snapshots with these names (repeat for multiple)") return cmd } @@ -270,19 +220,7 @@ func newSnapshotVerifyCommand() *cobra.Command { Use: "verify ", Short: "Verify snapshot integrity", Long: "Verifies that all blobs referenced in a snapshot exist", - Args: func(cmd *cobra.Command, args []string) error { - if len(args) != 1 { - _ = cmd.Help() - - if len(args) == 0 { - return errors.New("snapshot ID required") - } - - return fmt.Errorf("expected 1 argument, got %d", len(args)) - } - - return nil - }, + Args: requireSnapshotIDArg, RunE: func(cmd *cobra.Command, args []string) error { snapshotID := args[0] @@ -296,7 +234,7 @@ func newSnapshotVerifyCommand() *cobra.Command { return RunWithApp(cmd.Context(), AppOptions{ ConfigPath: configPath, - LogOptions: log.LogOptions{ + LogOptions: log.Options{ Verbose: rootFlags.Verbose, Debug: rootFlags.Debug, Quiet: rootFlags.Quiet || opts.JSON, @@ -305,7 +243,7 @@ func newSnapshotVerifyCommand() *cobra.Command { Invokes: []fx.Option{ fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) { lc.Append(fx.Hook{ - OnStart: func(ctx context.Context) error { + OnStart: func(_ context.Context) error { go func() { err := v.VerifySnapshotWithOptions(snapshotID, opts) if err != nil { @@ -327,7 +265,7 @@ func newSnapshotVerifyCommand() *cobra.Command { return nil }, - OnStop: func(ctx context.Context) error { + OnStop: func(_ context.Context) error { v.Cancel() return nil @@ -371,76 +309,24 @@ is reachable to finish remote cleanup. To wipe the entire destination store and start over, use 'vaultik remote nuke --force' — it is the single supported entry point for that.`, - Args: func(cmd *cobra.Command, args []string) error { - if len(args) != 1 { - _ = cmd.Help() - - if len(args) == 0 { - return errors.New("snapshot ID required") - } - - return fmt.Errorf("expected 1 argument, got %d", len(args)) - } - - return nil - }, + Args: requireSnapshotIDArg, RunE: func(cmd *cobra.Command, args []string) error { - // Use unified config resolution - configPath, err := ResolveConfigPath() - if err != nil { - return err - } + return runVaultikApp(cmd, opts.JSON, opts.JSON, + "Failed to remove snapshot", + func(v *vaultik.Vaultik) error { + _, err := v.RemoveSnapshot(args[0], opts) - rootFlags := GetRootFlags() - - return RunWithApp(cmd.Context(), AppOptions{ - ConfigPath: configPath, - LogOptions: log.LogOptions{ - Verbose: rootFlags.Verbose, - Debug: rootFlags.Debug, - Quiet: rootFlags.Quiet || opts.JSON, - }, - Modules: []fx.Option{}, - Invokes: []fx.Option{ - fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) { - lc.Append(fx.Hook{ - OnStart: func(ctx context.Context) error { - go func() { - _, err := v.RemoveSnapshot(args[0], opts) - if err != nil { - if !errors.Is(err, context.Canceled) { - if !opts.JSON { - log.Error("Failed to remove snapshot", "error", err) - ReportErrorf("Failed to remove snapshot: %v", err) - } - - os.Exit(1) - } - } - - if err := v.Shutdowner.Shutdown(); err != nil { - log.Error("Failed to shutdown", "error", err) - } - }() - - return nil - }, - OnStop: func(ctx context.Context) error { - v.Cancel() - - return nil - }, - }) - }), - }, - }) + return err + }) }, } cmd.Flags().BoolVarP(&opts.Force, "force", "f", false, "Skip confirmation prompt") - cmd.Flags().BoolVar(&opts.DryRun, "dry-run", false, "Show what would be removed without removing") + cmd.Flags().BoolVar(&opts.DryRun, "dry-run", false, + "Show what would be removed without removing") cmd.Flags().BoolVar(&opts.JSON, "json", false, "Output result as JSON") - cmd.Flags().BoolVar(&opts.LocalOnly, "local-only", false, "Skip remote cleanup; only touch the local index") + cmd.Flags().BoolVar(&opts.LocalOnly, "local-only", false, + "Skip remote cleanup; only touch the local index") return cmd } diff --git a/internal/cli/snapshot_restore.go b/internal/cli/snapshot_restore.go index 4f5d878..cf28e8e 100644 --- a/internal/cli/snapshot_restore.go +++ b/internal/cli/snapshot_restore.go @@ -14,6 +14,10 @@ import ( "sneak.berlin/go/vaultik/internal/vaultik" ) +// restoreMinArgs is the minimum positional argument count of +// `snapshot restore [paths...]`. +const restoreMinArgs = 2 + // RestoreOptions contains options for the restore command type RestoreOptions struct { TargetDir string @@ -39,31 +43,36 @@ func newSnapshotRestoreCommand() *cobra.Command { Short: "Restore files from a snapshot", Long: `Download and decrypt files from a backup snapshot. -This command will restore files from the specified snapshot to the target directory. +This command will restore files from the specified snapshot to the +target directory. If no paths are specified, all files are restored. If paths are specified, only matching files/directories are restored. -Requires the VAULTIK_AGE_SECRET_KEY environment variable to be set with the age private key. +Requires the VAULTIK_AGE_SECRET_KEY environment variable to be set with +the age private key. Examples: # Restore entire snapshot vaultik snapshot restore myhost_docs_2025-01-01T12:00:00Z /restore # Restore specific file - vaultik snapshot restore myhost_docs_2025-01-01T12:00:00Z /restore /home/user/important.txt + vaultik snapshot restore myhost_docs_2025-01-01T12:00:00Z /restore \ + /home/user/important.txt # Restore specific directory - vaultik snapshot restore myhost_docs_2025-01-01T12:00:00Z /restore /home/user/documents/ + vaultik snapshot restore myhost_docs_2025-01-01T12:00:00Z /restore \ + /home/user/documents/ # Restore and verify all files vaultik snapshot restore --verify myhost_docs_2025-01-01T12:00:00Z /restore`, - Args: cobra.MinimumNArgs(2), + Args: cobra.MinimumNArgs(restoreMinArgs), RunE: func(cmd *cobra.Command, args []string) error { return runRestore(cmd, args, opts) }, } - cmd.Flags().BoolVar(&opts.Verify, "verify", false, "Verify restored files by checking chunk hashes") + cmd.Flags().BoolVar(&opts.Verify, "verify", false, + "Verify restored files by checking chunk hashes") return cmd } @@ -73,8 +82,8 @@ func runRestore(cmd *cobra.Command, args []string, opts *RestoreOptions) error { snapshotID := args[0] opts.TargetDir = args[1] - if len(args) > 2 { - opts.Paths = args[2:] + if len(args) > restoreMinArgs { + opts.Paths = args[restoreMinArgs:] } // Use unified config resolution @@ -88,7 +97,7 @@ func runRestore(cmd *cobra.Command, args []string, opts *RestoreOptions) error { return RunWithApp(cmd.Context(), AppOptions{ ConfigPath: configPath, - LogOptions: log.LogOptions{ + LogOptions: log.Options{ Verbose: rootFlags.Verbose, Debug: rootFlags.Debug, Quiet: rootFlags.Quiet, @@ -121,7 +130,7 @@ func buildRestoreInvokes(snapshotID string, opts *RestoreOptions) []fx.Option { return []fx.Option{ fx.Invoke(func(app *RestoreApp, lc fx.Lifecycle) { lc.Append(fx.Hook{ - OnStart: func(ctx context.Context) error { + OnStart: func(_ context.Context) error { // Start the restore operation in a goroutine go func() { // Run the restore operation @@ -151,7 +160,7 @@ func buildRestoreInvokes(snapshotID string, opts *RestoreOptions) []fx.Option { return nil }, - OnStop: func(ctx context.Context) error { + OnStop: func(_ context.Context) error { log.Debug("Stopping restore operation") app.Vaultik.Cancel() diff --git a/internal/cli/vaultik_snapshot_types.go b/internal/cli/vaultik_snapshot_types.go index 9d6b7c6..1f9d4dd 100644 --- a/internal/cli/vaultik_snapshot_types.go +++ b/internal/cli/vaultik_snapshot_types.go @@ -3,6 +3,8 @@ package cli import "time" // SnapshotInfo represents snapshot information for listing +// +//nolint:tagliatelle // snake_case is the established output format type SnapshotInfo struct { ID string `json:"id"` Timestamp time.Time `json:"timestamp"` diff --git a/internal/cli/version.go b/internal/cli/version.go index 7c6d14c..c8cec85 100644 --- a/internal/cli/version.go +++ b/internal/cli/version.go @@ -2,6 +2,7 @@ package cli import ( "fmt" + "os" "runtime" "github.com/spf13/cobra" @@ -15,21 +16,25 @@ func NewVersionCommand() *cobra.Command { Short: "Print version information", Long: `Print version, git commit, and build information for vaultik.`, Args: cobra.NoArgs, - Run: func(cmd *cobra.Command, args []string) { - fmt.Printf("vaultik %s\n", globals.Version) - fmt.Printf(" commit: %s\n", globals.Commit) - fmt.Printf(" build date: %s\n", globals.CommitDate) - fmt.Printf(" go: %s\n", runtime.Version()) - fmt.Printf(" os/arch: %s/%s\n", runtime.GOOS, runtime.GOARCH) - fmt.Printf(" author: %s\n", globals.Author) - fmt.Printf(" homepage: %s\n", globals.Homepage) - fmt.Printf(" license: %s\n", globals.License) + Run: func(_ *cobra.Command, _ []string) { + _, _ = fmt.Fprintf(os.Stdout, "vaultik %s\n", globals.Version) + _, _ = fmt.Fprintf(os.Stdout, " commit: %s\n", globals.Commit) + _, _ = fmt.Fprintf(os.Stdout, " build date: %s\n", globals.CommitDate) + _, _ = fmt.Fprintf(os.Stdout, " go: %s\n", runtime.Version()) + _, _ = fmt.Fprintf(os.Stdout, " os/arch: %s/%s\n", + runtime.GOOS, runtime.GOARCH) + _, _ = fmt.Fprintf(os.Stdout, " author: %s\n", globals.Author) + _, _ = fmt.Fprintf(os.Stdout, " homepage: %s\n", globals.Homepage) + _, _ = fmt.Fprintf(os.Stdout, " license: %s\n", globals.License) if globals.Version == "dev" { - fmt.Println() - fmt.Println("This is a development build (no version information embedded).") - fmt.Println("Build a release binary with 'make vaultik' or download from") - fmt.Println("https://sneak.berlin/go/vaultik for embedded version metadata.") + _, _ = fmt.Fprintln(os.Stdout) + _, _ = fmt.Fprintln(os.Stdout, + "This is a development build (no version information embedded).") + _, _ = fmt.Fprintln(os.Stdout, + "Build a release binary with 'make vaultik' or download from") + _, _ = fmt.Fprintln(os.Stdout, + "https://sneak.berlin/go/vaultik for embedded version metadata.") } }, } diff --git a/internal/config/config.go b/internal/config/config.go index cabf735..27e7196 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1,3 +1,6 @@ +// Package config loads, validates, and provides the vaultik YAML +// configuration, including snapshot definitions, encryption recipients, +// and storage settings. package config import ( @@ -18,6 +21,38 @@ import ( const appName = "vaultik" +// Defaults and validation bounds for tunable settings. +const ( + defaultBlobSizeLimit = Size(10 * 1024 * 1024 * 1024) // 10GB + defaultChunkSize = Size(10 * 1024 * 1024) // 10MB + defaultS3PartSize = Size(5 * 1024 * 1024) // 5MB + defaultCompressionLevel = 3 + minChunkSize = 1024 * 1024 // 1MB + minCompressionLevel = 1 + maxCompressionLevel = 19 +) + +// Sentinel validation errors. +var ( + errNoConfigPath = errors.New("config path not provided") + errNoAgeRecipients = errors.New( + "at least one age_recipient is required (generate with: age-keygen)") + errNoSnapshots = errors.New( + "at least one snapshot must be configured (see config.example.yml)") + errSnapshotNoPaths = errors.New("snapshot must have at least one path") + errChunkSizeTooSmall = errors.New("chunk_size must be at least 1MB") + errBlobSizeTooSmall = errors.New("blob_size_limit must be at least chunk_size") + errBadCompression = errors.New("compression_level must be between 1 and 19") + errBadStorageScheme = errors.New( + "storage_url must start with s3://, file://, or rclone://") + errStorageNotConfigured = errors.New( + "storage not configured; set storage_url or provide s3.endpoint + " + + "s3.bucket + credentials") + errS3BucketRequired = errors.New("s3.bucket is required (or set storage_url)") + errS3KeyIDRequired = errors.New("s3.access_key_id is required") + errS3SecretRequired = errors.New("s3.secret_access_key is required") +) + // expandTilde expands ~ at the start of a path to the user's home directory. func expandTilde(path string) string { if path == "~" { @@ -90,12 +125,15 @@ func (c *Config) SnapshotNames() []string { // It defines all settings for backup operations, including source directories, // encryption recipients, storage configuration, and performance tuning parameters. // Configuration is typically loaded from a YAML file. +// +//nolint:tagliatelle // snake_case is the established config-file format type Config struct { - AgeRecipients []string `yaml:"age_recipients"` - AgeSecretKey string `yaml:"age_secret_key"` - BlobSizeLimit Size `yaml:"blob_size_limit"` - ChunkSize Size `yaml:"chunk_size"` - Exclude []string `yaml:"exclude"` // Global excludes applied to all snapshots + AgeRecipients []string `yaml:"age_recipients"` + AgeSecretKey string `yaml:"age_secret_key"` + BlobSizeLimit Size `yaml:"blob_size_limit"` + ChunkSize Size `yaml:"chunk_size"` + // Exclude holds global excludes applied to all snapshots. + Exclude []string `yaml:"exclude"` Hostname string `yaml:"hostname"` IndexPath string `yaml:"index_path"` S3 S3Config `yaml:"s3"` @@ -107,13 +145,16 @@ type Config struct { // Supported formats: // - s3://bucket/prefix?endpoint=host®ion=us-east-1 // - file:///path/to/backup - // For S3 URLs, credentials are still read from s3.access_key_id and s3.secret_access_key. + // For S3 URLs, credentials are still read from s3.access_key_id + // and s3.secret_access_key. StorageURL string `yaml:"storage_url"` } // S3Config represents S3 storage configuration for backup storage. // It supports both AWS S3 and S3-compatible storage services. // All fields except UseSSL and PartSize are required. +// +//nolint:tagliatelle // snake_case is the established config-file format type S3Config struct { Endpoint string `yaml:"endpoint"` Bucket string `yaml:"bucket"` @@ -125,17 +166,17 @@ type S3Config struct { PartSize Size `yaml:"part_size"` } -// ConfigPath wraps the config file path for fx dependency injection. +// Path wraps the config file path for fx dependency injection. // This type allows the config file path to be injected as a distinct type // rather than a plain string, avoiding conflicts with other string dependencies. -type ConfigPath string +type Path string // New creates a new Config instance by loading from the specified path. // This function is used by the fx dependency injection framework. // Returns an error if the path is empty or if loading fails. -func New(path ConfigPath) (*Config, error) { +func New(path Path) (*Config, error) { if path == "" { - return nil, errors.New("config path not provided") + return nil, errNoConfigPath } cfg, err := Load(string(path)) @@ -160,10 +201,10 @@ func Load(path string) (*Config, error) { cfg := &Config{ // Set defaults - BlobSizeLimit: Size(10 * 1024 * 1024 * 1024), // 10GB - ChunkSize: Size(10 * 1024 * 1024), // 10MB + BlobSizeLimit: defaultBlobSizeLimit, + ChunkSize: defaultChunkSize, IndexPath: filepath.Join(xdg.DataHome, appName, "index.sqlite"), - CompressionLevel: 3, + CompressionLevel: defaultCompressionLevel, } // Convert smartconfig data to YAML then unmarshal @@ -174,7 +215,8 @@ func Load(path string) (*Config, error) { return nil, fmt.Errorf("failed to marshal config data: %w", err) } - if err := yaml.Unmarshal(yamlBytes, cfg); err != nil { + err = yaml.Unmarshal(yamlBytes, cfg) + if err != nil { return nil, fmt.Errorf("failed to parse config: %w", err) } @@ -217,11 +259,13 @@ func Load(path string) (*Config, error) { } if cfg.S3.PartSize == 0 { - cfg.S3.PartSize = Size(5 * 1024 * 1024) // 5MB + cfg.S3.PartSize = defaultS3PartSize } // Check config file permissions (warn if world or group readable) - if info, err := os.Stat(path); err == nil { + //nolint:gosec // G703: config path is operator-supplied by design + info, statErr := os.Stat(path) + if statErr == nil { mode := info.Mode().Perm() if mode&0044 != 0 { // group or world readable log.Warn("Config file has insecure permissions (contains S3 credentials)", @@ -231,7 +275,8 @@ func Load(path string) (*Config, error) { } } - if err := cfg.Validate(); err != nil { + err = cfg.Validate() + if err != nil { return nil, fmt.Errorf("invalid config: %w", err) } @@ -249,16 +294,16 @@ func Load(path string) (*Config, error) { // Returns an error describing the first validation failure encountered. func (c *Config) Validate() error { if len(c.AgeRecipients) == 0 { - return errors.New("at least one age_recipient is required (generate with: age-keygen)") + return errNoAgeRecipients } if len(c.Snapshots) == 0 { - return errors.New("at least one snapshot must be configured (see config.example.yml)") + return errNoSnapshots } for name, snap := range c.Snapshots { if len(snap.Paths) == 0 { - return fmt.Errorf("snapshot %q must have at least one path", name) + return fmt.Errorf("%w: %q", errSnapshotNoPaths, name) } } @@ -268,16 +313,17 @@ func (c *Config) Validate() error { return err } - if c.ChunkSize.Int64() < 1024*1024 { // 1MB minimum - return errors.New("chunk_size must be at least 1MB") + if c.ChunkSize.Int64() < minChunkSize { + return errChunkSizeTooSmall } if c.BlobSizeLimit.Int64() < c.ChunkSize.Int64() { - return errors.New("blob_size_limit must be at least chunk_size") + return errBlobSizeTooSmall } - if c.CompressionLevel < 1 || c.CompressionLevel > 19 { - return errors.New("compression_level must be between 1 and 19") + if c.CompressionLevel < minCompressionLevel || + c.CompressionLevel > maxCompressionLevel { + return errBadCompression } return nil @@ -289,53 +335,56 @@ func (c *Config) Validate() error { // If StorageURL is not set, legacy S3 configuration is required. func (c *Config) validateStorage() error { if c.StorageURL != "" { - // URL-based configuration - if strings.HasPrefix(c.StorageURL, "file://") { - // File storage doesn't need S3 credentials - return nil - } - - if strings.HasPrefix(c.StorageURL, "s3://") { - // S3 storage needs credentials - if c.S3.AccessKeyID == "" { - return errors.New("s3.access_key_id is required for s3:// URLs") - } - - if c.S3.SecretAccessKey == "" { - return errors.New("s3.secret_access_key is required for s3:// URLs") - } - - return nil - } - - if strings.HasPrefix(c.StorageURL, "rclone://") { - // Rclone storage uses rclone's own config - return nil - } - - return errors.New("storage_url must start with s3://, file://, or rclone://") + return c.validateStorageURL() } // Legacy S3 configuration if c.S3.Endpoint == "" { - return errors.New("storage not configured; set storage_url or provide s3.endpoint + s3.bucket + credentials") + return errStorageNotConfigured } if c.S3.Bucket == "" { - return errors.New("s3.bucket is required (or set storage_url)") + return errS3BucketRequired } if c.S3.AccessKeyID == "" { - return errors.New("s3.access_key_id is required") + return errS3KeyIDRequired } if c.S3.SecretAccessKey == "" { - return errors.New("s3.secret_access_key is required") + return errS3SecretRequired } return nil } +// validateStorageURL validates URL-based storage configuration. File and +// rclone URLs need no credentials; S3 URLs require the legacy s3.* +// credential fields. +func (c *Config) validateStorageURL() error { + switch { + case strings.HasPrefix(c.StorageURL, "file://"): + // File storage doesn't need S3 credentials + return nil + case strings.HasPrefix(c.StorageURL, "rclone://"): + // Rclone storage uses rclone's own config + return nil + case strings.HasPrefix(c.StorageURL, "s3://"): + // S3 storage needs credentials + if c.S3.AccessKeyID == "" { + return fmt.Errorf("%w for s3:// URLs", errS3KeyIDRequired) + } + + if c.S3.SecretAccessKey == "" { + return fmt.Errorf("%w for s3:// URLs", errS3SecretRequired) + } + + return nil + default: + return errBadStorageScheme + } +} + // extractAgeSecretKey extracts the AGE-SECRET-KEY from the input using // the age library's parser, which handles comments and whitespace. func extractAgeSecretKey(input string) string { @@ -354,6 +403,8 @@ func extractAgeSecretKey(input string) string { // Module exports the config module for fx dependency injection. // It provides the Config type to other modules in the application. +// +//nolint:gochecknoglobals // fx module definitions are package globals var Module = fx.Module("config", fx.Provide(New), ) diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 08d3bce..0dfdd34 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -1,4 +1,4 @@ -package config +package config //nolint:testpackage // exercises unexported extractAgeSecretKey import ( "os" @@ -7,15 +7,20 @@ import ( ) const ( - TEST_SNEAK_AGE_PUBLIC_KEY = "age1278m9q7dp3chsh2dcy82qk27v047zywyvtxwnj4cvt0z65jw6a7q5dqhfj" - TEST_INTEGRATION_AGE_PUBLIC_KEY = "age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05gl0sjq9q9wjg" - TEST_INTEGRATION_AGE_PRIVATE_KEY = "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5" + testSneakAgePublicKey = "age1278m9q7dp3chsh2dcy82qk27v047zywyvt" + + "xwnj4cvt0z65jw6a7q5dqhfj" + testIntegrationAgePublicKey = "age1ezrjmfpwsc95svdg0y54mums3zevgzu" + + "0x0ecq2f7tp8a05gl0sjq9q9wjg" + testIntegrationAgePrivateKey = "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GX" + + "VEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5" ) func TestMain(m *testing.M) { // Set up test environment testConfigPath := filepath.Join("..", "..", "test", "config.yaml") - if absPath, err := filepath.Abs(testConfigPath); err == nil { + + absPath, err := filepath.Abs(testConfigPath) + if err == nil { _ = os.Setenv("VAULTIK_CONFIG", absPath) } @@ -23,8 +28,11 @@ func TestMain(m *testing.M) { os.Exit(code) } -// TestConfigLoad ensures the config package can be imported and basic functionality works +// TestConfigLoad ensures the config package can be imported and basic +// functionality works. func TestConfigLoad(t *testing.T) { + t.Parallel() + // Use the test config file configPath := os.Getenv("VAULTIK_CONFIG") if configPath == "" { @@ -42,8 +50,9 @@ func TestConfigLoad(t *testing.T) { t.Errorf("Expected 2 age recipients, got %d", len(cfg.AgeRecipients)) } - if cfg.AgeRecipients[0] != TEST_SNEAK_AGE_PUBLIC_KEY { - t.Errorf("Expected first age recipient to be %s, got '%s'", TEST_SNEAK_AGE_PUBLIC_KEY, cfg.AgeRecipients[0]) + if cfg.AgeRecipients[0] != testSneakAgePublicKey { + t.Errorf("Expected first age recipient to be %s, got '%s'", + testSneakAgePublicKey, cfg.AgeRecipients[0]) } if len(cfg.Snapshots) != 1 { @@ -60,11 +69,13 @@ func TestConfigLoad(t *testing.T) { } if testSnap.Paths[0] != "/tmp/vaultik-test-source" { - t.Errorf("Expected first path to be '/tmp/vaultik-test-source', got '%s'", testSnap.Paths[0]) + t.Errorf("Expected first path to be '/tmp/vaultik-test-source', got '%s'", + testSnap.Paths[0]) } if cfg.S3.Bucket != "vaultik-test-bucket" { - t.Errorf("Expected S3 bucket to be 'vaultik-test-bucket', got '%s'", cfg.S3.Bucket) + t.Errorf("Expected S3 bucket to be 'vaultik-test-bucket', got '%s'", + cfg.S3.Bucket) } if cfg.Hostname != "test-host" { @@ -74,19 +85,26 @@ func TestConfigLoad(t *testing.T) { // TestConfigFromEnv tests loading config path from environment variable func TestConfigFromEnv(t *testing.T) { + t.Parallel() + configPath := os.Getenv("VAULTIK_CONFIG") if configPath == "" { t.Skip("VAULTIK_CONFIG not set") } // Verify the file exists - if _, err := os.Stat(configPath); os.IsNotExist(err) { - t.Errorf("Config file does not exist at path from VAULTIK_CONFIG: %s", configPath) + //nolint:gosec // G703: test config path comes from the test environment + _, err := os.Stat(configPath) + if os.IsNotExist(err) { + t.Errorf("Config file does not exist at path from VAULTIK_CONFIG: %s", + configPath) } } // TestExtractAgeSecretKey tests extraction of AGE-SECRET-KEY from various inputs func TestExtractAgeSecretKey(t *testing.T) { + t.Parallel() + tests := []struct { name string input string @@ -94,36 +112,32 @@ func TestExtractAgeSecretKey(t *testing.T) { }{ { name: "plain key", - input: "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5", - expected: "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5", + input: testIntegrationAgePrivateKey, + expected: testIntegrationAgePrivateKey, }, { name: "key with trailing newline", - input: "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5\n", - expected: "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5", + input: testIntegrationAgePrivateKey + "\n", + expected: testIntegrationAgePrivateKey, }, { name: "full age-keygen output", - input: `# created: 2025-01-14T12:00:00Z -# public key: age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05gl0sjq9q9wjg -AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5 -`, - expected: "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5", + input: "# created: 2025-01-14T12:00:00Z\n" + + "# public key: " + testIntegrationAgePublicKey + "\n" + + testIntegrationAgePrivateKey + "\n", + expected: testIntegrationAgePrivateKey, }, { name: "age-keygen output with extra blank lines", - input: `# created: 2025-01-14T12:00:00Z -# public key: age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05gl0sjq9q9wjg - -AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5 - -`, - expected: "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5", + input: "# created: 2025-01-14T12:00:00Z\n" + + "# public key: " + testIntegrationAgePublicKey + "\n\n" + + testIntegrationAgePrivateKey + "\n\n", + expected: testIntegrationAgePrivateKey, }, { name: "key with leading whitespace", - input: " AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5 ", - expected: "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5", + input: " " + testIntegrationAgePrivateKey + " ", + expected: testIntegrationAgePrivateKey, }, { name: "empty input", @@ -139,9 +153,12 @@ AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5 for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + t.Parallel() + result := extractAgeSecretKey(tt.input) if result != tt.expected { - t.Errorf("extractAgeSecretKey(%q) = %q, want %q", tt.input, result, tt.expected) + t.Errorf("extractAgeSecretKey(%q) = %q, want %q", + tt.input, result, tt.expected) } }) } diff --git a/internal/config/size.go b/internal/config/size.go index 58dbf73..8f21a87 100644 --- a/internal/config/size.go +++ b/internal/config/size.go @@ -3,13 +3,21 @@ package config import ( "errors" "fmt" + "math" "github.com/dustin/go-humanize" ) +var ( + errSizeType = errors.New("size must be a number or string") + errSizeTooLarge = errors.New("size exceeds maximum supported value") +) + // Size represents a byte size that can be specified in configuration files. // It can unmarshal from both numeric values (interpreted as bytes) and // human-readable strings like "10MB", "2.5GB", or "1TB". +// +//nolint:recvcheck // UnmarshalYAML requires a pointer; String/Int64 are value reads type Size int64 // UnmarshalYAML implements yaml.Unmarshaler for Size, allowing it to be @@ -18,7 +26,9 @@ type Size int64 func (s *Size) UnmarshalYAML(unmarshal func(any) error) error { // Try to unmarshal as int64 first var intVal int64 - if err := unmarshal(&intVal); err == nil { + + err := unmarshal(&intVal) + if err == nil { *s = Size(intVal) return nil @@ -26,8 +36,10 @@ func (s *Size) UnmarshalYAML(unmarshal func(any) error) error { // Try to unmarshal as string var strVal string - if err := unmarshal(&strVal); err != nil { - return errors.New("size must be a number or string") + + err = unmarshal(&strVal) + if err != nil { + return errSizeType } // Parse the string using go-humanize @@ -36,6 +48,10 @@ func (s *Size) UnmarshalYAML(unmarshal func(any) error) error { return fmt.Errorf("invalid size format: %w", err) } + if bytes > math.MaxInt64 { + return fmt.Errorf("%w: %s", errSizeTooLarge, strVal) + } + *s = Size(bytes) return nil @@ -52,6 +68,7 @@ func (s Size) Int64() int64 { // For example, 1048576 bytes would be formatted as "1.0 MB". // This implements the fmt.Stringer interface. func (s Size) String() string { + //nolint:gosec // G115: sizes are non-negative by construction return humanize.Bytes(uint64(s)) } @@ -62,5 +79,9 @@ func ParseSize(s string) (Size, error) { return 0, fmt.Errorf("invalid size format: %w", err) } + if bytes > math.MaxInt64 { + return 0, fmt.Errorf("%w: %s", errSizeTooLarge, s) + } + return Size(bytes), nil } diff --git a/internal/crypto/encryption.go b/internal/crypto/encryption.go index d3380a7..36c2564 100644 --- a/internal/crypto/encryption.go +++ b/internal/crypto/encryption.go @@ -1,3 +1,5 @@ +// Package crypto provides thread-safe age encryption and decryption +// helpers used to protect blob and metadata content. package crypto import ( @@ -11,6 +13,10 @@ import ( "go.uber.org/fx" ) +// ErrNoRecipients is returned when an encryptor is created or updated +// without any recipient public keys. +var ErrNoRecipients = errors.New("at least one recipient is required") + // Encryptor provides thread-safe encryption using the age encryption library. // It supports encrypting data for multiple recipients simultaneously, allowing // any of the corresponding private keys to decrypt the data. This is useful @@ -26,7 +32,7 @@ type Encryptor struct { // public keys are invalid or if no recipients are specified. func NewEncryptor(publicKeys []string) (*Encryptor, error) { if len(publicKeys) == 0 { - return nil, errors.New("at least one recipient is required") + return nil, ErrNoRecipients } recipients := make([]age.Recipient, 0, len(publicKeys)) @@ -62,12 +68,14 @@ func (e *Encryptor) Encrypt(data []byte) ([]byte, error) { } // Write data - if _, err := w.Write(data); err != nil { + _, err = w.Write(data) + if err != nil { return nil, fmt.Errorf("writing encrypted data: %w", err) } // Close to flush - if err := w.Close(); err != nil { + err = w.Close() + if err != nil { return nil, fmt.Errorf("closing encrypted writer: %w", err) } @@ -90,12 +98,14 @@ func (e *Encryptor) EncryptStream(dst io.Writer, src io.Reader) error { } // Copy data - if _, err := io.Copy(w, src); err != nil { + _, err = io.Copy(w, src) + if err != nil { return fmt.Errorf("copying encrypted data: %w", err) } // Close to flush - if err := w.Close(); err != nil { + err = w.Close() + if err != nil { return fmt.Errorf("closing encrypted writer: %w", err) } @@ -128,7 +138,7 @@ func (e *Encryptor) EncryptWriter(dst io.Writer) (io.WriteCloser, error) { // of the public keys are invalid or if no recipients are specified. func (e *Encryptor) UpdateRecipients(publicKeys []string) error { if len(publicKeys) == 0 { - return errors.New("at least one recipient is required") + return ErrNoRecipients } recipients := make([]age.Recipient, 0, len(publicKeys)) @@ -209,4 +219,6 @@ func (d *Decryptor) DecryptStream(src io.Reader) (io.Reader, error) { } // Module exports the crypto module for fx dependency injection. +// +//nolint:gochecknoglobals // fx module definitions are package globals var Module = fx.Module("crypto") diff --git a/internal/crypto/encryption_test.go b/internal/crypto/encryption_test.go index 584b216..3f24d47 100644 --- a/internal/crypto/encryption_test.go +++ b/internal/crypto/encryption_test.go @@ -1,13 +1,16 @@ -package crypto +package crypto_test import ( "bytes" "testing" "filippo.io/age" + "sneak.berlin/go/vaultik/internal/crypto" ) func TestEncryptor(t *testing.T) { + t.Parallel() + // Generate a test key pair identity, err := age.GenerateX25519Identity() if err != nil { @@ -17,7 +20,7 @@ func TestEncryptor(t *testing.T) { publicKey := identity.Recipient().String() // Create encryptor - enc, err := NewEncryptor([]string{publicKey}) + enc, err := crypto.NewEncryptor([]string{publicKey}) if err != nil { t.Fatalf("failed to create encryptor: %v", err) } @@ -43,7 +46,9 @@ func TestEncryptor(t *testing.T) { } var decrypted bytes.Buffer - if _, err := decrypted.ReadFrom(r); err != nil { + + _, err = decrypted.ReadFrom(r) + if err != nil { t.Fatalf("failed to read decrypted data: %v", err) } @@ -53,6 +58,8 @@ func TestEncryptor(t *testing.T) { } func TestEncryptorMultipleRecipients(t *testing.T) { + t.Parallel() + // Generate three test key pairs identity1, err := age.GenerateX25519Identity() if err != nil { @@ -76,7 +83,7 @@ func TestEncryptorMultipleRecipients(t *testing.T) { } // Create encryptor with multiple recipients - enc, err := NewEncryptor(publicKeys) + enc, err := crypto.NewEncryptor(publicKeys) if err != nil { t.Fatalf("failed to create encryptor: %v", err) } @@ -99,7 +106,9 @@ func TestEncryptorMultipleRecipients(t *testing.T) { } var decrypted bytes.Buffer - if _, err := decrypted.ReadFrom(r); err != nil { + + _, err = decrypted.ReadFrom(r) + if err != nil { t.Fatalf("recipient %d failed to read decrypted data: %v", i+1, err) } @@ -110,6 +119,8 @@ func TestEncryptorMultipleRecipients(t *testing.T) { } func TestEncryptorUpdateRecipients(t *testing.T) { + t.Parallel() + // Generate two identities identity1, _ := age.GenerateX25519Identity() identity2, _ := age.GenerateX25519Identity() @@ -118,7 +129,7 @@ func TestEncryptorUpdateRecipients(t *testing.T) { publicKey2 := identity2.Recipient().String() // Create encryptor with first key - enc, err := NewEncryptor([]string{publicKey1}) + enc, err := crypto.NewEncryptor([]string{publicKey1}) if err != nil { t.Fatalf("failed to create encryptor: %v", err) } @@ -132,7 +143,8 @@ func TestEncryptorUpdateRecipients(t *testing.T) { } // Update to second key - if err := enc.UpdateRecipients([]string{publicKey2}); err != nil { + err = enc.UpdateRecipients([]string{publicKey2}) + if err != nil { t.Fatalf("failed to update recipients: %v", err) } @@ -143,20 +155,24 @@ func TestEncryptorUpdateRecipients(t *testing.T) { } // First ciphertext should only decrypt with first identity - if _, err := age.Decrypt(bytes.NewReader(ciphertext1), identity1); err != nil { + _, err = age.Decrypt(bytes.NewReader(ciphertext1), identity1) + if err != nil { t.Error("failed to decrypt with identity1") } - if _, err := age.Decrypt(bytes.NewReader(ciphertext1), identity2); err == nil { + _, err = age.Decrypt(bytes.NewReader(ciphertext1), identity2) + if err == nil { t.Error("should not decrypt with identity2") } // Second ciphertext should only decrypt with second identity - if _, err := age.Decrypt(bytes.NewReader(ciphertext2), identity2); err != nil { + _, err = age.Decrypt(bytes.NewReader(ciphertext2), identity2) + if err != nil { t.Error("failed to decrypt with identity2") } - if _, err := age.Decrypt(bytes.NewReader(ciphertext2), identity1); err == nil { + _, err = age.Decrypt(bytes.NewReader(ciphertext2), identity1) + if err == nil { t.Error("should not decrypt with identity1") } } diff --git a/internal/database/blob_chunks.go b/internal/database/blob_chunks.go index 6ff9522..0a009c9 100644 --- a/internal/database/blob_chunks.go +++ b/internal/database/blob_chunks.go @@ -3,18 +3,25 @@ package database import ( "context" "database/sql" + "errors" "fmt" ) +// BlobChunkRepository provides access to the blob_chunks table, which maps +// blobs to the chunks they contain (with offset and length). type BlobChunkRepository struct { db *DB } +// NewBlobChunkRepository creates a BlobChunkRepository backed by db. func NewBlobChunkRepository(db *DB) *BlobChunkRepository { return &BlobChunkRepository{db: db} } -func (r *BlobChunkRepository) Create(ctx context.Context, tx *sql.Tx, bc *BlobChunk) error { +// Create inserts a blob_chunks row, using tx when non-nil. +func (r *BlobChunkRepository) Create( + ctx context.Context, tx *sql.Tx, bc *BlobChunk, +) error { query := ` INSERT INTO blob_chunks (blob_id, chunk_hash, offset, length) VALUES (?, ?, ?, ?) @@ -34,7 +41,11 @@ func (r *BlobChunkRepository) Create(ctx context.Context, tx *sql.Tx, bc *BlobCh return nil } -func (r *BlobChunkRepository) GetByBlobID(ctx context.Context, blobID string) ([]*BlobChunk, error) { +// GetByBlobID returns all chunks contained in the given blob, ordered by +// their offset within the blob. +func (r *BlobChunkRepository) GetByBlobID( + ctx context.Context, blobID string, +) ([]*BlobChunk, error) { query := ` SELECT blob_id, chunk_hash, offset, length FROM blob_chunks @@ -64,7 +75,11 @@ func (r *BlobChunkRepository) GetByBlobID(ctx context.Context, blobID string) ([ return blobChunks, rows.Err() } -func (r *BlobChunkRepository) GetByChunkHash(ctx context.Context, chunkHash string) (*BlobChunk, error) { +// GetByChunkHash returns one blob_chunks row containing the given chunk, +// or nil if the chunk is not packed in any blob. +func (r *BlobChunkRepository) GetByChunkHash( + ctx context.Context, chunkHash string, +) (*BlobChunk, error) { query := ` SELECT blob_id, chunk_hash, offset, length FROM blob_chunks @@ -83,10 +98,10 @@ func (r *BlobChunkRepository) GetByChunkHash(ctx context.Context, chunkHash stri &bc.Length, ) - if err == sql.ErrNoRows { + if errors.Is(err, sql.ErrNoRows) { LogSQL("GetByChunkHash", "No rows found", chunkHash) - return nil, nil + return nil, nil //nolint:nilnil // nil,nil signals not-found; callers check nil } if err != nil { @@ -101,7 +116,9 @@ func (r *BlobChunkRepository) GetByChunkHash(ctx context.Context, chunkHash stri } // GetByChunkHashTx retrieves a blob chunk within a transaction -func (r *BlobChunkRepository) GetByChunkHashTx(ctx context.Context, tx *sql.Tx, chunkHash string) (*BlobChunk, error) { +func (r *BlobChunkRepository) GetByChunkHashTx( + ctx context.Context, tx *sql.Tx, chunkHash string, +) (*BlobChunk, error) { query := ` SELECT blob_id, chunk_hash, offset, length FROM blob_chunks @@ -120,10 +137,10 @@ func (r *BlobChunkRepository) GetByChunkHashTx(ctx context.Context, tx *sql.Tx, &bc.Length, ) - if err == sql.ErrNoRows { + if errors.Is(err, sql.ErrNoRows) { LogSQL("GetByChunkHashTx", "No rows found", chunkHash) - return nil, nil + return nil, nil //nolint:nilnil // nil,nil signals not-found; callers check nil } if err != nil { @@ -137,29 +154,34 @@ func (r *BlobChunkRepository) GetByChunkHashTx(ctx context.Context, tx *sql.Tx, return &bc, nil } -// DeleteOrphaned deletes blob_chunks entries where either the blob or chunk no longer exists +// DeleteOrphaned deletes blob_chunks entries where either the blob or the +// chunk no longer exists. func (r *BlobChunkRepository) DeleteOrphaned(ctx context.Context) error { // Delete blob_chunks where the blob doesn't exist query1 := ` - DELETE FROM blob_chunks + DELETE FROM blob_chunks WHERE NOT EXISTS ( - SELECT 1 FROM blobs + SELECT 1 FROM blobs WHERE blobs.id = blob_chunks.blob_id ) ` - if _, err := r.db.ExecWithLog(ctx, query1); err != nil { + + _, err := r.db.ExecWithLog(ctx, query1) + if err != nil { return fmt.Errorf("deleting blob_chunks with missing blobs: %w", err) } // Delete blob_chunks where the chunk doesn't exist query2 := ` - DELETE FROM blob_chunks + DELETE FROM blob_chunks WHERE NOT EXISTS ( - SELECT 1 FROM chunks + SELECT 1 FROM chunks WHERE chunks.chunk_hash = blob_chunks.chunk_hash ) ` - if _, err := r.db.ExecWithLog(ctx, query2); err != nil { + + _, err = r.db.ExecWithLog(ctx, query2) + if err != nil { return fmt.Errorf("deleting blob_chunks with missing chunks: %w", err) } diff --git a/internal/database/blob_chunks_test.go b/internal/database/blob_chunks_test.go index 97d3e68..5122d63 100644 --- a/internal/database/blob_chunks_test.go +++ b/internal/database/blob_chunks_test.go @@ -1,4 +1,4 @@ -package database +package database_test import ( "context" @@ -6,59 +6,91 @@ import ( "testing" "time" + "sneak.berlin/go/vaultik/internal/database" "sneak.berlin/go/vaultik/internal/types" ) -func TestBlobChunkRepository(t *testing.T) { - db, cleanup := setupTestDB(t) - defer cleanup() +// Chunk hashes used across the blob_chunks tests. +const ( + chunk1Hash = "chunk1" + chunk2Hash = "chunk2" + chunk3Hash = "chunk3" +) + +// mustCreateChunks registers the given chunk hashes (1024 bytes each). +func mustCreateChunks( + t *testing.T, + repos *database.Repositories, + hashes ...types.ChunkHash, +) { + t.Helper() ctx := context.Background() - repos := NewRepositories(db) - // Create blob first - blob := &Blob{ - ID: types.NewBlobID(), - Hash: types.BlobHash("blob1-hash"), - CreatedTS: time.Now(), - } - - err := repos.Blobs.Create(ctx, nil, blob) - if err != nil { - t.Fatalf("failed to create blob: %v", err) - } - - // Create chunks - chunks := []types.ChunkHash{"chunk1", "chunk2", "chunk3"} - for _, chunkHash := range chunks { - chunk := &Chunk{ + for _, chunkHash := range hashes { + chunk := &database.Chunk{ ChunkHash: chunkHash, Size: 1024, } - err = repos.Chunks.Create(ctx, nil, chunk) + err := repos.Chunks.Create(ctx, nil, chunk) if err != nil { t.Fatalf("failed to create chunk %s: %v", chunkHash, err) } } +} + +// mustCreateBlob creates a blob row with the given hash. +func mustCreateBlob( + t *testing.T, + repos *database.Repositories, + hash types.BlobHash, +) *database.Blob { + t.Helper() + + blob := &database.Blob{ + ID: types.NewBlobID(), + Hash: hash, + CreatedTS: time.Now(), + } + + err := repos.Blobs.Create(context.Background(), nil, blob) + if err != nil { + t.Fatalf("failed to create blob %s: %v", hash, err) + } + + return blob +} + +func TestBlobChunkRepository(t *testing.T) { + t.Parallel() + + db, cleanup := setupTestDB(t) + defer cleanup() + + ctx := context.Background() + repos := database.NewRepositories(db) + + blob := mustCreateBlob(t, repos, "blob1-hash") + mustCreateChunks(t, repos, chunk1Hash, chunk2Hash, chunk3Hash) // Test Create - bc1 := &BlobChunk{ + bc1 := &database.BlobChunk{ BlobID: blob.ID, - ChunkHash: types.ChunkHash("chunk1"), + ChunkHash: types.ChunkHash(chunk1Hash), Offset: 0, Length: 1024, } - err = repos.BlobChunks.Create(ctx, nil, bc1) + err := repos.BlobChunks.Create(ctx, nil, bc1) if err != nil { t.Fatalf("failed to create blob chunk: %v", err) } // Add more chunks to the same blob - bc2 := &BlobChunk{ + bc2 := &database.BlobChunk{ BlobID: blob.ID, - ChunkHash: types.ChunkHash("chunk2"), + ChunkHash: types.ChunkHash(chunk2Hash), Offset: 1024, Length: 2048, } @@ -68,9 +100,9 @@ func TestBlobChunkRepository(t *testing.T) { t.Fatalf("failed to create second blob chunk: %v", err) } - bc3 := &BlobChunk{ + bc3 := &database.BlobChunk{ BlobID: blob.ID, - ChunkHash: types.ChunkHash("chunk3"), + ChunkHash: types.ChunkHash(chunk3Hash), Offset: 3072, Length: 512, } @@ -94,12 +126,49 @@ func TestBlobChunkRepository(t *testing.T) { expectedOffsets := []int64{0, 1024, 3072} for i, bc := range blobChunks { if bc.Offset != expectedOffsets[i] { - t.Errorf("wrong chunk order: expected offset %d, got %d", expectedOffsets[i], bc.Offset) + t.Errorf("wrong chunk order: expected offset %d, got %d", + expectedOffsets[i], bc.Offset) } } + // Test duplicate insert (should fail due to primary key constraint) + err = repos.BlobChunks.Create(ctx, nil, bc1) + if err == nil { + t.Fatal("duplicate blob_chunk insert should fail due to primary key constraint") + } + + if !strings.Contains(err.Error(), "UNIQUE") && + !strings.Contains(err.Error(), "constraint") { + t.Fatalf("expected constraint error, got: %v", err) + } +} + +func TestBlobChunkRepositoryGetByChunkHash(t *testing.T) { + t.Parallel() + + db, cleanup := setupTestDB(t) + defer cleanup() + + ctx := context.Background() + repos := database.NewRepositories(db) + + blob := mustCreateBlob(t, repos, "blob-gbch-hash") + mustCreateChunks(t, repos, chunk2Hash) + + bc2 := &database.BlobChunk{ + BlobID: blob.ID, + ChunkHash: types.ChunkHash(chunk2Hash), + Offset: 1024, + Length: 2048, + } + + err := repos.BlobChunks.Create(ctx, nil, bc2) + if err != nil { + t.Fatalf("failed to create blob chunk: %v", err) + } + // Test GetByChunkHash - bc, err := repos.BlobChunks.GetByChunkHash(ctx, "chunk2") + bc, err := repos.BlobChunks.GetByChunkHash(ctx, chunk2Hash) if err != nil { t.Fatalf("failed to get blob chunk by chunk hash: %v", err) } @@ -116,16 +185,6 @@ func TestBlobChunkRepository(t *testing.T) { t.Errorf("wrong offset: expected 1024, got %d", bc.Offset) } - // Test duplicate insert (should fail due to primary key constraint) - err = repos.BlobChunks.Create(ctx, nil, bc1) - if err == nil { - t.Fatal("duplicate blob_chunk insert should fail due to primary key constraint") - } - - if !strings.Contains(err.Error(), "UNIQUE") && !strings.Contains(err.Error(), "constraint") { - t.Fatalf("expected constraint error, got: %v", err) - } - // Test non-existent chunk bc, err = repos.BlobChunks.GetByChunkHash(ctx, "nonexistent") if err != nil { @@ -138,55 +197,26 @@ func TestBlobChunkRepository(t *testing.T) { } func TestBlobChunkRepositoryMultipleBlobs(t *testing.T) { + t.Parallel() + db, cleanup := setupTestDB(t) defer cleanup() ctx := context.Background() - repos := NewRepositories(db) + repos := database.NewRepositories(db) - // Create blobs - blob1 := &Blob{ - ID: types.NewBlobID(), - Hash: types.BlobHash("blob1-hash"), - CreatedTS: time.Now(), - } - blob2 := &Blob{ - ID: types.NewBlobID(), - Hash: types.BlobHash("blob2-hash"), - CreatedTS: time.Now(), - } - - err := repos.Blobs.Create(ctx, nil, blob1) - if err != nil { - t.Fatalf("failed to create blob1: %v", err) - } - - err = repos.Blobs.Create(ctx, nil, blob2) - if err != nil { - t.Fatalf("failed to create blob2: %v", err) - } - - // Create chunks - chunkHashes := []types.ChunkHash{"chunk1", "chunk2", "chunk3"} - for _, chunkHash := range chunkHashes { - chunk := &Chunk{ - ChunkHash: chunkHash, - Size: 1024, - } - - err = repos.Chunks.Create(ctx, nil, chunk) - if err != nil { - t.Fatalf("failed to create chunk %s: %v", chunkHash, err) - } - } + blob1 := mustCreateBlob(t, repos, "blob1-hash") + blob2 := mustCreateBlob(t, repos, "blob2-hash") + mustCreateChunks(t, repos, chunk1Hash, chunk2Hash, chunk3Hash) // Create chunks across multiple blobs // Some chunks are shared between blobs (deduplication scenario) - blobChunks := []BlobChunk{ - {BlobID: blob1.ID, ChunkHash: types.ChunkHash("chunk1"), Offset: 0, Length: 1024}, - {BlobID: blob1.ID, ChunkHash: types.ChunkHash("chunk2"), Offset: 1024, Length: 1024}, - {BlobID: blob2.ID, ChunkHash: types.ChunkHash("chunk2"), Offset: 0, Length: 1024}, // chunk2 is shared - {BlobID: blob2.ID, ChunkHash: types.ChunkHash("chunk3"), Offset: 1024, Length: 1024}, + blobChunks := []database.BlobChunk{ + {BlobID: blob1.ID, ChunkHash: chunk1Hash, Offset: 0, Length: 1024}, + {BlobID: blob1.ID, ChunkHash: chunk2Hash, Offset: 1024, Length: 1024}, + // chunk2 is shared between the blobs + {BlobID: blob2.ID, ChunkHash: chunk2Hash, Offset: 0, Length: 1024}, + {BlobID: blob2.ID, ChunkHash: chunk3Hash, Offset: 1024, Length: 1024}, } for _, bc := range blobChunks { @@ -217,7 +247,7 @@ func TestBlobChunkRepositoryMultipleBlobs(t *testing.T) { } // Verify shared chunk - bc, err := repos.BlobChunks.GetByChunkHash(ctx, "chunk2") + bc, err := repos.BlobChunks.GetByChunkHash(ctx, chunk2Hash) if err != nil { t.Fatalf("failed to get shared chunk: %v", err) } diff --git a/internal/database/blobs.go b/internal/database/blobs.go index 43a48e7..0acd4cd 100644 --- a/internal/database/blobs.go +++ b/internal/database/blobs.go @@ -3,23 +3,29 @@ package database import ( "context" "database/sql" + "errors" "fmt" "time" "sneak.berlin/go/vaultik/internal/log" ) +// BlobRepository provides access to the blobs table, which tracks the +// packed, encrypted storage units uploaded to the destination. type BlobRepository struct { db *DB } +// NewBlobRepository creates a BlobRepository backed by db. func NewBlobRepository(db *DB) *BlobRepository { return &BlobRepository{db: db} } +// Create inserts a blob row, using tx when non-nil. func (r *BlobRepository) Create(ctx context.Context, tx *sql.Tx, blob *Blob) error { query := ` - INSERT INTO blobs (id, blob_hash, created_ts, finished_ts, uncompressed_size, compressed_size, uploaded_ts) + INSERT INTO blobs (id, blob_hash, created_ts, finished_ts, + uncompressed_size, compressed_size, uploaded_ts) VALUES (?, ?, ?, ?, ?, ?, ?) ` @@ -51,95 +57,15 @@ func (r *BlobRepository) Create(ctx context.Context, tx *sql.Tx, blob *Blob) err return nil } +// GetByHash returns the blob with the given content hash, or nil if no +// such blob exists. func (r *BlobRepository) GetByHash(ctx context.Context, hash string) (*Blob, error) { - query := ` - SELECT id, blob_hash, created_ts, finished_ts, uncompressed_size, compressed_size, uploaded_ts - FROM blobs - WHERE blob_hash = ? - ` - - var ( - blob Blob - createdTSUnix int64 - finishedTSUnix, uploadedTSUnix sql.NullInt64 - ) - - err := r.db.conn.QueryRowContext(ctx, query, hash).Scan( - &blob.ID, - &blob.Hash, - &createdTSUnix, - &finishedTSUnix, - &blob.UncompressedSize, - &blob.CompressedSize, - &uploadedTSUnix, - ) - - if err == sql.ErrNoRows { - return nil, nil - } - - if err != nil { - return nil, fmt.Errorf("querying blob: %w", err) - } - - blob.CreatedTS = time.Unix(createdTSUnix, 0).UTC() - if finishedTSUnix.Valid { - ts := time.Unix(finishedTSUnix.Int64, 0).UTC() - blob.FinishedTS = &ts - } - - if uploadedTSUnix.Valid { - ts := time.Unix(uploadedTSUnix.Int64, 0).UTC() - blob.UploadedTS = &ts - } - - return &blob, nil + return r.getOne(ctx, "blob_hash", hash) } // GetByID retrieves a blob by its ID func (r *BlobRepository) GetByID(ctx context.Context, id string) (*Blob, error) { - query := ` - SELECT id, blob_hash, created_ts, finished_ts, uncompressed_size, compressed_size, uploaded_ts - FROM blobs - WHERE id = ? - ` - - var ( - blob Blob - createdTSUnix int64 - finishedTSUnix, uploadedTSUnix sql.NullInt64 - ) - - err := r.db.conn.QueryRowContext(ctx, query, id).Scan( - &blob.ID, - &blob.Hash, - &createdTSUnix, - &finishedTSUnix, - &blob.UncompressedSize, - &blob.CompressedSize, - &uploadedTSUnix, - ) - - if err == sql.ErrNoRows { - return nil, nil - } - - if err != nil { - return nil, fmt.Errorf("querying blob: %w", err) - } - - blob.CreatedTS = time.Unix(createdTSUnix, 0).UTC() - if finishedTSUnix.Valid { - ts := time.Unix(finishedTSUnix.Int64, 0).UTC() - blob.FinishedTS = &ts - } - - if uploadedTSUnix.Valid { - ts := time.Unix(uploadedTSUnix.Int64, 0).UTC() - blob.UploadedTS = &ts - } - - return &blob, nil + return r.getOne(ctx, "id", id) } // GetAll returns every blob row keyed by blob ID. Useful at restore @@ -147,7 +73,8 @@ func (r *BlobRepository) GetByID(ctx context.Context, id string) (*Blob, error) // into blob hashes without doing one GetByID query per chunk. func (r *BlobRepository) GetAll(ctx context.Context) (map[string]*Blob, error) { query := ` - SELECT id, blob_hash, created_ts, finished_ts, uncompressed_size, compressed_size, uploaded_ts + SELECT id, blob_hash, created_ts, finished_ts, + uncompressed_size, compressed_size, uploaded_ts FROM blobs ` @@ -197,7 +124,13 @@ func (r *BlobRepository) GetAll(ctx context.Context) (map[string]*Blob, error) { } // UpdateFinished updates a blob when it's finalized -func (r *BlobRepository) UpdateFinished(ctx context.Context, tx *sql.Tx, id string, hash string, uncompressedSize, compressedSize int64) error { +func (r *BlobRepository) UpdateFinished( + ctx context.Context, + tx *sql.Tx, + id string, + hash string, + uncompressedSize, compressedSize int64, +) error { query := ` UPDATE blobs SET blob_hash = ?, finished_ts = ?, uncompressed_size = ?, compressed_size = ? @@ -221,7 +154,9 @@ func (r *BlobRepository) UpdateFinished(ctx context.Context, tx *sql.Tx, id stri } // UpdateUploaded marks a blob as uploaded -func (r *BlobRepository) UpdateUploaded(ctx context.Context, tx *sql.Tx, id string) error { +func (r *BlobRepository) UpdateUploaded( + ctx context.Context, tx *sql.Tx, id string, +) error { query := ` UPDATE blobs SET uploaded_ts = ? @@ -266,3 +201,52 @@ func (r *BlobRepository) DeleteOrphaned(ctx context.Context) error { return nil } + +// getOne fetches a single blob row matched on the given column, or +// (nil, nil) when no row matches. +func (r *BlobRepository) getOne( + ctx context.Context, column, value string, +) (*Blob, error) { + query := ` + SELECT id, blob_hash, created_ts, finished_ts, + uncompressed_size, compressed_size, uploaded_ts + FROM blobs + WHERE ` + column + ` = ?` + + var ( + blob Blob + createdTSUnix int64 + finishedTSUnix, uploadedTSUnix sql.NullInt64 + ) + + err := r.db.conn.QueryRowContext(ctx, query, value).Scan( + &blob.ID, + &blob.Hash, + &createdTSUnix, + &finishedTSUnix, + &blob.UncompressedSize, + &blob.CompressedSize, + &uploadedTSUnix, + ) + + if errors.Is(err, sql.ErrNoRows) { + return nil, nil //nolint:nilnil // nil,nil signals not-found; callers check nil + } + + if err != nil { + return nil, fmt.Errorf("querying blob: %w", err) + } + + blob.CreatedTS = time.Unix(createdTSUnix, 0).UTC() + if finishedTSUnix.Valid { + ts := time.Unix(finishedTSUnix.Int64, 0).UTC() + blob.FinishedTS = &ts + } + + if uploadedTSUnix.Valid { + ts := time.Unix(uploadedTSUnix.Int64, 0).UTC() + blob.UploadedTS = &ts + } + + return &blob, nil +} diff --git a/internal/database/blobs_test.go b/internal/database/blobs_test.go index 2b8e88b..457b0db 100644 --- a/internal/database/blobs_test.go +++ b/internal/database/blobs_test.go @@ -1,22 +1,25 @@ -package database +package database_test import ( "context" "testing" "time" + "sneak.berlin/go/vaultik/internal/database" "sneak.berlin/go/vaultik/internal/types" ) func TestBlobRepository(t *testing.T) { + t.Parallel() + db, cleanup := setupTestDB(t) defer cleanup() ctx := context.Background() - repo := NewBlobRepository(db) + repo := database.NewBlobRepository(db) // Test Create - blob := &Blob{ + blob := &database.Blob{ ID: types.NewBlobID(), Hash: types.BlobHash("blobhash123"), CreatedTS: time.Now().Truncate(time.Second), @@ -42,7 +45,8 @@ func TestBlobRepository(t *testing.T) { } if !retrieved.CreatedTS.Equal(blob.CreatedTS) { - t.Errorf("created timestamp mismatch: got %v, want %v", retrieved.CreatedTS, blob.CreatedTS) + t.Errorf("created timestamp mismatch: got %v, want %v", + retrieved.CreatedTS, blob.CreatedTS) } // Test GetByID @@ -60,7 +64,7 @@ func TestBlobRepository(t *testing.T) { } // Test with second blob - blob2 := &Blob{ + blob2 := &database.Blob{ ID: types.NewBlobID(), Hash: types.BlobHash("blobhash456"), CreatedTS: time.Now().Truncate(time.Second), @@ -70,6 +74,27 @@ func TestBlobRepository(t *testing.T) { if err != nil { t.Fatalf("failed to create second blob: %v", err) } +} + +func TestBlobRepositoryUpdates(t *testing.T) { + t.Parallel() + + db, cleanup := setupTestDB(t) + defer cleanup() + + ctx := context.Background() + repo := database.NewBlobRepository(db) + + blob := &database.Blob{ + ID: types.NewBlobID(), + Hash: types.BlobHash("blobhash123"), + CreatedTS: time.Now().Truncate(time.Second), + } + + err := repo.Create(ctx, nil, blob) + if err != nil { + t.Fatalf("failed to create blob: %v", err) + } // Test UpdateFinished now := time.Now() @@ -119,13 +144,15 @@ func TestBlobRepository(t *testing.T) { } func TestBlobRepositoryDuplicate(t *testing.T) { + t.Parallel() + db, cleanup := setupTestDB(t) defer cleanup() ctx := context.Background() - repo := NewBlobRepository(db) + repo := database.NewBlobRepository(db) - blob := &Blob{ + blob := &database.Blob{ ID: types.NewBlobID(), Hash: types.BlobHash("duplicate_blob"), CreatedTS: time.Now().Truncate(time.Second), diff --git a/internal/database/cascade_debug_test.go b/internal/database/cascade_debug_test.go index fc71f9f..bcde357 100644 --- a/internal/database/cascade_debug_test.go +++ b/internal/database/cascade_debug_test.go @@ -1,3 +1,4 @@ +//nolint:testpackage // inspects the unexported database connection package database import ( @@ -9,25 +10,13 @@ import ( "sneak.berlin/go/vaultik/internal/types" ) -// TestCascadeDeleteDebug tests cascade delete with debug output -func TestCascadeDeleteDebug(t *testing.T) { - db, cleanup := setupTestDB(t) - defer cleanup() +// createCascadeFixtures creates a file with three chunk mappings for the +// cascade-delete test. +func createCascadeFixtures(t *testing.T, repos *Repositories) *File { + t.Helper() ctx := context.Background() - repos := NewRepositories(db) - // Check if foreign keys are enabled - var fkEnabled int - - err := db.conn.QueryRow("PRAGMA foreign_keys").Scan(&fkEnabled) - if err != nil { - t.Fatal(err) - } - - t.Logf("Foreign keys enabled: %d", fkEnabled) - - // Create a file file := &File{ Path: "/cascade-test.txt", MTime: time.Now().Truncate(time.Second), @@ -37,7 +26,7 @@ func TestCascadeDeleteDebug(t *testing.T) { GID: 1000, } - err = repos.Files.Create(ctx, nil, file) + err := repos.Files.Create(ctx, nil, file) if err != nil { t.Fatalf("failed to create file: %v", err) } @@ -67,9 +56,56 @@ func TestCascadeDeleteDebug(t *testing.T) { t.Fatalf("failed to create file chunk: %v", err) } - t.Logf("Created file chunk mapping: file_id=%s, idx=%d, chunk=%s", fc.FileID, fc.Idx, fc.ChunkHash) + t.Logf("Created file chunk mapping: file_id=%s, idx=%d, chunk=%s", + fc.FileID, fc.Idx, fc.ChunkHash) } + return file +} + +// logCascadeDebugInfo logs foreign-key state and the file_chunks table +// definition for cascade-delete debugging. +func logCascadeDebugInfo(ctx context.Context, t *testing.T, db *DB) { + t.Helper() + + // Check if foreign keys are enabled + var fkEnabled int + + err := db.conn.QueryRowContext(ctx, "PRAGMA foreign_keys").Scan(&fkEnabled) + if err != nil { + t.Fatal(err) + } + + t.Logf("Foreign keys enabled: %d", fkEnabled) + + // Check the foreign key constraint + var fkInfo string + + err = db.conn.QueryRowContext(ctx, ` + SELECT sql FROM sqlite_master + WHERE type='table' AND name='file_chunks' + `).Scan(&fkInfo) + if err != nil { + t.Fatal(err) + } + + t.Logf("file_chunks table definition:\n%s", fkInfo) +} + +// TestCascadeDeleteDebug tests cascade delete with debug output +func TestCascadeDeleteDebug(t *testing.T) { + t.Parallel() + + db, cleanup := setupTestDB(t) + defer cleanup() + + ctx := context.Background() + repos := NewRepositories(db) + + logCascadeDebugInfo(ctx, t, db) + + file := createCascadeFixtures(t, repos) + // Verify file chunks exist fileChunks, err := repos.FileChunks.GetByFileID(ctx, file.ID) if err != nil { @@ -78,19 +114,6 @@ func TestCascadeDeleteDebug(t *testing.T) { t.Logf("File chunks before delete: %d", len(fileChunks)) - // Check the foreign key constraint - var fkInfo string - - err = db.conn.QueryRow(` - SELECT sql FROM sqlite_master - WHERE type='table' AND name='file_chunks' - `).Scan(&fkInfo) - if err != nil { - t.Fatal(err) - } - - t.Logf("file_chunks table definition:\n%s", fkInfo) - // Delete the file t.Log("Deleting file...") @@ -122,7 +145,9 @@ func TestCascadeDeleteDebug(t *testing.T) { // Manually check the database var count int - err = db.conn.QueryRow("SELECT COUNT(*) FROM file_chunks WHERE file_id = ?", file.ID).Scan(&count) + err = db.conn.QueryRowContext(ctx, + "SELECT COUNT(*) FROM file_chunks WHERE file_id = ?", file.ID, + ).Scan(&count) if err != nil { t.Fatal(err) } @@ -133,7 +158,8 @@ func TestCascadeDeleteDebug(t *testing.T) { t.Errorf("expected 0 file chunks after cascade delete, got %d", len(fileChunks)) // List the remaining chunks for _, fc := range fileChunks { - t.Logf("Remaining chunk: file_id=%s, idx=%d, chunk=%s", fc.FileID, fc.Idx, fc.ChunkHash) + t.Logf("Remaining chunk: file_id=%s, idx=%d, chunk=%s", + fc.FileID, fc.Idx, fc.ChunkHash) } } } diff --git a/internal/database/chunk_files.go b/internal/database/chunk_files.go index 01e9175..f6b00bd 100644 --- a/internal/database/chunk_files.go +++ b/internal/database/chunk_files.go @@ -9,15 +9,21 @@ import ( "sneak.berlin/go/vaultik/internal/types" ) +// ChunkFileRepository provides access to the chunk_files table, the +// reverse mapping from chunks to the files that contain them. type ChunkFileRepository struct { db *DB } +// NewChunkFileRepository creates a ChunkFileRepository backed by db. func NewChunkFileRepository(db *DB) *ChunkFileRepository { return &ChunkFileRepository{db: db} } -func (r *ChunkFileRepository) Create(ctx context.Context, tx *sql.Tx, cf *ChunkFile) error { +// Create inserts a chunk_files row (idempotently), using tx when non-nil. +func (r *ChunkFileRepository) Create( + ctx context.Context, tx *sql.Tx, cf *ChunkFile, +) error { query := ` INSERT INTO chunk_files (chunk_hash, file_id, file_offset, length) VALUES (?, ?, ?, ?) @@ -26,9 +32,11 @@ func (r *ChunkFileRepository) Create(ctx context.Context, tx *sql.Tx, cf *ChunkF var err error if tx != nil { - _, err = tx.ExecContext(ctx, query, cf.ChunkHash.String(), cf.FileID.String(), cf.FileOffset, cf.Length) + _, err = tx.ExecContext(ctx, query, + cf.ChunkHash.String(), cf.FileID.String(), cf.FileOffset, cf.Length) } else { - _, err = r.db.ExecWithLog(ctx, query, cf.ChunkHash.String(), cf.FileID.String(), cf.FileOffset, cf.Length) + _, err = r.db.ExecWithLog(ctx, query, + cf.ChunkHash.String(), cf.FileID.String(), cf.FileOffset, cf.Length) } if err != nil { @@ -38,7 +46,10 @@ func (r *ChunkFileRepository) Create(ctx context.Context, tx *sql.Tx, cf *ChunkF return nil } -func (r *ChunkFileRepository) GetByChunkHash(ctx context.Context, chunkHash types.ChunkHash) ([]*ChunkFile, error) { +// GetByChunkHash returns all chunk_files rows for the given chunk hash. +func (r *ChunkFileRepository) GetByChunkHash( + ctx context.Context, chunkHash types.ChunkHash, +) ([]*ChunkFile, error) { query := ` SELECT chunk_hash, file_id, file_offset, length FROM chunk_files @@ -54,7 +65,10 @@ func (r *ChunkFileRepository) GetByChunkHash(ctx context.Context, chunkHash type return r.scanChunkFiles(rows) } -func (r *ChunkFileRepository) GetByFilePath(ctx context.Context, filePath string) ([]*ChunkFile, error) { +// GetByFilePath returns all chunk_files rows for the file at the given path. +func (r *ChunkFileRepository) GetByFilePath( + ctx context.Context, filePath string, +) ([]*ChunkFile, error) { query := ` SELECT cf.chunk_hash, cf.file_id, cf.file_offset, cf.length FROM chunk_files cf @@ -72,7 +86,9 @@ func (r *ChunkFileRepository) GetByFilePath(ctx context.Context, filePath string } // GetByFileID retrieves chunk files by file ID -func (r *ChunkFileRepository) GetByFileID(ctx context.Context, fileID types.FileID) ([]*ChunkFile, error) { +func (r *ChunkFileRepository) GetByFileID( + ctx context.Context, fileID types.FileID, +) ([]*ChunkFile, error) { query := ` SELECT chunk_hash, file_id, file_offset, length FROM chunk_files @@ -88,7 +104,124 @@ func (r *ChunkFileRepository) GetByFileID(ctx context.Context, fileID types.File return r.scanChunkFiles(rows) } -// scanChunkFiles is a helper that scans chunk file rows +// DeleteByFileID deletes all chunk_files entries for a given file ID +func (r *ChunkFileRepository) DeleteByFileID( + ctx context.Context, tx *sql.Tx, fileID types.FileID, +) error { + query := `DELETE FROM chunk_files WHERE file_id = ?` + + var err error + if tx != nil { + _, err = tx.ExecContext(ctx, query, fileID.String()) + } else { + _, err = r.db.ExecWithLog(ctx, query, fileID.String()) + } + + if err != nil { + return fmt.Errorf("deleting chunk files: %w", err) + } + + return nil +} + +// DeleteByFileIDs deletes all chunk_files for multiple files in a single statement. +// +//nolint:dupl // symmetric implementation for a parallel association table +func (r *ChunkFileRepository) DeleteByFileIDs( + ctx context.Context, tx *sql.Tx, fileIDs []types.FileID, +) error { + if len(fileIDs) == 0 { + return nil + } + + // Batch at 500 to stay within SQLite's variable limit + const batchSize = 500 + + for i := 0; i < len(fileIDs); i += batchSize { + end := min(i+batchSize, len(fileIDs)) + + batch := fileIDs[i:end] + + //nolint:gosec // G202: concatenates constant SQL and "?" placeholders only + query := "DELETE FROM chunk_files WHERE file_id IN (?" + + repeatPlaceholder(len(batch)-1) + ")" + + args := make([]any, len(batch)) + for j, id := range batch { + args[j] = id.String() + } + + var err error + if tx != nil { + _, err = tx.ExecContext(ctx, query, args...) + } else { + _, err = r.db.ExecWithLog(ctx, query, args...) + } + + if err != nil { + return fmt.Errorf("batch deleting chunk_files: %w", err) + } + } + + return nil +} + +// CreateBatch inserts multiple chunk_files in a single statement for efficiency. +func (r *ChunkFileRepository) CreateBatch( + ctx context.Context, tx *sql.Tx, cfs []ChunkFile, +) error { + if len(cfs) == 0 { + return nil + } + + // Each chunk_files row binds this many SQL variables. + const chunkFileCols = 4 + + // Batch at 200 rows to be safe with SQLite's variable limit. + const batchSize = 200 + + for i := 0; i < len(cfs); i += batchSize { + end := min(i+batchSize, len(cfs)) + + batch := cfs[i:end] + + query := "INSERT INTO chunk_files (chunk_hash, file_id, file_offset, length) VALUES " + + args := make([]any, 0, len(batch)*chunkFileCols) + + var querySb183 strings.Builder + + for j, cf := range batch { + if j > 0 { + querySb183.WriteString(", ") + } + + querySb183.WriteString("(?, ?, ?, ?)") + + args = append(args, + cf.ChunkHash.String(), cf.FileID.String(), cf.FileOffset, cf.Length) + } + + query += querySb183.String() //nolint:gosec // G202: appends "?" placeholders only + + query += " ON CONFLICT(chunk_hash, file_id) DO NOTHING" + + var err error + if tx != nil { + _, err = tx.ExecContext(ctx, query, args...) + } else { + _, err = r.db.ExecWithLog(ctx, query, args...) + } + + if err != nil { + return fmt.Errorf("batch inserting chunk_files: %w", err) + } + } + + return nil +} + +// scanChunkFiles is a helper that scans chunk file rows. func (r *ChunkFileRepository) scanChunkFiles(rows *sql.Rows) ([]*ChunkFile, error) { var chunkFiles []*ChunkFile @@ -115,106 +248,3 @@ func (r *ChunkFileRepository) scanChunkFiles(rows *sql.Rows) ([]*ChunkFile, erro return chunkFiles, rows.Err() } - -// DeleteByFileID deletes all chunk_files entries for a given file ID -func (r *ChunkFileRepository) DeleteByFileID(ctx context.Context, tx *sql.Tx, fileID types.FileID) error { - query := `DELETE FROM chunk_files WHERE file_id = ?` - - var err error - if tx != nil { - _, err = tx.ExecContext(ctx, query, fileID.String()) - } else { - _, err = r.db.ExecWithLog(ctx, query, fileID.String()) - } - - if err != nil { - return fmt.Errorf("deleting chunk files: %w", err) - } - - return nil -} - -// DeleteByFileIDs deletes all chunk_files for multiple files in a single statement. -func (r *ChunkFileRepository) DeleteByFileIDs(ctx context.Context, tx *sql.Tx, fileIDs []types.FileID) error { - if len(fileIDs) == 0 { - return nil - } - - // Batch at 500 to stay within SQLite's variable limit - const batchSize = 500 - - for i := 0; i < len(fileIDs); i += batchSize { - end := min(i+batchSize, len(fileIDs)) - - batch := fileIDs[i:end] - - query := "DELETE FROM chunk_files WHERE file_id IN (?" + repeatPlaceholder(len(batch)-1) + ")" - - args := make([]any, len(batch)) - for j, id := range batch { - args[j] = id.String() - } - - var err error - if tx != nil { - _, err = tx.ExecContext(ctx, query, args...) - } else { - _, err = r.db.ExecWithLog(ctx, query, args...) - } - - if err != nil { - return fmt.Errorf("batch deleting chunk_files: %w", err) - } - } - - return nil -} - -// CreateBatch inserts multiple chunk_files in a single statement for efficiency. -func (r *ChunkFileRepository) CreateBatch(ctx context.Context, tx *sql.Tx, cfs []ChunkFile) error { - if len(cfs) == 0 { - return nil - } - - // Each ChunkFile has 4 values, so batch at 200 to be safe with SQLite's variable limit - const batchSize = 200 - - for i := 0; i < len(cfs); i += batchSize { - end := min(i+batchSize, len(cfs)) - - batch := cfs[i:end] - - query := "INSERT INTO chunk_files (chunk_hash, file_id, file_offset, length) VALUES " - - args := make([]any, 0, len(batch)*4) - - var querySb183 strings.Builder - - for j, cf := range batch { - if j > 0 { - querySb183.WriteString(", ") - } - - querySb183.WriteString("(?, ?, ?, ?)") - - args = append(args, cf.ChunkHash.String(), cf.FileID.String(), cf.FileOffset, cf.Length) - } - - query += querySb183.String() - - query += " ON CONFLICT(chunk_hash, file_id) DO NOTHING" - - var err error - if tx != nil { - _, err = tx.ExecContext(ctx, query, args...) - } else { - _, err = r.db.ExecWithLog(ctx, query, args...) - } - - if err != nil { - return fmt.Errorf("batch inserting chunk_files: %w", err) - } - } - - return nil -} diff --git a/internal/database/chunk_files_test.go b/internal/database/chunk_files_test.go index ca3fd98..aa47e73 100644 --- a/internal/database/chunk_files_test.go +++ b/internal/database/chunk_files_test.go @@ -1,81 +1,105 @@ -package database +package database_test import ( "context" "testing" "time" + "sneak.berlin/go/vaultik/internal/database" "sneak.berlin/go/vaultik/internal/types" ) +const chunk4Hash = "chunk4" + +// verifyChunkFilePair asserts that the chunk-file rows cover both test +// files at their expected offsets. +func verifyChunkFilePair( + t *testing.T, chunkFiles []*database.ChunkFile, + file1ID, file2ID types.FileID, +) { + t.Helper() + + foundFile1 := false + foundFile2 := false + + for _, cf := range chunkFiles { + if cf.FileID == file1ID && cf.FileOffset == 0 { + foundFile1 = true + } + + if cf.FileID == file2ID && cf.FileOffset == 2048 { + foundFile2 = true + } + } + + if !foundFile1 || !foundFile2 { + t.Error("not all expected files found") + } +} + +// createChunkFileTestFiles creates the two files used by the chunk-file +// repository tests. +func createChunkFileTestFiles( + t *testing.T, fileRepo *database.FileRepository, +) (*database.File, *database.File) { + t.Helper() + + testTime := time.Now().Truncate(time.Second) + file1 := &database.File{ + Path: testFilePath1, + MTime: testTime, + Size: 1024, + Mode: 0644, + UID: 1000, + GID: 1000, + LinkTarget: "", + } + file2 := &database.File{ + Path: testFilePath2, + MTime: testTime, + Size: 1024, + Mode: 0644, + UID: 1000, + GID: 1000, + LinkTarget: "", + } + + mustCreateFile(t, fileRepo, file1) + mustCreateFile(t, fileRepo, file2) + + return file1, file2 +} + func TestChunkFileRepository(t *testing.T) { + t.Parallel() + db, cleanup := setupTestDB(t) defer cleanup() ctx := context.Background() - repo := NewChunkFileRepository(db) - fileRepo := NewFileRepository(db) - chunksRepo := NewChunkRepository(db) + repo := database.NewChunkFileRepository(db) + fileRepo := database.NewFileRepository(db) + repos := database.NewRepositories(db) - // Create test files first - testTime := time.Now().Truncate(time.Second) - file1 := &File{ - Path: "/file1.txt", - MTime: testTime, - Size: 1024, - Mode: 0644, - UID: 1000, - GID: 1000, - LinkTarget: "", - } - - err := fileRepo.Create(ctx, nil, file1) - if err != nil { - t.Fatalf("failed to create file1: %v", err) - } - - file2 := &File{ - Path: "/file2.txt", - MTime: testTime, - Size: 1024, - Mode: 0644, - UID: 1000, - GID: 1000, - LinkTarget: "", - } - - err = fileRepo.Create(ctx, nil, file2) - if err != nil { - t.Fatalf("failed to create file2: %v", err) - } - - // Create chunk first - chunk := &Chunk{ - ChunkHash: types.ChunkHash("chunk1"), - Size: 1024, - } - - err = chunksRepo.Create(ctx, nil, chunk) - if err != nil { - t.Fatalf("failed to create chunk: %v", err) - } + file1, file2 := createChunkFileTestFiles(t, fileRepo) + mustCreateChunks(t, repos, chunk1Hash) // Test Create - cf1 := &ChunkFile{ - ChunkHash: types.ChunkHash("chunk1"), + cf1 := &database.ChunkFile{ + ChunkHash: types.ChunkHash(chunk1Hash), FileID: file1.ID, FileOffset: 0, Length: 1024, } - err = repo.Create(ctx, nil, cf1) + err := repo.Create(ctx, nil, cf1) if err != nil { t.Fatalf("failed to create chunk file: %v", err) } // Add same chunk in different file (deduplication scenario) - cf2 := &ChunkFile{ - ChunkHash: types.ChunkHash("chunk1"), + cf2 := &database.ChunkFile{ + ChunkHash: types.ChunkHash(chunk1Hash), FileID: file2.ID, FileOffset: 2048, Length: 1024, @@ -87,7 +111,7 @@ func TestChunkFileRepository(t *testing.T) { } // Test GetByChunkHash - chunkFiles, err := repo.GetByChunkHash(ctx, "chunk1") + chunkFiles, err := repo.GetByChunkHash(ctx, chunk1Hash) if err != nil { t.Fatalf("failed to get chunk files: %v", err) } @@ -97,22 +121,7 @@ func TestChunkFileRepository(t *testing.T) { } // Verify both files are returned - foundFile1 := false - foundFile2 := false - - for _, cf := range chunkFiles { - if cf.FileID == file1.ID && cf.FileOffset == 0 { - foundFile1 = true - } - - if cf.FileID == file2.ID && cf.FileOffset == 2048 { - foundFile2 = true - } - } - - if !foundFile1 || !foundFile2 { - t.Error("not all expected files found") - } + verifyChunkFilePair(t, chunkFiles, file1.ID, file2.ID) // Test GetByFileID chunkFiles, err = repo.GetByFileID(ctx, file1.ID) @@ -124,7 +133,7 @@ func TestChunkFileRepository(t *testing.T) { t.Errorf("expected 1 chunk for file, got %d", len(chunkFiles)) } - if chunkFiles[0].ChunkHash != types.ChunkHash("chunk1") { + if chunkFiles[0].ChunkHash != types.ChunkHash(chunk1Hash) { t.Errorf("wrong chunk hash: expected chunk1, got %s", chunkFiles[0].ChunkHash) } @@ -136,63 +145,53 @@ func TestChunkFileRepository(t *testing.T) { } func TestChunkFileRepositoryComplexDeduplication(t *testing.T) { + t.Parallel() + db, cleanup := setupTestDB(t) defer cleanup() ctx := context.Background() - repo := NewChunkFileRepository(db) - fileRepo := NewFileRepository(db) - chunksRepo := NewChunkRepository(db) + repo := database.NewChunkFileRepository(db) + fileRepo := database.NewFileRepository(db) + repos := database.NewRepositories(db) // Create test files testTime := time.Now().Truncate(time.Second) - file1 := &File{Path: "/file1.txt", MTime: testTime, Size: 3072, Mode: 0644, UID: 1000, GID: 1000} - file2 := &File{Path: "/file2.txt", MTime: testTime, Size: 3072, Mode: 0644, UID: 1000, GID: 1000} - file3 := &File{Path: "/file3.txt", MTime: testTime, Size: 2048, Mode: 0644, UID: 1000, GID: 1000} - - if err := fileRepo.Create(ctx, nil, file1); err != nil { - t.Fatalf("failed to create file1: %v", err) + file1 := &database.File{ + Path: testFilePath1, MTime: testTime, Size: 3072, + Mode: 0644, UID: 1000, GID: 1000, + } + file2 := &database.File{ + Path: testFilePath2, MTime: testTime, Size: 3072, + Mode: 0644, UID: 1000, GID: 1000, + } + file3 := &database.File{ + Path: "/file3.txt", MTime: testTime, Size: 2048, + Mode: 0644, UID: 1000, GID: 1000, } - if err := fileRepo.Create(ctx, nil, file2); err != nil { - t.Fatalf("failed to create file2: %v", err) - } - - if err := fileRepo.Create(ctx, nil, file3); err != nil { - t.Fatalf("failed to create file3: %v", err) - } - - // Create chunks first - chunks := []types.ChunkHash{"chunk1", "chunk2", "chunk3", "chunk4"} - for _, chunkHash := range chunks { - chunk := &Chunk{ - ChunkHash: chunkHash, - Size: 1024, - } - - err := chunksRepo.Create(ctx, nil, chunk) - if err != nil { - t.Fatalf("failed to create chunk %s: %v", chunkHash, err) - } - } + mustCreateFile(t, fileRepo, file1) + mustCreateFile(t, fileRepo, file2) + mustCreateFile(t, fileRepo, file3) + mustCreateChunks(t, repos, chunk1Hash, chunk2Hash, chunk3Hash, chunk4Hash) // Simulate a scenario where multiple files share chunks // File1: chunk1, chunk2, chunk3 // File2: chunk2, chunk3, chunk4 // File3: chunk1, chunk4 - chunkFiles := []ChunkFile{ + chunkFiles := []database.ChunkFile{ // File1 - {ChunkHash: types.ChunkHash("chunk1"), FileID: file1.ID, FileOffset: 0, Length: 1024}, - {ChunkHash: types.ChunkHash("chunk2"), FileID: file1.ID, FileOffset: 1024, Length: 1024}, - {ChunkHash: types.ChunkHash("chunk3"), FileID: file1.ID, FileOffset: 2048, Length: 1024}, + {ChunkHash: chunk1Hash, FileID: file1.ID, FileOffset: 0, Length: 1024}, + {ChunkHash: chunk2Hash, FileID: file1.ID, FileOffset: 1024, Length: 1024}, + {ChunkHash: chunk3Hash, FileID: file1.ID, FileOffset: 2048, Length: 1024}, // File2 - {ChunkHash: types.ChunkHash("chunk2"), FileID: file2.ID, FileOffset: 0, Length: 1024}, - {ChunkHash: types.ChunkHash("chunk3"), FileID: file2.ID, FileOffset: 1024, Length: 1024}, - {ChunkHash: types.ChunkHash("chunk4"), FileID: file2.ID, FileOffset: 2048, Length: 1024}, + {ChunkHash: chunk2Hash, FileID: file2.ID, FileOffset: 0, Length: 1024}, + {ChunkHash: chunk3Hash, FileID: file2.ID, FileOffset: 1024, Length: 1024}, + {ChunkHash: chunk4Hash, FileID: file2.ID, FileOffset: 2048, Length: 1024}, // File3 - {ChunkHash: types.ChunkHash("chunk1"), FileID: file3.ID, FileOffset: 0, Length: 1024}, - {ChunkHash: types.ChunkHash("chunk4"), FileID: file3.ID, FileOffset: 1024, Length: 1024}, + {ChunkHash: chunk1Hash, FileID: file3.ID, FileOffset: 0, Length: 1024}, + {ChunkHash: chunk4Hash, FileID: file3.ID, FileOffset: 1024, Length: 1024}, } for _, cf := range chunkFiles { @@ -203,7 +202,7 @@ func TestChunkFileRepositoryComplexDeduplication(t *testing.T) { } // Test chunk1 (used by file1 and file3) - files, err := repo.GetByChunkHash(ctx, "chunk1") + files, err := repo.GetByChunkHash(ctx, chunk1Hash) if err != nil { t.Fatalf("failed to get files for chunk1: %v", err) } @@ -213,7 +212,7 @@ func TestChunkFileRepositoryComplexDeduplication(t *testing.T) { } // Test chunk2 (used by file1 and file2) - files, err = repo.GetByChunkHash(ctx, "chunk2") + files, err = repo.GetByChunkHash(ctx, chunk2Hash) if err != nil { t.Fatalf("failed to get files for chunk2: %v", err) } diff --git a/internal/database/chunks.go b/internal/database/chunks.go index 5c54a14..dbd0c74 100644 --- a/internal/database/chunks.go +++ b/internal/database/chunks.go @@ -3,20 +3,25 @@ package database import ( "context" "database/sql" + "errors" "fmt" "strings" "sneak.berlin/go/vaultik/internal/log" ) +// ChunkRepository provides access to the chunks table, which tracks +// content-defined chunks by hash and size. type ChunkRepository struct { db *DB } +// NewChunkRepository creates a ChunkRepository backed by db. func NewChunkRepository(db *DB) *ChunkRepository { return &ChunkRepository{db: db} } +// Create inserts a chunk row (idempotently), using tx when non-nil. func (r *ChunkRepository) Create(ctx context.Context, tx *sql.Tx, chunk *Chunk) error { query := ` INSERT INTO chunks (chunk_hash, size) @@ -38,6 +43,8 @@ func (r *ChunkRepository) Create(ctx context.Context, tx *sql.Tx, chunk *Chunk) return nil } +// GetByHash returns the chunk with the given hash, or nil if it is not +// known to the index. func (r *ChunkRepository) GetByHash(ctx context.Context, hash string) (*Chunk, error) { query := ` SELECT chunk_hash, size @@ -52,8 +59,8 @@ func (r *ChunkRepository) GetByHash(ctx context.Context, hash string) (*Chunk, e &chunk.Size, ) - if err == sql.ErrNoRows { - return nil, nil + if errors.Is(err, sql.ErrNoRows) { + return nil, nil //nolint:nilnil // nil,nil signals not-found; callers check nil } if err != nil { @@ -63,7 +70,11 @@ func (r *ChunkRepository) GetByHash(ctx context.Context, hash string) (*Chunk, e return &chunk, nil } -func (r *ChunkRepository) GetByHashes(ctx context.Context, hashes []string) ([]*Chunk, error) { +// GetByHashes returns the chunks whose hashes appear in hashes, ordered by +// chunk hash. Unknown hashes are silently omitted from the result. +func (r *ChunkRepository) GetByHashes( + ctx context.Context, hashes []string, +) ([]*Chunk, error) { if len(hashes) == 0 { return nil, nil } @@ -87,7 +98,7 @@ func (r *ChunkRepository) GetByHashes(ctx context.Context, hashes []string) ([]* args[i] = hash } - query += querySb75.String() + query += querySb75.String() //nolint:gosec // G202: appends "?" placeholders only query += ") ORDER BY chunk_hash" @@ -116,7 +127,11 @@ func (r *ChunkRepository) GetByHashes(ctx context.Context, hashes []string) ([]* return chunks, rows.Err() } -func (r *ChunkRepository) ListUnpacked(ctx context.Context, limit int) ([]*Chunk, error) { +// ListUnpacked returns up to limit chunks that are not yet stored in any +// blob, ordered by chunk hash. +func (r *ChunkRepository) ListUnpacked( + ctx context.Context, limit int, +) ([]*Chunk, error) { query := ` SELECT c.chunk_hash, c.size FROM chunks c diff --git a/internal/database/chunks_ext.go b/internal/database/chunks_ext.go index a322a9a..fee4030 100644 --- a/internal/database/chunks_ext.go +++ b/internal/database/chunks_ext.go @@ -5,6 +5,7 @@ import ( "fmt" ) +// List returns every chunk in the index, ordered by chunk hash. func (r *ChunkRepository) List(ctx context.Context) ([]*Chunk, error) { query := ` SELECT chunk_hash, size diff --git a/internal/database/chunks_test.go b/internal/database/chunks_test.go index 8f96bd1..f0d6e52 100644 --- a/internal/database/chunks_test.go +++ b/internal/database/chunks_test.go @@ -1,21 +1,24 @@ -package database +package database_test import ( "context" "testing" + "sneak.berlin/go/vaultik/internal/database" "sneak.berlin/go/vaultik/internal/types" ) func TestChunkRepository(t *testing.T) { + t.Parallel() + db, cleanup := setupTestDB(t) defer cleanup() ctx := context.Background() - repo := NewChunkRepository(db) + repo := database.NewChunkRepository(db) // Test Create - chunk := &Chunk{ + chunk := &database.Chunk{ ChunkHash: types.ChunkHash("chunkhash123"), Size: 4096, } @@ -50,7 +53,7 @@ func TestChunkRepository(t *testing.T) { } // Test GetByHashes - chunk2 := &Chunk{ + chunk2 := &database.Chunk{ ChunkHash: types.ChunkHash("chunkhash456"), Size: 8192, } @@ -60,7 +63,9 @@ func TestChunkRepository(t *testing.T) { t.Fatalf("failed to create second chunk: %v", err) } - chunks, err := repo.GetByHashes(ctx, []string{chunk.ChunkHash.String(), chunk2.ChunkHash.String()}) + chunks, err := repo.GetByHashes(ctx, []string{ + chunk.ChunkHash.String(), chunk2.ChunkHash.String(), + }) if err != nil { t.Fatalf("failed to get chunks by hashes: %v", err) } @@ -81,11 +86,13 @@ func TestChunkRepository(t *testing.T) { } func TestChunkRepositoryNotFound(t *testing.T) { + t.Parallel() + db, cleanup := setupTestDB(t) defer cleanup() ctx := context.Background() - repo := NewChunkRepository(db) + repo := database.NewChunkRepository(db) // Test GetByHash with non-existent hash chunk, err := repo.GetByHash(ctx, "nonexistent") diff --git a/internal/database/database.go b/internal/database/database.go index 7fc717f..b272637 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -15,6 +15,7 @@ import ( "context" "database/sql" "embed" + "errors" "fmt" "os" "path/filepath" @@ -22,10 +23,15 @@ import ( "strconv" "strings" + // Register the pure-Go sqlite driver. _ "modernc.org/sqlite" "sneak.berlin/go/vaultik/internal/log" ) +// errInvalidMigrationFilename is returned when an embedded migration file +// does not follow the "[_].sql" naming pattern. +var errInvalidMigrationFilename = errors.New("invalid migration filename") + //go:embed schema/*.sql var schemaFS embed.FS @@ -51,7 +57,7 @@ type DB struct { func ParseMigrationVersion(filename string) (int, error) { name := strings.TrimSuffix(filename, filepath.Ext(filename)) if name == "" { - return 0, fmt.Errorf("invalid migration filename %q: empty name", filename) + return 0, fmt.Errorf("%w %q: empty name", errInvalidMigrationFilename, filename) } // Split on underscore to separate version from description. @@ -62,15 +68,17 @@ func ParseMigrationVersion(filename string) (int, error) { } if versionStr == "" { - return 0, fmt.Errorf("invalid migration filename %q: empty version prefix", filename) + return 0, fmt.Errorf( + "%w %q: empty version prefix", errInvalidMigrationFilename, filename, + ) } // Validate the version is purely numeric. for _, ch := range versionStr { if ch < '0' || ch > '9' { return 0, fmt.Errorf( - "invalid migration filename %q: version %q contains non-numeric character %q", - filename, versionStr, string(ch), + "%w %q: version %q contains non-numeric character %q", + errInvalidMigrationFilename, filename, versionStr, string(ch), ) } } @@ -101,66 +109,87 @@ func New(ctx context.Context, path string) (*DB, error) { conn, err := sql.Open( "sqlite", - path+"?_journal_mode=WAL&_synchronous=NORMAL&_busy_timeout=10000&_locking_mode=NORMAL&_foreign_keys=ON", + path+"?_journal_mode=WAL&_synchronous=NORMAL&_busy_timeout=10000"+ + "&_locking_mode=NORMAL&_foreign_keys=ON", ) if err == nil { - // Set connection pool settings - // SQLite can handle multiple readers but only one writer at a time. - // Setting MaxOpenConns to 1 ensures all writes are serialized through - // a single connection, preventing SQLITE_BUSY errors. - conn.SetMaxOpenConns(1) - conn.SetMaxIdleConns(1) + configureConnPool(conn) - err := conn.PingContext(ctx) + err = conn.PingContext(ctx) if err == nil { // Success on first try log.Debug("Database opened successfully with WAL mode", "path", path) - // Enable foreign keys explicitly - if _, err := conn.ExecContext(ctx, "PRAGMA foreign_keys = ON"); err != nil { - log.Warn("Failed to enable foreign keys", "error", err) - } - - db := &DB{conn: conn, path: path} - - err := applyMigrations(ctx, conn) - if err != nil { - _ = conn.Close() - - return nil, fmt.Errorf("applying migrations: %w", err) - } - - return db, nil + return finishOpen(ctx, conn, path) } - log.Debug("Failed to ping database, closing connection", "path", path, "error", err) + log.Debug( + "Failed to ping database, closing connection", + "path", path, "error", err, + ) _ = conn.Close() } // If first attempt failed, try with TRUNCATE mode to clear any locks + return openWithRecovery(ctx, path) +} + +// configureConnPool serializes all database access through one connection. +// SQLite can handle multiple readers but only one writer at a time; setting +// MaxOpenConns to 1 ensures all writes go through a single connection, +// preventing SQLITE_BUSY errors. +func configureConnPool(conn *sql.DB) { + conn.SetMaxOpenConns(1) + conn.SetMaxIdleConns(1) +} + +// finishOpen enables foreign keys, wraps the connection, and applies any +// pending migrations. On migration failure the connection is closed. +func finishOpen(ctx context.Context, conn *sql.DB, path string) (*DB, error) { + // Enable foreign keys explicitly + _, err := conn.ExecContext(ctx, "PRAGMA foreign_keys = ON") + if err != nil { + log.Warn("Failed to enable foreign keys", "path", path, "error", err) + } + + db := &DB{conn: conn, path: path} + + err = applyMigrations(ctx, conn) + if err != nil { + _ = conn.Close() + + return nil, fmt.Errorf("applying migrations: %w", err) + } + + return db, nil +} + +// openWithRecovery retries opening the database in TRUNCATE journal mode to +// clear stale locks, then switches back to WAL mode. +func openWithRecovery(ctx context.Context, path string) (*DB, error) { log.Info( "Database appears locked, attempting recovery with TRUNCATE mode", "path", path, ) - conn, err = sql.Open( + conn, err := sql.Open( "sqlite", - path+"?_journal_mode=TRUNCATE&_synchronous=NORMAL&_busy_timeout=10000&_foreign_keys=ON", + path+"?_journal_mode=TRUNCATE&_synchronous=NORMAL&_busy_timeout=10000"+ + "&_foreign_keys=ON", ) if err != nil { return nil, fmt.Errorf("opening database in recovery mode: %w", err) } - // Set connection pool settings - // SQLite can handle multiple readers but only one writer at a time. - // Setting MaxOpenConns to 1 ensures all writes are serialized through - // a single connection, preventing SQLITE_BUSY errors. - conn.SetMaxOpenConns(1) - conn.SetMaxIdleConns(1) + configureConnPool(conn) - if err := conn.PingContext(ctx); err != nil { - log.Debug("Failed to ping database in recovery mode, closing", "path", path, "error", err) + err = conn.PingContext(ctx) + if err != nil { + log.Debug( + "Failed to ping database in recovery mode, closing", + "path", path, "error", err, + ) _ = conn.Close() @@ -175,20 +204,14 @@ func New(ctx context.Context, path string) (*DB, error) { // Switch back to WAL mode log.Debug("Switching database back to WAL mode", "path", path) - if _, err := conn.ExecContext(ctx, "PRAGMA journal_mode=WAL"); err != nil { + _, err = conn.ExecContext(ctx, "PRAGMA journal_mode=WAL") + if err != nil { log.Warn("Failed to switch back to WAL mode", "path", path, "error", err) } - // Ensure foreign keys are enabled - if _, err := conn.ExecContext(ctx, "PRAGMA foreign_keys=ON"); err != nil { - log.Warn("Failed to enable foreign keys", "path", path, "error", err) - } - - db := &DB{conn: conn, path: path} - if err := applyMigrations(ctx, conn); err != nil { - _ = conn.Close() - - return nil, fmt.Errorf("applying migrations: %w", err) + db, err := finishOpen(ctx, conn, path) + if err != nil { + return nil, err } log.Debug("Database connection established successfully", "path", path) @@ -196,6 +219,13 @@ func New(ctx context.Context, path string) (*DB, error) { return db, nil } +// NewTestDB creates an in-memory SQLite database for testing purposes. +// The database is automatically initialized with the schema and is ready +// for use. Each call creates a new independent database instance. +func NewTestDB() (*DB, error) { + return New(context.Background(), ":memory:") +} + // Close closes the database connection. // It ensures all pending operations are completed before closing. // Returns an error if the database connection cannot be closed properly. @@ -253,10 +283,11 @@ func (db *DB) ExecWithLog( return db.conn.ExecContext(ctx, query, args...) } -// QueryRowWithLog executes a query that returns at most one row with SQL logging. -// This is useful for queries that modify data and return values (e.g., INSERT ... RETURNING). -// SQLite handles its own locking internally. -// The query and args parameters follow the same format as sql.DB.QueryRowContext. +// QueryRowWithLog executes a query that returns at most one row with SQL +// logging. This is useful for queries that modify data and return values +// (e.g., INSERT ... RETURNING). SQLite handles its own locking internally. +// The query and args parameters follow the same format as +// sql.DB.QueryRowContext. func (db *DB) QueryRowWithLog( ctx context.Context, query string, @@ -323,7 +354,8 @@ func bootstrapMigrationsTable(ctx context.Context, db *sql.DB) error { // the schema_migrations table via 000.sql, then iterates through remaining // migration files in order. func applyMigrations(ctx context.Context, db *sql.DB) error { - if err := bootstrapMigrationsTable(ctx, db); err != nil { + err := bootstrapMigrationsTable(ctx, db) + if err != nil { return err } @@ -383,15 +415,8 @@ func applyMigrations(ctx context.Context, db *sql.DB) error { return nil } -// NewTestDB creates an in-memory SQLite database for testing purposes. -// The database is automatically initialized with the schema and is ready for use. -// Each call creates a new independent database instance. -func NewTestDB() (*DB, error) { - return New(context.Background(), ":memory:") -} - -// repeatPlaceholder generates a string of ", ?" repeated n times for IN clause construction. -// For example, repeatPlaceholder(2) returns ", ?, ?". +// repeatPlaceholder generates a string of ", ?" repeated n times for IN +// clause construction. For example, repeatPlaceholder(2) returns ", ?, ?". func repeatPlaceholder(n int) string { if n <= 0 { return "" @@ -401,12 +426,14 @@ func repeatPlaceholder(n int) string { } // LogSQL logs SQL queries and their arguments when debug mode is enabled. -// Debug mode is activated by setting the GODEBUG environment variable to include "vaultik". -// This is useful for troubleshooting database operations and understanding query patterns. +// Debug mode is activated by setting the GODEBUG environment variable to +// include "vaultik". This is useful for troubleshooting database operations +// and understanding query patterns. // -// The operation parameter describes the type of SQL operation (e.g., "Execute", "Query"). -// The query parameter is the SQL statement being executed. -// The args parameter contains the query arguments that will be interpolated. +// The operation parameter describes the type of SQL operation (e.g., +// "Execute", "Query"). The query parameter is the SQL statement being +// executed. The args parameter contains the query arguments that will be +// interpolated. func LogSQL(operation, query string, args ...any) { if strings.Contains(os.Getenv("GODEBUG"), "vaultik") { log.Debug( diff --git a/internal/database/database_test.go b/internal/database/database_test.go index 7700ac1..9e1c214 100644 --- a/internal/database/database_test.go +++ b/internal/database/database_test.go @@ -1,3 +1,4 @@ +//nolint:testpackage // exercises unexported migration internals package database import ( @@ -9,6 +10,8 @@ import ( ) func TestDatabase(t *testing.T) { + t.Parallel() + ctx := context.Background() dbPath := filepath.Join(t.TempDir(), "test.db") @@ -39,7 +42,9 @@ func TestDatabase(t *testing.T) { for _, table := range tables { var name string - err := db.conn.QueryRow("SELECT name FROM sqlite_master WHERE type='table' AND name=?", table).Scan(&name) + err := db.conn.QueryRowContext(ctx, + "SELECT name FROM sqlite_master WHERE type='table' AND name=?", table, + ).Scan(&name) if err != nil { t.Errorf("table %s does not exist: %v", table, err) } @@ -47,6 +52,8 @@ func TestDatabase(t *testing.T) { } func TestDatabaseInvalidPath(t *testing.T) { + t.Parallel() + ctx := context.Background() // Test with invalid path @@ -57,6 +64,8 @@ func TestDatabaseInvalidPath(t *testing.T) { } func TestDatabaseConcurrentAccess(t *testing.T) { + t.Parallel() + ctx := context.Background() dbPath := filepath.Join(t.TempDir(), "test.db") @@ -81,7 +90,8 @@ func TestDatabaseConcurrentAccess(t *testing.T) { for i := range 10 { go func(i int) { - _, err := db.ExecWithLog(ctx, "INSERT INTO chunks (chunk_hash, size) VALUES (?, ?)", + _, err := db.ExecWithLog(ctx, + "INSERT INTO chunks (chunk_hash, size) VALUES (?, ?)", fmt.Sprintf("hash%d", i), i*1024) results <- result{index: i, err: err} }(i) @@ -109,6 +119,8 @@ func TestDatabaseConcurrentAccess(t *testing.T) { } func TestParseMigrationVersion(t *testing.T) { + t.Parallel() + tests := []struct { name string filename string @@ -118,8 +130,14 @@ func TestParseMigrationVersion(t *testing.T) { {name: "valid 000.sql", filename: "000.sql", wantVer: 0, wantError: false}, {name: "valid 001.sql", filename: "001.sql", wantVer: 1, wantError: false}, {name: "valid 099.sql", filename: "099.sql", wantVer: 99, wantError: false}, - {name: "valid with description", filename: "001_initial_schema.sql", wantVer: 1, wantError: false}, - {name: "valid large version", filename: "123_big_migration.sql", wantVer: 123, wantError: false}, + { + name: "valid with description", filename: "001_initial_schema.sql", + wantVer: 1, wantError: false, + }, + { + name: "valid large version", filename: "123_big_migration.sql", + wantVer: 123, wantError: false, + }, {name: "invalid alpha version", filename: "abc.sql", wantVer: 0, wantError: true}, {name: "invalid mixed chars", filename: "12a.sql", wantVer: 0, wantError: true}, {name: "invalid no extension", filename: "schema.sql", wantVer: 0, wantError: true}, @@ -128,29 +146,36 @@ func TestParseMigrationVersion(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { + t.Parallel() + got, err := ParseMigrationVersion(tc.filename) if tc.wantError { if err == nil { - t.Errorf("ParseMigrationVersion(%q) = %d, nil; want error", tc.filename, got) + t.Errorf("ParseMigrationVersion(%q) = %d, nil; want error", + tc.filename, got) } return } if err != nil { - t.Errorf("ParseMigrationVersion(%q) unexpected error: %v", tc.filename, err) + t.Errorf("ParseMigrationVersion(%q) unexpected error: %v", + tc.filename, err) return } if got != tc.wantVer { - t.Errorf("ParseMigrationVersion(%q) = %d; want %d", tc.filename, got, tc.wantVer) + t.Errorf("ParseMigrationVersion(%q) = %d; want %d", + tc.filename, got, tc.wantVer) } }) } } func TestApplyMigrations_Idempotent(t *testing.T) { + t.Parallel() + ctx := context.Background() conn, err := sql.Open("sqlite", ":memory:?_foreign_keys=ON") @@ -168,33 +193,46 @@ func TestApplyMigrations_Idempotent(t *testing.T) { conn.SetMaxIdleConns(1) // First run: apply all migrations. - if err := applyMigrations(ctx, conn); err != nil { + err = applyMigrations(ctx, conn) + if err != nil { t.Fatalf("first applyMigrations failed: %v", err) } // Count rows in schema_migrations after first run. var countBefore int - if err := conn.QueryRowContext(ctx, "SELECT COUNT(*) FROM schema_migrations").Scan(&countBefore); err != nil { + + err = conn.QueryRowContext(ctx, + "SELECT COUNT(*) FROM schema_migrations", + ).Scan(&countBefore) + if err != nil { t.Fatalf("failed to count schema_migrations after first run: %v", err) } // Second run: must be a no-op. - if err := applyMigrations(ctx, conn); err != nil { + err = applyMigrations(ctx, conn) + if err != nil { t.Fatalf("second applyMigrations failed: %v", err) } // Count rows in schema_migrations after second run — must be unchanged. var countAfter int - if err := conn.QueryRowContext(ctx, "SELECT COUNT(*) FROM schema_migrations").Scan(&countAfter); err != nil { + + err = conn.QueryRowContext(ctx, + "SELECT COUNT(*) FROM schema_migrations", + ).Scan(&countAfter) + if err != nil { t.Fatalf("failed to count schema_migrations after second run: %v", err) } if countBefore != countAfter { - t.Errorf("schema_migrations row count changed: before=%d, after=%d", countBefore, countAfter) + t.Errorf("schema_migrations row count changed: before=%d, after=%d", + countBefore, countAfter) } } func TestBootstrapMigrationsTable_FreshDatabase(t *testing.T) { + t.Parallel() + ctx := context.Background() conn, err := sql.Open("sqlite", ":memory:?_foreign_keys=ON") @@ -213,9 +251,11 @@ func TestBootstrapMigrationsTable_FreshDatabase(t *testing.T) { // Verify schema_migrations does NOT exist yet. var tableBefore int - if err := conn.QueryRowContext(ctx, + + err = conn.QueryRowContext(ctx, "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='schema_migrations'", - ).Scan(&tableBefore); err != nil { + ).Scan(&tableBefore) + if err != nil { t.Fatalf("failed to check for table before bootstrap: %v", err) } @@ -224,27 +264,33 @@ func TestBootstrapMigrationsTable_FreshDatabase(t *testing.T) { } // Run bootstrap. - if err := bootstrapMigrationsTable(ctx, conn); err != nil { + err = bootstrapMigrationsTable(ctx, conn) + if err != nil { t.Fatalf("bootstrapMigrationsTable failed: %v", err) } // Verify schema_migrations now exists. var tableAfter int - if err := conn.QueryRowContext(ctx, + + err = conn.QueryRowContext(ctx, "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='schema_migrations'", - ).Scan(&tableAfter); err != nil { + ).Scan(&tableAfter) + if err != nil { t.Fatalf("failed to check for table after bootstrap: %v", err) } if tableAfter != 1 { - t.Fatalf("schema_migrations table should exist after bootstrap, got count=%d", tableAfter) + t.Fatalf("schema_migrations table should exist after bootstrap, got count=%d", + tableAfter) } // Verify version 0 row exists. var version int - if err := conn.QueryRowContext(ctx, + + err = conn.QueryRowContext(ctx, "SELECT version FROM schema_migrations WHERE version = 0", - ).Scan(&version); err != nil { + ).Scan(&version) + if err != nil { t.Fatalf("version 0 row not found in schema_migrations: %v", err) } diff --git a/internal/database/file_chunks.go b/internal/database/file_chunks.go index c6b32f4..11711f7 100644 --- a/internal/database/file_chunks.go +++ b/internal/database/file_chunks.go @@ -9,15 +9,21 @@ import ( "sneak.berlin/go/vaultik/internal/types" ) +// FileChunkRepository provides access to the file_chunks table, which maps +// files to their ordered constituent chunks. type FileChunkRepository struct { db *DB } +// NewFileChunkRepository creates a FileChunkRepository backed by db. func NewFileChunkRepository(db *DB) *FileChunkRepository { return &FileChunkRepository{db: db} } -func (r *FileChunkRepository) Create(ctx context.Context, tx *sql.Tx, fc *FileChunk) error { +// Create inserts a file_chunks row (idempotently), using tx when non-nil. +func (r *FileChunkRepository) Create( + ctx context.Context, tx *sql.Tx, fc *FileChunk, +) error { query := ` INSERT INTO file_chunks (file_id, idx, chunk_hash) VALUES (?, ?, ?) @@ -28,7 +34,8 @@ func (r *FileChunkRepository) Create(ctx context.Context, tx *sql.Tx, fc *FileCh if tx != nil { _, err = tx.ExecContext(ctx, query, fc.FileID.String(), fc.Idx, fc.ChunkHash.String()) } else { - _, err = r.db.ExecWithLog(ctx, query, fc.FileID.String(), fc.Idx, fc.ChunkHash.String()) + _, err = r.db.ExecWithLog(ctx, query, + fc.FileID.String(), fc.Idx, fc.ChunkHash.String()) } if err != nil { @@ -38,7 +45,10 @@ func (r *FileChunkRepository) Create(ctx context.Context, tx *sql.Tx, fc *FileCh return nil } -func (r *FileChunkRepository) GetByPath(ctx context.Context, path string) ([]*FileChunk, error) { +// GetByPath returns the ordered chunks of the file at the given path. +func (r *FileChunkRepository) GetByPath( + ctx context.Context, path string, +) ([]*FileChunk, error) { query := ` SELECT fc.file_id, fc.idx, fc.chunk_hash FROM file_chunks fc @@ -57,7 +67,9 @@ func (r *FileChunkRepository) GetByPath(ctx context.Context, path string) ([]*Fi } // GetByFileID retrieves file chunks by file ID -func (r *FileChunkRepository) GetByFileID(ctx context.Context, fileID types.FileID) ([]*FileChunk, error) { +func (r *FileChunkRepository) GetByFileID( + ctx context.Context, fileID types.FileID, +) ([]*FileChunk, error) { query := ` SELECT file_id, idx, chunk_hash FROM file_chunks @@ -75,7 +87,9 @@ func (r *FileChunkRepository) GetByFileID(ctx context.Context, fileID types.File } // GetByPathTx retrieves file chunks within a transaction -func (r *FileChunkRepository) GetByPathTx(ctx context.Context, tx *sql.Tx, path string) ([]*FileChunk, error) { +func (r *FileChunkRepository) GetByPathTx( + ctx context.Context, tx *sql.Tx, path string, +) ([]*FileChunk, error) { query := ` SELECT fc.file_id, fc.idx, fc.chunk_hash FROM file_chunks fc @@ -98,6 +112,170 @@ func (r *FileChunkRepository) GetByPathTx(ctx context.Context, tx *sql.Tx, path return fileChunks, err } +// DeleteByPath deletes all file_chunks rows for the file at the given path. +func (r *FileChunkRepository) DeleteByPath( + ctx context.Context, tx *sql.Tx, path string, +) error { + query := ` + DELETE FROM file_chunks + WHERE file_id = (SELECT id FROM files WHERE path = ?) + ` + + var err error + if tx != nil { + _, err = tx.ExecContext(ctx, query, path) + } else { + _, err = r.db.ExecWithLog(ctx, query, path) + } + + if err != nil { + return fmt.Errorf("deleting file chunks: %w", err) + } + + return nil +} + +// DeleteByFileID deletes all chunks for a file by its UUID +func (r *FileChunkRepository) DeleteByFileID( + ctx context.Context, tx *sql.Tx, fileID types.FileID, +) error { + query := `DELETE FROM file_chunks WHERE file_id = ?` + + var err error + if tx != nil { + _, err = tx.ExecContext(ctx, query, fileID.String()) + } else { + _, err = r.db.ExecWithLog(ctx, query, fileID.String()) + } + + if err != nil { + return fmt.Errorf("deleting file chunks: %w", err) + } + + return nil +} + +// DeleteByFileIDs deletes all chunks for multiple files in a single statement. +// +//nolint:dupl // symmetric implementation for a parallel association table +func (r *FileChunkRepository) DeleteByFileIDs( + ctx context.Context, tx *sql.Tx, fileIDs []types.FileID, +) error { + if len(fileIDs) == 0 { + return nil + } + + // Batch at 500 to stay within SQLite's variable limit + const batchSize = 500 + + for i := 0; i < len(fileIDs); i += batchSize { + end := min(i+batchSize, len(fileIDs)) + + batch := fileIDs[i:end] + + //nolint:gosec // G202: concatenates constant SQL and "?" placeholders only + query := "DELETE FROM file_chunks WHERE file_id IN (?" + + repeatPlaceholder(len(batch)-1) + ")" + + args := make([]any, len(batch)) + for j, id := range batch { + args[j] = id.String() + } + + var err error + if tx != nil { + _, err = tx.ExecContext(ctx, query, args...) + } else { + _, err = r.db.ExecWithLog(ctx, query, args...) + } + + if err != nil { + return fmt.Errorf("batch deleting file_chunks: %w", err) + } + } + + return nil +} + +// CreateBatch inserts multiple file_chunks in a single statement for efficiency. +// Batches are automatically split to stay within SQLite's variable limit. +func (r *FileChunkRepository) CreateBatch( + ctx context.Context, tx *sql.Tx, fcs []FileChunk, +) error { + if len(fcs) == 0 { + return nil + } + + // Each file_chunks row binds this many SQL variables. + const fileChunkCols = 3 + + // SQLite has a limit on variables (typically 999 or 32766), so batch + // at 300 rows to be safe. + const batchSize = 300 + + for i := 0; i < len(fcs); i += batchSize { + end := min(i+batchSize, len(fcs)) + + batch := fcs[i:end] + + // Build the query with multiple value sets + query := "INSERT INTO file_chunks (file_id, idx, chunk_hash) VALUES " + + args := make([]any, 0, len(batch)*fileChunkCols) + + var querySb211 strings.Builder + + for j, fc := range batch { + if j > 0 { + querySb211.WriteString(", ") + } + + querySb211.WriteString("(?, ?, ?)") + + args = append(args, fc.FileID.String(), fc.Idx, fc.ChunkHash.String()) + } + + query += querySb211.String() //nolint:gosec // G202: appends "?" placeholders only + + query += " ON CONFLICT(file_id, idx) DO NOTHING" + + var err error + if tx != nil { + _, err = tx.ExecContext(ctx, query, args...) + } else { + _, err = r.db.ExecWithLog(ctx, query, args...) + } + + if err != nil { + return fmt.Errorf("batch inserting file_chunks: %w", err) + } + } + + return nil +} + +// GetByFile is an alias for GetByPath for compatibility +func (r *FileChunkRepository) GetByFile( + ctx context.Context, path string, +) ([]*FileChunk, error) { + LogSQL("GetByFile", "Starting", path) + result, err := r.GetByPath(ctx, path) + LogSQL("GetByFile", "Complete", path, "count", len(result)) + + return result, err +} + +// GetByFileTx retrieves file chunks within a transaction +func (r *FileChunkRepository) GetByFileTx( + ctx context.Context, tx *sql.Tx, path string, +) ([]*FileChunk, error) { + LogSQL("GetByFileTx", "Starting", path) + result, err := r.GetByPathTx(ctx, tx, path) + LogSQL("GetByFileTx", "Complete", path, "count", len(result)) + + return result, err +} + // scanFileChunks is a helper that scans file chunk rows func (r *FileChunkRepository) scanFileChunks(rows *sql.Rows) ([]*FileChunk, error) { var fileChunks []*FileChunk @@ -124,144 +302,3 @@ func (r *FileChunkRepository) scanFileChunks(rows *sql.Rows) ([]*FileChunk, erro return fileChunks, rows.Err() } - -func (r *FileChunkRepository) DeleteByPath(ctx context.Context, tx *sql.Tx, path string) error { - query := `DELETE FROM file_chunks WHERE file_id = (SELECT id FROM files WHERE path = ?)` - - var err error - if tx != nil { - _, err = tx.ExecContext(ctx, query, path) - } else { - _, err = r.db.ExecWithLog(ctx, query, path) - } - - if err != nil { - return fmt.Errorf("deleting file chunks: %w", err) - } - - return nil -} - -// DeleteByFileID deletes all chunks for a file by its UUID -func (r *FileChunkRepository) DeleteByFileID(ctx context.Context, tx *sql.Tx, fileID types.FileID) error { - query := `DELETE FROM file_chunks WHERE file_id = ?` - - var err error - if tx != nil { - _, err = tx.ExecContext(ctx, query, fileID.String()) - } else { - _, err = r.db.ExecWithLog(ctx, query, fileID.String()) - } - - if err != nil { - return fmt.Errorf("deleting file chunks: %w", err) - } - - return nil -} - -// DeleteByFileIDs deletes all chunks for multiple files in a single statement. -func (r *FileChunkRepository) DeleteByFileIDs(ctx context.Context, tx *sql.Tx, fileIDs []types.FileID) error { - if len(fileIDs) == 0 { - return nil - } - - // Batch at 500 to stay within SQLite's variable limit - const batchSize = 500 - - for i := 0; i < len(fileIDs); i += batchSize { - end := min(i+batchSize, len(fileIDs)) - - batch := fileIDs[i:end] - - query := "DELETE FROM file_chunks WHERE file_id IN (?" + repeatPlaceholder(len(batch)-1) + ")" - - args := make([]any, len(batch)) - for j, id := range batch { - args[j] = id.String() - } - - var err error - if tx != nil { - _, err = tx.ExecContext(ctx, query, args...) - } else { - _, err = r.db.ExecWithLog(ctx, query, args...) - } - - if err != nil { - return fmt.Errorf("batch deleting file_chunks: %w", err) - } - } - - return nil -} - -// CreateBatch inserts multiple file_chunks in a single statement for efficiency. -// Batches are automatically split to stay within SQLite's variable limit. -func (r *FileChunkRepository) CreateBatch(ctx context.Context, tx *sql.Tx, fcs []FileChunk) error { - if len(fcs) == 0 { - return nil - } - - // SQLite has a limit on variables (typically 999 or 32766). - // Each FileChunk has 3 values, so batch at 300 to be safe. - const batchSize = 300 - - for i := 0; i < len(fcs); i += batchSize { - end := min(i+batchSize, len(fcs)) - - batch := fcs[i:end] - - // Build the query with multiple value sets - query := "INSERT INTO file_chunks (file_id, idx, chunk_hash) VALUES " - - args := make([]any, 0, len(batch)*3) - - var querySb211 strings.Builder - - for j, fc := range batch { - if j > 0 { - querySb211.WriteString(", ") - } - - querySb211.WriteString("(?, ?, ?)") - - args = append(args, fc.FileID.String(), fc.Idx, fc.ChunkHash.String()) - } - - query += querySb211.String() - - query += " ON CONFLICT(file_id, idx) DO NOTHING" - - var err error - if tx != nil { - _, err = tx.ExecContext(ctx, query, args...) - } else { - _, err = r.db.ExecWithLog(ctx, query, args...) - } - - if err != nil { - return fmt.Errorf("batch inserting file_chunks: %w", err) - } - } - - return nil -} - -// GetByFile is an alias for GetByPath for compatibility -func (r *FileChunkRepository) GetByFile(ctx context.Context, path string) ([]*FileChunk, error) { - LogSQL("GetByFile", "Starting", path) - result, err := r.GetByPath(ctx, path) - LogSQL("GetByFile", "Complete", path, "count", len(result)) - - return result, err -} - -// GetByFileTx retrieves file chunks within a transaction -func (r *FileChunkRepository) GetByFileTx(ctx context.Context, tx *sql.Tx, path string) ([]*FileChunk, error) { - LogSQL("GetByFileTx", "Starting", path) - result, err := r.GetByPathTx(ctx, tx, path) - LogSQL("GetByFileTx", "Complete", path, "count", len(result)) - - return result, err -} diff --git a/internal/database/file_chunks_test.go b/internal/database/file_chunks_test.go index e1eb2b9..7fe58e2 100644 --- a/internal/database/file_chunks_test.go +++ b/internal/database/file_chunks_test.go @@ -1,4 +1,4 @@ -package database +package database_test import ( "context" @@ -6,21 +6,25 @@ import ( "testing" "time" + "sneak.berlin/go/vaultik/internal/database" "sneak.berlin/go/vaultik/internal/types" ) func TestFileChunkRepository(t *testing.T) { + t.Parallel() + db, cleanup := setupTestDB(t) defer cleanup() ctx := context.Background() - repo := NewFileChunkRepository(db) - fileRepo := NewFileRepository(db) + repo := database.NewFileChunkRepository(db) + fileRepo := database.NewFileRepository(db) + repos := database.NewRepositories(db) // Create test file first testTime := time.Now().Truncate(time.Second) - file := &File{ - Path: "/test/file.txt", + file := &database.File{ + Path: testFileTxt, MTime: testTime, Size: 3072, Mode: 0644, @@ -29,44 +33,26 @@ func TestFileChunkRepository(t *testing.T) { LinkTarget: "", } - err := fileRepo.Create(ctx, nil, file) - if err != nil { - t.Fatalf("failed to create file: %v", err) - } - - // Create chunks first - chunks := []types.ChunkHash{"chunk1", "chunk2", "chunk3"} - chunkRepo := NewChunkRepository(db) - - for _, chunkHash := range chunks { - chunk := &Chunk{ - ChunkHash: chunkHash, - Size: 1024, - } - - err = chunkRepo.Create(ctx, nil, chunk) - if err != nil { - t.Fatalf("failed to create chunk %s: %v", chunkHash, err) - } - } + mustCreateFile(t, fileRepo, file) + mustCreateChunks(t, repos, chunk1Hash, chunk2Hash, chunk3Hash) // Test Create - fc1 := &FileChunk{ + fc1 := &database.FileChunk{ FileID: file.ID, Idx: 0, - ChunkHash: types.ChunkHash("chunk1"), + ChunkHash: types.ChunkHash(chunk1Hash), } - err = repo.Create(ctx, nil, fc1) + err := repo.Create(ctx, nil, fc1) if err != nil { t.Fatalf("failed to create file chunk: %v", err) } // Add more chunks for the same file - fc2 := &FileChunk{ + fc2 := &database.FileChunk{ FileID: file.ID, Idx: 1, - ChunkHash: types.ChunkHash("chunk2"), + ChunkHash: types.ChunkHash(chunk2Hash), } err = repo.Create(ctx, nil, fc2) @@ -74,10 +60,10 @@ func TestFileChunkRepository(t *testing.T) { t.Fatalf("failed to create second file chunk: %v", err) } - fc3 := &FileChunk{ + fc3 := &database.FileChunk{ FileID: file.ID, Idx: 2, - ChunkHash: types.ChunkHash("chunk3"), + ChunkHash: types.ChunkHash(chunk3Hash), } err = repo.Create(ctx, nil, fc3) @@ -86,7 +72,7 @@ func TestFileChunkRepository(t *testing.T) { } // Test GetByFile - fileChunks, err := repo.GetByFile(ctx, "/test/file.txt") + fileChunks, err := repo.GetByFile(ctx, testFileTxt) if err != nil { t.Fatalf("failed to get file chunks: %v", err) } @@ -107,6 +93,41 @@ func TestFileChunkRepository(t *testing.T) { if err != nil { t.Fatalf("failed to create duplicate file chunk: %v", err) } +} + +func TestFileChunkRepositoryDeleteByFileID(t *testing.T) { + t.Parallel() + + db, cleanup := setupTestDB(t) + defer cleanup() + + ctx := context.Background() + repo := database.NewFileChunkRepository(db) + fileRepo := database.NewFileRepository(db) + repos := database.NewRepositories(db) + + file := &database.File{ + Path: testFileTxt, + MTime: time.Now().Truncate(time.Second), + Size: 1024, + Mode: 0644, + UID: 1000, + GID: 1000, + } + + mustCreateFile(t, fileRepo, file) + mustCreateChunks(t, repos, chunk1Hash) + + fc := &database.FileChunk{ + FileID: file.ID, + Idx: 0, + ChunkHash: types.ChunkHash(chunk1Hash), + } + + err := repo.Create(ctx, nil, fc) + if err != nil { + t.Fatalf("failed to create file chunk: %v", err) + } // Test DeleteByFileID err = repo.DeleteByFileID(ctx, nil, file.ID) @@ -114,7 +135,7 @@ func TestFileChunkRepository(t *testing.T) { t.Fatalf("failed to delete file chunks: %v", err) } - fileChunks, err = repo.GetByFileID(ctx, file.ID) + fileChunks, err := repo.GetByFileID(ctx, file.ID) if err != nil { t.Fatalf("failed to get deleted file chunks: %v", err) } @@ -125,20 +146,22 @@ func TestFileChunkRepository(t *testing.T) { } func TestFileChunkRepositoryMultipleFiles(t *testing.T) { + t.Parallel() + db, cleanup := setupTestDB(t) defer cleanup() ctx := context.Background() - repo := NewFileChunkRepository(db) - fileRepo := NewFileRepository(db) + repo := database.NewFileChunkRepository(db) + fileRepo := database.NewFileRepository(db) // Create test files testTime := time.Now().Truncate(time.Second) - filePaths := []string{"/file1.txt", "/file2.txt", "/file3.txt"} - files := make([]*File, len(filePaths)) + filePaths := []string{testFilePath1, testFilePath2, "/file3.txt"} + files := make([]*database.File, len(filePaths)) for i, path := range filePaths { - file := &File{ + file := &database.File{ Path: types.FilePath(path), MTime: testTime, Size: 2048, @@ -148,21 +171,18 @@ func TestFileChunkRepositoryMultipleFiles(t *testing.T) { LinkTarget: "", } - err := fileRepo.Create(ctx, nil, file) - if err != nil { - t.Fatalf("failed to create file %s: %v", path, err) - } + mustCreateFile(t, fileRepo, file) files[i] = file } // Create all chunks first - chunkRepo := NewChunkRepository(db) + chunkRepo := database.NewChunkRepository(db) for i := range files { for j := range 2 { chunkHash := types.ChunkHash(fmt.Sprintf("file%d_chunk%d", i, j)) - chunk := &Chunk{ + chunk := &database.Chunk{ ChunkHash: chunkHash, Size: 1024, } @@ -177,7 +197,7 @@ func TestFileChunkRepositoryMultipleFiles(t *testing.T) { // Create chunks for multiple files for i, file := range files { for j := range 2 { - fc := &FileChunk{ + fc := &database.FileChunk{ FileID: file.ID, Idx: j, ChunkHash: types.ChunkHash(fmt.Sprintf("file%d_chunk%d", i, j)), diff --git a/internal/database/files.go b/internal/database/files.go index 77ec1ec..4beb5d6 100644 --- a/internal/database/files.go +++ b/internal/database/files.go @@ -12,14 +12,20 @@ import ( "sneak.berlin/go/vaultik/internal/types" ) +// FileRepository provides access to the files table, which stores file +// metadata (path, times, permissions, ownership, symlink targets). type FileRepository struct { db *DB } +// NewFileRepository creates a FileRepository backed by db. func NewFileRepository(db *DB) *FileRepository { return &FileRepository{db: db} } +// Create inserts or updates a file row (upsert on path), using tx when +// non-nil. The file's ID is generated when zero and updated from the +// database's RETURNING clause. func (r *FileRepository) Create(ctx context.Context, tx *sql.Tx, file *File) error { // Generate UUID if not provided if file.ID.IsZero() { @@ -46,10 +52,19 @@ func (r *FileRepository) Create(ctx context.Context, tx *sql.Tx, file *File) err ) if tx != nil { - LogSQL("Execute", query, file.ID.String(), file.Path.String(), file.SourcePath.String(), file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID, file.LinkTarget.String()) - err = tx.QueryRowContext(ctx, query, file.ID.String(), file.Path.String(), file.SourcePath.String(), file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID, file.LinkTarget.String()).Scan(&idStr) + LogSQL("Execute", query, + file.ID.String(), file.Path.String(), file.SourcePath.String(), + file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID, + file.LinkTarget.String()) + err = tx.QueryRowContext(ctx, query, + file.ID.String(), file.Path.String(), file.SourcePath.String(), + file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID, + file.LinkTarget.String()).Scan(&idStr) } else { - err = r.db.QueryRowWithLog(ctx, query, file.ID.String(), file.Path.String(), file.SourcePath.String(), file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID, file.LinkTarget.String()).Scan(&idStr) + err = r.db.QueryRowWithLog(ctx, query, + file.ID.String(), file.Path.String(), file.SourcePath.String(), + file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID, + file.LinkTarget.String()).Scan(&idStr) } if err != nil { @@ -65,6 +80,8 @@ func (r *FileRepository) Create(ctx context.Context, tx *sql.Tx, file *File) err return nil } +// GetByPath returns the file at the given path, or nil if the path is not +// in the index. func (r *FileRepository) GetByPath(ctx context.Context, path string) (*File, error) { query := ` SELECT id, path, source_path, mtime, size, mode, uid, gid, link_target @@ -74,7 +91,7 @@ func (r *FileRepository) GetByPath(ctx context.Context, path string) (*File, err file, err := r.scanFile(r.db.conn.QueryRowContext(ctx, query, path)) if errors.Is(err, sql.ErrNoRows) { - return nil, nil + return nil, nil //nolint:nilnil // nil,nil signals not-found; callers check nil } if err != nil { @@ -94,7 +111,7 @@ func (r *FileRepository) GetByID(ctx context.Context, id types.FileID) (*File, e file, err := r.scanFile(r.db.conn.QueryRowContext(ctx, query, id.String())) if errors.Is(err, sql.ErrNoRows) { - return nil, nil + return nil, nil //nolint:nilnil // nil,nil signals not-found; callers check nil } if err != nil { @@ -104,7 +121,11 @@ func (r *FileRepository) GetByID(ctx context.Context, id types.FileID) (*File, e return file, nil } -func (r *FileRepository) GetByPathTx(ctx context.Context, tx *sql.Tx, path string) (*File, error) { +// GetByPathTx returns the file at the given path within a transaction, or +// nil if the path is not in the index. +func (r *FileRepository) GetByPathTx( + ctx context.Context, tx *sql.Tx, path string, +) (*File, error) { query := ` SELECT id, path, source_path, mtime, size, mode, uid, gid, link_target FROM files @@ -116,7 +137,7 @@ func (r *FileRepository) GetByPathTx(ctx context.Context, tx *sql.Tx, path strin LogSQL("GetByPathTx Scan complete", query, path) if errors.Is(err, sql.ErrNoRows) { - return nil, nil + return nil, nil //nolint:nilnil // nil,nil signals not-found; callers check nil } if err != nil { @@ -126,87 +147,16 @@ func (r *FileRepository) GetByPathTx(ctx context.Context, tx *sql.Tx, path strin return file, nil } -// scanFile is a helper that scans a single file row -func (r *FileRepository) scanFile(row *sql.Row) (*File, error) { - var ( - file File - idStr, pathStr, sourcePathStr string - mtimeUnix int64 - linkTarget sql.NullString - ) - - err := row.Scan( - &idStr, - &pathStr, - &sourcePathStr, - &mtimeUnix, - &file.Size, - &file.Mode, - &file.UID, - &file.GID, - &linkTarget, - ) - if err != nil { - return nil, err - } - - file.ID, err = types.ParseFileID(idStr) - if err != nil { - return nil, fmt.Errorf("parsing file ID: %w", err) - } - - file.Path = types.FilePath(pathStr) - file.SourcePath = types.SourcePath(sourcePathStr) - - file.MTime = time.Unix(mtimeUnix, 0).UTC() - if linkTarget.Valid { - file.LinkTarget = types.FilePath(linkTarget.String) - } - - return &file, nil +// fileRowScanner abstracts *sql.Row and *sql.Rows for scanning a file row. +type fileRowScanner interface { + Scan(dest ...any) error } -// scanFileRows is a helper that scans a file row from rows iterator -func (r *FileRepository) scanFileRows(rows *sql.Rows) (*File, error) { - var ( - file File - idStr, pathStr, sourcePathStr string - mtimeUnix int64 - linkTarget sql.NullString - ) - - err := rows.Scan( - &idStr, - &pathStr, - &sourcePathStr, - &mtimeUnix, - &file.Size, - &file.Mode, - &file.UID, - &file.GID, - &linkTarget, - ) - if err != nil { - return nil, err - } - - file.ID, err = types.ParseFileID(idStr) - if err != nil { - return nil, fmt.Errorf("parsing file ID: %w", err) - } - - file.Path = types.FilePath(pathStr) - file.SourcePath = types.SourcePath(sourcePathStr) - - file.MTime = time.Unix(mtimeUnix, 0).UTC() - if linkTarget.Valid { - file.LinkTarget = types.FilePath(linkTarget.String) - } - - return &file, nil -} - -func (r *FileRepository) ListModifiedSince(ctx context.Context, since time.Time) ([]*File, error) { +// ListModifiedSince returns all files whose recorded mtime is at or after +// since, ordered by path. +func (r *FileRepository) ListModifiedSince( + ctx context.Context, since time.Time, +) ([]*File, error) { query := ` SELECT id, path, source_path, mtime, size, mode, uid, gid, link_target FROM files @@ -234,6 +184,7 @@ func (r *FileRepository) ListModifiedSince(ctx context.Context, since time.Time) return files, rows.Err() } +// Delete removes the file row at the given path, using tx when non-nil. func (r *FileRepository) Delete(ctx context.Context, tx *sql.Tx, path string) error { query := `DELETE FROM files WHERE path = ?` @@ -252,7 +203,9 @@ func (r *FileRepository) Delete(ctx context.Context, tx *sql.Tx, path string) er } // DeleteByID deletes a file by its UUID -func (r *FileRepository) DeleteByID(ctx context.Context, tx *sql.Tx, id types.FileID) error { +func (r *FileRepository) DeleteByID( + ctx context.Context, tx *sql.Tx, id types.FileID, +) error { query := `DELETE FROM files WHERE id = ?` var err error @@ -269,7 +222,11 @@ func (r *FileRepository) DeleteByID(ctx context.Context, tx *sql.Tx, id types.Fi return nil } -func (r *FileRepository) ListByPrefix(ctx context.Context, prefix string) ([]*File, error) { +// ListByPrefix returns all files whose path starts with prefix, ordered by +// path. +func (r *FileRepository) ListByPrefix( + ctx context.Context, prefix string, +) ([]*File, error) { query := ` SELECT id, path, source_path, mtime, size, mode, uid, gid, link_target FROM files @@ -327,12 +284,17 @@ func (r *FileRepository) ListAll(ctx context.Context) ([]*File, error) { // CreateBatch inserts or updates multiple files in a single statement for efficiency. // File IDs must be pre-generated before calling this method. -func (r *FileRepository) CreateBatch(ctx context.Context, tx *sql.Tx, files []*File) error { +func (r *FileRepository) CreateBatch( + ctx context.Context, tx *sql.Tx, files []*File, +) error { if len(files) == 0 { return nil } - // Each File has 9 values, so batch at 100 to be safe with SQLite's variable limit + // Each files row binds this many SQL variables. + const fileCols = 9 + + // Batch at 100 rows to be safe with SQLite's variable limit. const batchSize = 100 for i := 0; i < len(files); i += batchSize { @@ -340,9 +302,11 @@ func (r *FileRepository) CreateBatch(ctx context.Context, tx *sql.Tx, files []*F batch := files[i:end] - query := `INSERT INTO files (id, path, source_path, mtime, size, mode, uid, gid, link_target) VALUES ` + query := `INSERT INTO files + (id, path, source_path, mtime, size, mode, uid, gid, link_target) + VALUES ` - args := make([]any, 0, len(batch)*9) + args := make([]any, 0, len(batch)*fileCols) var querySb325 strings.Builder @@ -353,10 +317,13 @@ func (r *FileRepository) CreateBatch(ctx context.Context, tx *sql.Tx, files []*F querySb325.WriteString("(?, ?, ?, ?, ?, ?, ?, ?, ?)") - args = append(args, f.ID.String(), f.Path.String(), f.SourcePath.String(), f.MTime.Unix(), f.Size, f.Mode, f.UID, f.GID, f.LinkTarget.String()) + args = append(args, + f.ID.String(), f.Path.String(), f.SourcePath.String(), + f.MTime.Unix(), f.Size, f.Mode, f.UID, f.GID, + f.LinkTarget.String()) } - query += querySb325.String() + query += querySb325.String() //nolint:gosec // G202: appends "?" placeholders only query += ` ON CONFLICT(path) DO UPDATE SET source_path = excluded.source_path, @@ -404,3 +371,53 @@ func (r *FileRepository) DeleteOrphaned(ctx context.Context) error { return nil } + +// scanFile is a helper that scans a single file row +func (r *FileRepository) scanFile(row *sql.Row) (*File, error) { + return r.scanFileFrom(row) +} + +// scanFileRows is a helper that scans a file row from rows iterator +func (r *FileRepository) scanFileRows(rows *sql.Rows) (*File, error) { + return r.scanFileFrom(rows) +} + +// scanFileFrom scans one file row from any row scanner. +func (r *FileRepository) scanFileFrom(row fileRowScanner) (*File, error) { + var ( + file File + idStr, pathStr, sourcePathStr string + mtimeUnix int64 + linkTarget sql.NullString + ) + + err := row.Scan( + &idStr, + &pathStr, + &sourcePathStr, + &mtimeUnix, + &file.Size, + &file.Mode, + &file.UID, + &file.GID, + &linkTarget, + ) + if err != nil { + return nil, err + } + + file.ID, err = types.ParseFileID(idStr) + if err != nil { + return nil, fmt.Errorf("parsing file ID: %w", err) + } + + file.Path = types.FilePath(pathStr) + file.SourcePath = types.SourcePath(sourcePathStr) + + file.MTime = time.Unix(mtimeUnix, 0).UTC() + if linkTarget.Valid { + file.LinkTarget = types.FilePath(linkTarget.String) + } + + return &file, nil +} diff --git a/internal/database/files_test.go b/internal/database/files_test.go index 7f4b472..2d4823b 100644 --- a/internal/database/files_test.go +++ b/internal/database/files_test.go @@ -1,44 +1,32 @@ -package database +package database_test import ( "context" "database/sql" "errors" "os" - "path/filepath" "testing" "time" + + "sneak.berlin/go/vaultik/internal/database" ) -func setupTestDB(t *testing.T) (*DB, func()) { - ctx := context.Background() - dbPath := filepath.Join(t.TempDir(), "test.db") - - db, err := New(ctx, dbPath) - if err != nil { - t.Fatalf("failed to create database: %v", err) - } - - cleanup := func() { - err := db.Close() - if err != nil { - t.Errorf("failed to close database: %v", err) - } - } - - return db, cleanup -} +// errTestRollback is the sentinel returned from transaction bodies to +// force a rollback in tests. +var errTestRollback = errors.New("test rollback") func TestFileRepository(t *testing.T) { + t.Parallel() + db, cleanup := setupTestDB(t) defer cleanup() ctx := context.Background() - repo := NewFileRepository(db) + repo := database.NewFileRepository(db) // Test Create - file := &File{ - Path: "/test/file.txt", + file := &database.File{ + Path: testFileTxt, MTime: time.Now().Truncate(time.Second), Size: 1024, Mode: 0644, @@ -95,6 +83,30 @@ func TestFileRepository(t *testing.T) { if retrieved.Size != 2048 { t.Errorf("size not updated: got %d, want %d", retrieved.Size, 2048) } +} + +func TestFileRepositoryListDelete(t *testing.T) { + t.Parallel() + + db, cleanup := setupTestDB(t) + defer cleanup() + + ctx := context.Background() + repo := database.NewFileRepository(db) + + file := &database.File{ + Path: testFileTxt, + MTime: time.Now().Truncate(time.Second), + Size: 1024, + Mode: 0644, + UID: 1000, + GID: 1000, + } + + err := repo.Create(ctx, nil, file) + if err != nil { + t.Fatalf("failed to create file: %v", err) + } // Test ListModifiedSince files, err := repo.ListModifiedSince(ctx, time.Now().Add(-1*time.Hour)) @@ -112,7 +124,7 @@ func TestFileRepository(t *testing.T) { t.Fatalf("failed to delete file: %v", err) } - retrieved, err = repo.GetByPath(ctx, file.Path.String()) + retrieved, err := repo.GetByPath(ctx, file.Path.String()) if err != nil { t.Fatalf("error getting deleted file: %v", err) } @@ -123,14 +135,16 @@ func TestFileRepository(t *testing.T) { } func TestFileRepositorySymlink(t *testing.T) { + t.Parallel() + db, cleanup := setupTestDB(t) defer cleanup() ctx := context.Background() - repo := NewFileRepository(db) + repo := database.NewFileRepository(db) // Test symlink - symlink := &File{ + symlink := &database.File{ Path: "/test/link", MTime: time.Now().Truncate(time.Second), Size: 0, @@ -155,21 +169,24 @@ func TestFileRepositorySymlink(t *testing.T) { } if retrieved.LinkTarget != symlink.LinkTarget { - t.Errorf("link target mismatch: got %s, want %s", retrieved.LinkTarget, symlink.LinkTarget) + t.Errorf("link target mismatch: got %s, want %s", + retrieved.LinkTarget, symlink.LinkTarget) } } func TestFileRepositoryTransaction(t *testing.T) { + t.Parallel() + db, cleanup := setupTestDB(t) defer cleanup() ctx := context.Background() - repos := NewRepositories(db) + repos := database.NewRepositories(db) // Test transaction rollback err := repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { - file := &File{ - Path: "/test/tx_file.txt", + file := &database.File{ + Path: testTxFile, MTime: time.Now().Truncate(time.Second), Size: 1024, Mode: 0644, @@ -183,15 +200,14 @@ func TestFileRepositoryTransaction(t *testing.T) { } // Return error to trigger rollback - return errors.New("test rollback") + return errTestRollback }) - - if err == nil || err.Error() != "test rollback" { + if !errors.Is(err, errTestRollback) { t.Fatalf("expected rollback error, got: %v", err) } // Verify file was not created - retrieved, err := repos.Files.GetByPath(ctx, "/test/tx_file.txt") + retrieved, err := repos.Files.GetByPath(ctx, testTxFile) if err != nil { t.Fatalf("error checking for file: %v", err) } diff --git a/internal/database/helpers_internal_test.go b/internal/database/helpers_internal_test.go new file mode 100644 index 0000000..8b27f67 --- /dev/null +++ b/internal/database/helpers_internal_test.go @@ -0,0 +1,81 @@ +package database + +import ( + "context" + "path/filepath" + "testing" + + "sneak.berlin/go/vaultik/internal/types" +) + +// Common fixture values shared by the internal repository tests. +const ( + internalTestHost = "test-host" + internalTestSnapshotID = "test-snapshot" + internalTestFilePath = "/test.txt" + internalTestFile1 = "/file1.txt" + internalTestFile2 = "/file2.txt" + + // countFilesQuery counts the rows of the files table. + countFilesQuery = "SELECT COUNT(*) FROM files" +) + +// mustCreateFileRow inserts the file row, failing the test on error. +func mustCreateFileRow(t *testing.T, repos *Repositories, file *File) { + t.Helper() + + err := repos.Files.Create(context.Background(), nil, file) + if err != nil { + t.Fatalf("failed to create file %s: %v", file.Path, err) + } +} + +// mustAddFileToSnapshot associates a file with a snapshot, failing the +// test on error. +func mustAddFileToSnapshot( + t *testing.T, repos *Repositories, snapshotID string, fileID types.FileID, +) { + t.Helper() + + err := repos.Snapshots.AddFileByID(context.Background(), nil, snapshotID, fileID) + if err != nil { + t.Fatal(err) + } +} + +// setupTestDB creates an on-disk test database in a per-test temp +// directory and returns it along with a cleanup func that closes it. +func setupTestDB(t *testing.T) (*DB, func()) { + t.Helper() + + ctx := context.Background() + dbPath := filepath.Join(t.TempDir(), "test.db") + + db, err := New(ctx, dbPath) + if err != nil { + t.Fatalf("failed to create database: %v", err) + } + + cleanup := func() { + err := db.Close() + if err != nil { + t.Errorf("failed to close database: %v", err) + } + } + + return db, cleanup +} + +// countRow runs a single-integer COUNT-style query and returns the value. +func countRow(t *testing.T, db *DB, query string, args ...any) int { + t.Helper() + + var count int + + err := db.conn.QueryRowContext(context.Background(), query, args...).Scan(&count) + if err != nil { + t.Fatal(err) + } + + return count +} diff --git a/internal/database/helpers_test.go b/internal/database/helpers_test.go new file mode 100644 index 0000000..8a2b705 --- /dev/null +++ b/internal/database/helpers_test.go @@ -0,0 +1,52 @@ +package database_test + +import ( + "context" + "path/filepath" + "testing" + + "sneak.berlin/go/vaultik/internal/database" +) + +// Common fixture values shared by the repository tests. +const ( + testFilePath1 = "/file1.txt" + testFilePath2 = "/file2.txt" + testFileTxt = "/test/file.txt" + testTxFile = "/test/tx_file.txt" + testHostname = "test-host" + testVersion = "1.0.0" +) + +// mustCreateFile inserts the given file row, failing the test on error. +func mustCreateFile(t *testing.T, repo *database.FileRepository, file *database.File) { + t.Helper() + + err := repo.Create(context.Background(), nil, file) + if err != nil { + t.Fatalf("failed to create file %s: %v", file.Path, err) + } +} + +// setupTestDB creates an on-disk test database in a per-test temp +// directory and returns it along with a cleanup func that closes it. +func setupTestDB(t *testing.T) (*database.DB, func()) { + t.Helper() + + ctx := context.Background() + dbPath := filepath.Join(t.TempDir(), "test.db") + + db, err := database.New(ctx, dbPath) + if err != nil { + t.Fatalf("failed to create database: %v", err) + } + + cleanup := func() { + err := db.Close() + if err != nil { + t.Errorf("failed to close database: %v", err) + } + } + + return db, cleanup +} diff --git a/internal/database/local_meta.go b/internal/database/local_meta.go index 83c2048..27e23bb 100644 --- a/internal/database/local_meta.go +++ b/internal/database/local_meta.go @@ -18,6 +18,7 @@ type LocalMetaRepository struct { db *DB } +// NewLocalMetaRepository creates a LocalMetaRepository backed by db. func NewLocalMetaRepository(db *DB) *LocalMetaRepository { return &LocalMetaRepository{db: db} } diff --git a/internal/database/local_meta_test.go b/internal/database/local_meta_test.go index d9b1d64..52df419 100644 --- a/internal/database/local_meta_test.go +++ b/internal/database/local_meta_test.go @@ -9,6 +9,8 @@ import ( ) func TestLocalMetaEmptyOnFresh(t *testing.T) { + t.Parallel() + db, err := database.NewTestDB() require.NoError(t, err) @@ -22,6 +24,8 @@ func TestLocalMetaEmptyOnFresh(t *testing.T) { } func TestLocalMetaSetGetRoundTrip(t *testing.T) { + t.Parallel() + db, err := database.NewTestDB() require.NoError(t, err) @@ -30,7 +34,8 @@ func TestLocalMetaSetGetRoundTrip(t *testing.T) { repos := database.NewRepositories(db) ctx := context.Background() - require.NoError(t, repos.LocalMeta.Set(ctx, database.LocalMetaKeyStorageURL, "file:///mnt/backups")) + require.NoError(t, repos.LocalMeta.Set( + ctx, database.LocalMetaKeyStorageURL, "file:///mnt/backups")) got, err := repos.LocalMeta.Get(ctx, database.LocalMetaKeyStorageURL) require.NoError(t, err) @@ -38,6 +43,8 @@ func TestLocalMetaSetGetRoundTrip(t *testing.T) { } func TestLocalMetaSetOverwrites(t *testing.T) { + t.Parallel() + db, err := database.NewTestDB() require.NoError(t, err) @@ -46,8 +53,10 @@ func TestLocalMetaSetOverwrites(t *testing.T) { repos := database.NewRepositories(db) ctx := context.Background() - require.NoError(t, repos.LocalMeta.Set(ctx, database.LocalMetaKeyStorageURL, "s3://old")) - require.NoError(t, repos.LocalMeta.Set(ctx, database.LocalMetaKeyStorageURL, "s3://new")) + require.NoError(t, repos.LocalMeta.Set( + ctx, database.LocalMetaKeyStorageURL, "s3://old")) + require.NoError(t, repos.LocalMeta.Set( + ctx, database.LocalMetaKeyStorageURL, "s3://new")) got, err := repos.LocalMeta.Get(ctx, database.LocalMetaKeyStorageURL) require.NoError(t, err) diff --git a/internal/database/models.go b/internal/database/models.go index f95831b..69f8730 100644 --- a/internal/database/models.go +++ b/internal/database/models.go @@ -1,5 +1,3 @@ -// Package database provides data models and repository interfaces for the Vaultik backup system. -// It includes types for files, chunks, blobs, snapshots, and their relationships. package database import ( @@ -13,9 +11,12 @@ import ( // and symlink targets. This information is used to restore files with their // original attributes. type File struct { - ID types.FileID // UUID primary key - Path types.FilePath // Absolute path of the file - SourcePath types.SourcePath // The source directory this file came from (for restore path stripping) + ID types.FileID // UUID primary key + Path types.FilePath // Absolute path of the file + + // SourcePath is the source directory this file came from (used for + // restore path stripping). + SourcePath types.SourcePath MTime time.Time Size int64 Mode uint32 @@ -55,13 +56,16 @@ type Chunk struct { // The blob creation process is: chunks are accumulated -> compressed with zstd // -> encrypted with age -> hashed -> uploaded to S3 with the hash as filename. type Blob struct { - ID types.BlobID // UUID assigned when blob creation starts - Hash types.BlobHash // SHA256 of final compressed+encrypted content (empty until finalized) - CreatedTS time.Time // When blob creation started - FinishedTS *time.Time // When blob was finalized (nil if still packing) - UncompressedSize int64 // Total size of raw chunks before compression - CompressedSize int64 // Size after compression and encryption - UploadedTS *time.Time // When blob was uploaded to S3 (nil if not uploaded) + ID types.BlobID // UUID assigned when blob creation starts + + // Hash is the SHA256 of the final compressed+encrypted content + // (empty until finalized). + Hash types.BlobHash + CreatedTS time.Time // When blob creation started + FinishedTS *time.Time // When blob was finalized (nil if still packing) + UncompressedSize int64 // Total size of raw chunks before compression + CompressedSize int64 // Size after compression and encryption + UploadedTS *time.Time // When blob was uploaded to S3 (nil if not uploaded) } // BlobChunk represents the mapping between blobs and the chunks they contain. @@ -75,9 +79,10 @@ type BlobChunk struct { Length int64 } -// ChunkFile represents the reverse mapping showing which files contain a specific chunk. -// This is used during deduplication to identify all files that share a chunk, -// which is important for garbage collection and integrity verification. +// ChunkFile represents the reverse mapping showing which files contain a +// specific chunk. This is used during deduplication to identify all files +// that share a chunk, which is important for garbage collection and +// integrity verification. type ChunkFile struct { ChunkHash types.ChunkHash FileID types.FileID @@ -87,17 +92,20 @@ type ChunkFile struct { // Snapshot represents a snapshot record in the database type Snapshot struct { - ID types.SnapshotID - Hostname types.Hostname - VaultikVersion types.Version - VaultikGitRevision types.GitRevision - StartedAt time.Time - CompletedAt *time.Time // nil if still in progress - FileCount int64 - ChunkCount int64 - BlobCount int64 - TotalSize int64 // Total size of all referenced files - BlobSize int64 // Total size of all referenced blobs (compressed and encrypted) + ID types.SnapshotID + Hostname types.Hostname + VaultikVersion types.Version + VaultikGitRevision types.GitRevision + StartedAt time.Time + CompletedAt *time.Time // nil if still in progress + FileCount int64 + ChunkCount int64 + BlobCount int64 + TotalSize int64 // Total size of all referenced files + + // BlobSize is the total size of all referenced blobs (compressed and + // encrypted). + BlobSize int64 BlobUncompressedSize int64 // Total uncompressed size of all referenced blobs CompressionRatio float64 // Compression ratio (BlobSize / BlobUncompressedSize) CompressionLevel int // Compression level used for this snapshot diff --git a/internal/database/module.go b/internal/database/module.go index e4af208..ec8fa47 100644 --- a/internal/database/module.go +++ b/internal/database/module.go @@ -11,7 +11,13 @@ import ( "sneak.berlin/go/vaultik/internal/log" ) +// indexDirPerm restricts the local index directory to the owning user; +// the index describes the backed-up file tree and must stay private. +const indexDirPerm = 0o700 + // Module provides database dependencies +// +//nolint:gochecknoglobals // fx module definitions are package globals by convention var Module = fx.Module("database", fx.Provide( provideDatabase, @@ -22,7 +28,9 @@ var Module = fx.Module("database", func provideDatabase(lc fx.Lifecycle, cfg *config.Config) (*DB, error) { // Ensure the index directory exists indexDir := filepath.Dir(cfg.IndexPath) - if err := os.MkdirAll(indexDir, 0700); err != nil { + + err := os.MkdirAll(indexDir, indexDirPerm) + if err != nil { return nil, fmt.Errorf("creating index directory: %w", err) } @@ -32,7 +40,7 @@ func provideDatabase(lc fx.Lifecycle, cfg *config.Config) (*DB, error) { } lc.Append(fx.Hook{ - OnStop: func(ctx context.Context) error { + OnStop: func(_ context.Context) error { log.Debug("Database module OnStop hook called") err := db.Close() diff --git a/internal/database/repositories_test.go b/internal/database/repositories_test.go index 485ba61..f4e2199 100644 --- a/internal/database/repositories_test.go +++ b/internal/database/repositories_test.go @@ -1,4 +1,4 @@ -package database +package database_test import ( "context" @@ -7,21 +7,21 @@ import ( "testing" "time" + "sneak.berlin/go/vaultik/internal/database" "sneak.berlin/go/vaultik/internal/types" ) -func TestRepositoriesTransaction(t *testing.T) { - db, cleanup := setupTestDB(t) - defer cleanup() +// errIntentionalRollback forces a transaction rollback in tests. +var errIntentionalRollback = errors.New("intentional rollback") - ctx := context.Background() - repos := NewRepositories(db) - - // Test successful transaction with multiple operations - err := repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { - // Create a file - file := &File{ - Path: "/test/tx_file.txt", +// createTxTestData returns a transaction body that creates a file with +// two chunks packed into one blob. +func createTxTestData( + repos *database.Repositories, +) func(context.Context, *sql.Tx) error { + return func(ctx context.Context, tx *sql.Tx) error { + file := &database.File{ + Path: testTxFile, MTime: time.Now().Truncate(time.Second), Size: 1024, Mode: 0644, @@ -34,95 +34,116 @@ func TestRepositoriesTransaction(t *testing.T) { return err } - // Create chunks - chunk1 := &Chunk{ - ChunkHash: types.ChunkHash("tx_chunk1"), - Size: 512, - } - - err = repos.Chunks.Create(ctx, tx, chunk1) + err = createTxFileChunks(ctx, tx, repos, file.ID) if err != nil { return err } - chunk2 := &Chunk{ - ChunkHash: types.ChunkHash("tx_chunk2"), - Size: 512, - } + return createTxBlob(ctx, tx, repos) + } +} - err = repos.Chunks.Create(ctx, tx, chunk2) - if err != nil { - return err - } +// createTxFileChunks creates the two test chunks and maps them to the file. +func createTxFileChunks( + ctx context.Context, tx *sql.Tx, + repos *database.Repositories, fileID types.FileID, +) error { + // Create chunks + chunk1 := &database.Chunk{ + ChunkHash: types.ChunkHash("tx_chunk1"), + Size: 512, + } - // Map chunks to file - fc1 := &FileChunk{ - FileID: file.ID, - Idx: 0, - ChunkHash: chunk1.ChunkHash, - } + err := repos.Chunks.Create(ctx, tx, chunk1) + if err != nil { + return err + } - err = repos.FileChunks.Create(ctx, tx, fc1) - if err != nil { - return err - } + chunk2 := &database.Chunk{ + ChunkHash: types.ChunkHash("tx_chunk2"), + Size: 512, + } - fc2 := &FileChunk{ - FileID: file.ID, - Idx: 1, - ChunkHash: chunk2.ChunkHash, - } + err = repos.Chunks.Create(ctx, tx, chunk2) + if err != nil { + return err + } - err = repos.FileChunks.Create(ctx, tx, fc2) - if err != nil { - return err - } + // Map chunks to file + fc1 := &database.FileChunk{ + FileID: fileID, + Idx: 0, + ChunkHash: chunk1.ChunkHash, + } - // Create blob - blob := &Blob{ - ID: types.NewBlobID(), - Hash: types.BlobHash("tx_blob1"), - CreatedTS: time.Now().Truncate(time.Second), - } + err = repos.FileChunks.Create(ctx, tx, fc1) + if err != nil { + return err + } - err = repos.Blobs.Create(ctx, tx, blob) - if err != nil { - return err - } + fc2 := &database.FileChunk{ + FileID: fileID, + Idx: 1, + ChunkHash: chunk2.ChunkHash, + } - // Map chunks to blob - bc1 := &BlobChunk{ - BlobID: blob.ID, - ChunkHash: chunk1.ChunkHash, - Offset: 0, - Length: 512, - } + return repos.FileChunks.Create(ctx, tx, fc2) +} - err = repos.BlobChunks.Create(ctx, tx, bc1) - if err != nil { - return err - } +// createTxBlob creates the test blob and maps both chunks into it. +func createTxBlob( + ctx context.Context, tx *sql.Tx, repos *database.Repositories, +) error { + blob := &database.Blob{ + ID: types.NewBlobID(), + Hash: types.BlobHash("tx_blob1"), + CreatedTS: time.Now().Truncate(time.Second), + } - bc2 := &BlobChunk{ - BlobID: blob.ID, - ChunkHash: chunk2.ChunkHash, - Offset: 512, - Length: 512, - } + err := repos.Blobs.Create(ctx, tx, blob) + if err != nil { + return err + } - err = repos.BlobChunks.Create(ctx, tx, bc2) - if err != nil { - return err - } + // Map chunks to blob + bc1 := &database.BlobChunk{ + BlobID: blob.ID, + ChunkHash: types.ChunkHash("tx_chunk1"), + Offset: 0, + Length: 512, + } - return nil - }) + err = repos.BlobChunks.Create(ctx, tx, bc1) + if err != nil { + return err + } + + bc2 := &database.BlobChunk{ + BlobID: blob.ID, + ChunkHash: types.ChunkHash("tx_chunk2"), + Offset: 512, + Length: 512, + } + + return repos.BlobChunks.Create(ctx, tx, bc2) +} + +func TestRepositoriesTransaction(t *testing.T) { + t.Parallel() + + db, cleanup := setupTestDB(t) + defer cleanup() + + ctx := context.Background() + repos := database.NewRepositories(db) + + err := repos.WithTx(ctx, createTxTestData(repos)) if err != nil { t.Fatalf("transaction failed: %v", err) } // Verify all data was committed - file, err := repos.Files.GetByPath(ctx, "/test/tx_file.txt") + file, err := repos.Files.GetByPath(ctx, testTxFile) if err != nil { t.Fatalf("failed to get file: %v", err) } @@ -131,7 +152,7 @@ func TestRepositoriesTransaction(t *testing.T) { t.Error("expected file after transaction") } - chunks, err := repos.FileChunks.GetByFile(ctx, "/test/tx_file.txt") + chunks, err := repos.FileChunks.GetByFile(ctx, testTxFile) if err != nil { t.Fatalf("failed to get file chunks: %v", err) } @@ -151,16 +172,18 @@ func TestRepositoriesTransaction(t *testing.T) { } func TestRepositoriesTransactionRollback(t *testing.T) { + t.Parallel() + db, cleanup := setupTestDB(t) defer cleanup() ctx := context.Background() - repos := NewRepositories(db) + repos := database.NewRepositories(db) // Test transaction rollback err := repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { // Create a file - file := &File{ + file := &database.File{ Path: "/test/rollback_file.txt", MTime: time.Now().Truncate(time.Second), Size: 1024, @@ -175,7 +198,7 @@ func TestRepositoriesTransactionRollback(t *testing.T) { } // Create a chunk - chunk := &Chunk{ + chunk := &database.Chunk{ ChunkHash: types.ChunkHash("rollback_chunk"), Size: 1024, } @@ -186,10 +209,9 @@ func TestRepositoriesTransactionRollback(t *testing.T) { } // Return error to trigger rollback - return errors.New("intentional rollback") + return errIntentionalRollback }) - - if err == nil || err.Error() != "intentional rollback" { + if !errors.Is(err, errIntentionalRollback) { t.Fatalf("expected rollback error, got: %v", err) } @@ -214,14 +236,16 @@ func TestRepositoriesTransactionRollback(t *testing.T) { } func TestRepositoriesReadTransaction(t *testing.T) { + t.Parallel() + db, cleanup := setupTestDB(t) defer cleanup() ctx := context.Background() - repos := NewRepositories(db) + repos := database.NewRepositories(db) // First, create some data - file := &File{ + file := &database.File{ Path: "/test/read_file.txt", MTime: time.Now().Truncate(time.Second), Size: 1024, @@ -236,7 +260,7 @@ func TestRepositoriesReadTransaction(t *testing.T) { } // Test read-only transaction - var retrievedFile *File + var retrievedFile *database.File err = repos.WithReadTx(ctx, func(ctx context.Context, tx *sql.Tx) error { var err error @@ -247,7 +271,7 @@ func TestRepositoriesReadTransaction(t *testing.T) { } // Try to write in read-only transaction (should fail) - _ = repos.Files.Create(ctx, tx, &File{ + _ = repos.Files.Create(ctx, tx, &database.File{ Path: "/test/should_fail.txt", MTime: time.Now(), Size: 0, diff --git a/internal/database/repository_comprehensive_test.go b/internal/database/repository_comprehensive_test.go index f3fce80..4799458 100644 --- a/internal/database/repository_comprehensive_test.go +++ b/internal/database/repository_comprehensive_test.go @@ -1,3 +1,4 @@ +//nolint:testpackage // inspects the unexported database connection package database import ( @@ -11,8 +12,13 @@ import ( "sneak.berlin/go/vaultik/internal/types" ) +// errTxIntentionalRollback forces a transaction rollback in tests. +var errTxIntentionalRollback = errors.New("intentional rollback") + // TestFileRepositoryUUIDGeneration tests that files get unique UUIDs func TestFileRepositoryUUIDGeneration(t *testing.T) { + t.Parallel() + db, cleanup := setupTestDB(t) defer cleanup() @@ -22,7 +28,7 @@ func TestFileRepositoryUUIDGeneration(t *testing.T) { // Create multiple files files := []*File{ { - Path: "/file1.txt", + Path: internalTestFile1, MTime: time.Now().Truncate(time.Second), Size: 1024, Mode: 0644, @@ -30,7 +36,7 @@ func TestFileRepositoryUUIDGeneration(t *testing.T) { GID: 1000, }, { - Path: "/file2.txt", + Path: internalTestFile2, MTime: time.Now().Truncate(time.Second), Size: 2048, Mode: 0644, @@ -63,6 +69,8 @@ func TestFileRepositoryUUIDGeneration(t *testing.T) { // TestFileRepositoryGetByID tests retrieving files by UUID func TestFileRepositoryGetByID(t *testing.T) { + t.Parallel() + db, cleanup := setupTestDB(t) defer cleanup() @@ -71,7 +79,7 @@ func TestFileRepositoryGetByID(t *testing.T) { // Create a file file := &File{ - Path: "/test.txt", + Path: internalTestFilePath, MTime: time.Now().Truncate(time.Second), Size: 1024, Mode: 0644, @@ -98,8 +106,9 @@ func TestFileRepositoryGetByID(t *testing.T) { t.Errorf("Path mismatch: expected %s, got %s", file.Path, retrieved.Path) } - // Test non-existent ID - nonExistentID := types.NewFileID() // Generate a new UUID that won't exist in the database + // Test non-existent ID: generate a new UUID that won't exist in the + // database. + nonExistentID := types.NewFileID() nonExistent, err := repo.GetByID(ctx, nonExistentID) if err != nil { @@ -113,6 +122,8 @@ func TestFileRepositoryGetByID(t *testing.T) { // TestOrphanedFileCleanup tests the cleanup of orphaned files func TestOrphanedFileCleanup(t *testing.T) { + t.Parallel() + db, cleanup := setupTestDB(t) defer cleanup() @@ -149,8 +160,8 @@ func TestOrphanedFileCleanup(t *testing.T) { // Create a snapshot and reference only file2 snapshot := &Snapshot{ - ID: "test-snapshot", - Hostname: "test-host", + ID: internalTestSnapshotID, + Hostname: internalTestHost, StartedAt: time.Now(), } @@ -160,10 +171,7 @@ func TestOrphanedFileCleanup(t *testing.T) { } // Add file2 to snapshot - err = repos.Snapshots.AddFileByID(ctx, nil, snapshot.ID.String(), file2.ID) - if err != nil { - t.Fatalf("failed to add file to snapshot: %v", err) - } + mustAddFileToSnapshot(t, repos, snapshot.ID.String(), file2.ID) // Run orphaned cleanup err = repos.Files.DeleteOrphaned(ctx) @@ -194,6 +202,8 @@ func TestOrphanedFileCleanup(t *testing.T) { // TestOrphanedChunkCleanup tests the cleanup of orphaned chunks func TestOrphanedChunkCleanup(t *testing.T) { + t.Parallel() + db, cleanup := setupTestDB(t) defer cleanup() @@ -222,7 +232,7 @@ func TestOrphanedChunkCleanup(t *testing.T) { // Create a file and reference only chunk2 file := &File{ - Path: "/test.txt", + Path: internalTestFilePath, MTime: time.Now().Truncate(time.Second), Size: 1024, Mode: 0644, @@ -276,6 +286,8 @@ func TestOrphanedChunkCleanup(t *testing.T) { // TestOrphanedBlobCleanup tests the cleanup of orphaned blobs func TestOrphanedBlobCleanup(t *testing.T) { + t.Parallel() + db, cleanup := setupTestDB(t) defer cleanup() @@ -306,8 +318,8 @@ func TestOrphanedBlobCleanup(t *testing.T) { // Create a snapshot and reference only blob2 snapshot := &Snapshot{ - ID: "test-snapshot", - Hostname: "test-host", + ID: internalTestSnapshotID, + Hostname: internalTestHost, StartedAt: time.Now(), } @@ -351,6 +363,8 @@ func TestOrphanedBlobCleanup(t *testing.T) { // TestFileChunkRepositoryWithUUIDs tests file-chunk relationships with UUIDs func TestFileChunkRepositoryWithUUIDs(t *testing.T) { + t.Parallel() + db, cleanup := setupTestDB(t) defer cleanup() @@ -359,7 +373,7 @@ func TestFileChunkRepositoryWithUUIDs(t *testing.T) { // Create a file file := &File{ - Path: "/test.txt", + Path: internalTestFilePath, MTime: time.Now().Truncate(time.Second), Size: 3072, Mode: 0644, @@ -367,10 +381,7 @@ func TestFileChunkRepositoryWithUUIDs(t *testing.T) { GID: 1000, } - err := repos.Files.Create(ctx, nil, file) - if err != nil { - t.Fatalf("failed to create file: %v", err) - } + mustCreateFileRow(t, repos, file) // Create chunks chunks := []types.ChunkHash{"chunk1", "chunk2", "chunk3"} @@ -380,7 +391,7 @@ func TestFileChunkRepositoryWithUUIDs(t *testing.T) { Size: 1024, } - err = repos.Chunks.Create(ctx, nil, chunk) + err := repos.Chunks.Create(ctx, nil, chunk) if err != nil { t.Fatalf("failed to create chunk: %v", err) } @@ -426,6 +437,8 @@ func TestFileChunkRepositoryWithUUIDs(t *testing.T) { // TestChunkFileRepositoryWithUUIDs tests chunk-file relationships with UUIDs func TestChunkFileRepositoryWithUUIDs(t *testing.T) { + t.Parallel() + db, cleanup := setupTestDB(t) defer cleanup() @@ -434,7 +447,7 @@ func TestChunkFileRepositoryWithUUIDs(t *testing.T) { // Create files file1 := &File{ - Path: "/file1.txt", + Path: internalTestFile1, MTime: time.Now().Truncate(time.Second), Size: 1024, Mode: 0644, @@ -442,7 +455,7 @@ func TestChunkFileRepositoryWithUUIDs(t *testing.T) { GID: 1000, } file2 := &File{ - Path: "/file2.txt", + Path: internalTestFile2, MTime: time.Now().Truncate(time.Second), Size: 1024, Mode: 0644, @@ -450,15 +463,8 @@ func TestChunkFileRepositoryWithUUIDs(t *testing.T) { GID: 1000, } - err := repos.Files.Create(ctx, nil, file1) - if err != nil { - t.Fatalf("failed to create file1: %v", err) - } - - err = repos.Files.Create(ctx, nil, file2) - if err != nil { - t.Fatalf("failed to create file2: %v", err) - } + mustCreateFileRow(t, repos, file1) + mustCreateFileRow(t, repos, file2) // Create a chunk that appears in both files (deduplication) chunk := &Chunk{ @@ -466,7 +472,7 @@ func TestChunkFileRepositoryWithUUIDs(t *testing.T) { Size: 1024, } - err = repos.Chunks.Create(ctx, nil, chunk) + err := repos.Chunks.Create(ctx, nil, chunk) if err != nil { t.Fatalf("failed to create chunk: %v", err) } @@ -518,6 +524,8 @@ func TestChunkFileRepositoryWithUUIDs(t *testing.T) { // TestSnapshotRepositoryExtendedFields tests snapshot with version and git revision func TestSnapshotRepositoryExtendedFields(t *testing.T) { + t.Parallel() + db, cleanup := setupTestDB(t) defer cleanup() @@ -527,7 +535,7 @@ func TestSnapshotRepositoryExtendedFields(t *testing.T) { // Create snapshot with extended fields snapshot := &Snapshot{ ID: "test-20250722-120000Z", - Hostname: "test-host", + Hostname: internalTestHost, VaultikVersion: "0.0.1", VaultikGitRevision: "abc123def456", StartedAt: time.Now(), @@ -555,35 +563,39 @@ func TestSnapshotRepositoryExtendedFields(t *testing.T) { } if retrieved.VaultikVersion != snapshot.VaultikVersion { - t.Errorf("version mismatch: expected %s, got %s", snapshot.VaultikVersion, retrieved.VaultikVersion) + t.Errorf("version mismatch: expected %s, got %s", + snapshot.VaultikVersion, retrieved.VaultikVersion) } if retrieved.VaultikGitRevision != snapshot.VaultikGitRevision { - t.Errorf("git revision mismatch: expected %s, got %s", snapshot.VaultikGitRevision, retrieved.VaultikGitRevision) + t.Errorf("git revision mismatch: expected %s, got %s", + snapshot.VaultikGitRevision, retrieved.VaultikGitRevision) } if retrieved.CompressionLevel != snapshot.CompressionLevel { - t.Errorf("compression level mismatch: expected %d, got %d", snapshot.CompressionLevel, retrieved.CompressionLevel) + t.Errorf("compression level mismatch: expected %d, got %d", + snapshot.CompressionLevel, retrieved.CompressionLevel) } if retrieved.BlobUncompressedSize != snapshot.BlobUncompressedSize { - t.Errorf("uncompressed size mismatch: expected %d, got %d", snapshot.BlobUncompressedSize, retrieved.BlobUncompressedSize) + t.Errorf("uncompressed size mismatch: expected %d, got %d", + snapshot.BlobUncompressedSize, retrieved.BlobUncompressedSize) } if retrieved.UploadDurationMs != snapshot.UploadDurationMs { - t.Errorf("upload duration mismatch: expected %d, got %d", snapshot.UploadDurationMs, retrieved.UploadDurationMs) + t.Errorf("upload duration mismatch: expected %d, got %d", + snapshot.UploadDurationMs, retrieved.UploadDurationMs) } } // TestComplexOrphanedDataScenario tests a complex scenario with multiple relationships -func TestComplexOrphanedDataScenario(t *testing.T) { - db, cleanup := setupTestDB(t) - defer cleanup() +// createOrphanScenarioFixtures creates two snapshots and three files for +// the orphaned-data cleanup scenario. +func createOrphanScenarioFixtures( + ctx context.Context, t *testing.T, repos *Repositories, +) (*Snapshot, *Snapshot, []*File) { + t.Helper() - ctx := context.Background() - repos := NewRepositories(db) - - // Create snapshots snapshot1 := &Snapshot{ ID: "snapshot1", Hostname: "host1", @@ -623,34 +635,33 @@ func TestComplexOrphanedDataScenario(t *testing.T) { } } + return snapshot1, snapshot2, files +} + +func TestComplexOrphanedDataScenario(t *testing.T) { + t.Parallel() + + db, cleanup := setupTestDB(t) + defer cleanup() + + ctx := context.Background() + repos := NewRepositories(db) + + snapshot1, snapshot2, files := createOrphanScenarioFixtures(ctx, t, repos) + // Add files to snapshots // Snapshot1: file0, file1 // Snapshot2: file1, file2 // file0: only in snapshot1 // file1: in both snapshots // file2: only in snapshot2 - err = repos.Snapshots.AddFileByID(ctx, nil, snapshot1.ID.String(), files[0].ID) - if err != nil { - t.Fatal(err) - } - - err = repos.Snapshots.AddFileByID(ctx, nil, snapshot1.ID.String(), files[1].ID) - if err != nil { - t.Fatal(err) - } - - err = repos.Snapshots.AddFileByID(ctx, nil, snapshot2.ID.String(), files[1].ID) - if err != nil { - t.Fatal(err) - } - - err = repos.Snapshots.AddFileByID(ctx, nil, snapshot2.ID.String(), files[2].ID) - if err != nil { - t.Fatal(err) - } + mustAddFileToSnapshot(t, repos, snapshot1.ID.String(), files[0].ID) + mustAddFileToSnapshot(t, repos, snapshot1.ID.String(), files[1].ID) + mustAddFileToSnapshot(t, repos, snapshot2.ID.String(), files[1].ID) + mustAddFileToSnapshot(t, repos, snapshot2.ID.String(), files[2].ID) // Delete snapshot1 - err = repos.Snapshots.DeleteSnapshotFiles(ctx, snapshot1.ID.String()) + err := repos.Snapshots.DeleteSnapshotFiles(ctx, snapshot1.ID.String()) if err != nil { t.Fatal(err) } @@ -700,6 +711,8 @@ func TestComplexOrphanedDataScenario(t *testing.T) { // TestCascadeDelete tests that cascade deletes work properly func TestCascadeDelete(t *testing.T) { + t.Parallel() + db, cleanup := setupTestDB(t) defer cleanup() @@ -774,6 +787,8 @@ func TestCascadeDelete(t *testing.T) { // TestTransactionIsolation tests that transactions properly isolate changes func TestTransactionIsolation(t *testing.T) { + t.Parallel() + db, cleanup := setupTestDB(t) defer cleanup() @@ -802,7 +817,7 @@ func TestTransactionIsolation(t *testing.T) { // For now, we'll just test that rollback works // Return an error to trigger rollback - return errors.New("intentional rollback") + return errTxIntentionalRollback }) if err == nil { t.Fatal("expected error from transaction") @@ -819,32 +834,15 @@ func TestTransactionIsolation(t *testing.T) { } } -// TestConcurrentOrphanedCleanup tests that concurrent cleanup operations don't interfere -func TestConcurrentOrphanedCleanup(t *testing.T) { - db, cleanup := setupTestDB(t) - defer cleanup() +// TestConcurrentOrphanedCleanup tests that concurrent cleanup operations +// don't interfere. +// createConcurrentCleanupFiles creates 20 files and associates the +// even-numbered ones with the snapshot, leaving the rest orphaned. +func createConcurrentCleanupFiles( + ctx context.Context, t *testing.T, repos *Repositories, snapshotID string, +) { + t.Helper() - ctx := context.Background() - repos := NewRepositories(db) - - // Set a 5-second busy timeout to handle concurrent operations - if _, err := db.conn.Exec("PRAGMA busy_timeout = 5000"); err != nil { - t.Fatalf("failed to set busy timeout: %v", err) - } - - // Create a snapshot - snapshot := &Snapshot{ - ID: "concurrent-test", - Hostname: "test-host", - StartedAt: time.Now(), - } - - err := repos.Snapshots.Create(ctx, nil, snapshot) - if err != nil { - t.Fatal(err) - } - - // Create many files, some orphaned for i := range 20 { file := &File{ Path: types.FilePath(fmt.Sprintf("/concurrent-%d.txt", i)), @@ -855,19 +853,49 @@ func TestConcurrentOrphanedCleanup(t *testing.T) { GID: 1000, } - err = repos.Files.Create(ctx, nil, file) + err := repos.Files.Create(ctx, nil, file) if err != nil { t.Fatal(err) } // Add even-numbered files to snapshot if i%2 == 0 { - err = repos.Snapshots.AddFileByID(ctx, nil, snapshot.ID.String(), file.ID) + err = repos.Snapshots.AddFileByID(ctx, nil, snapshotID, file.ID) if err != nil { t.Fatal(err) } } } +} + +func TestConcurrentOrphanedCleanup(t *testing.T) { + t.Parallel() + + db, cleanup := setupTestDB(t) + defer cleanup() + + ctx := context.Background() + repos := NewRepositories(db) + + // Set a 5-second busy timeout to handle concurrent operations + _, err := db.conn.ExecContext(ctx, "PRAGMA busy_timeout = 5000") + if err != nil { + t.Fatalf("failed to set busy timeout: %v", err) + } + + // Create a snapshot + snapshot := &Snapshot{ + ID: "concurrent-test", + Hostname: internalTestHost, + StartedAt: time.Now(), + } + + err = repos.Snapshots.Create(ctx, nil, snapshot) + if err != nil { + t.Fatal(err) + } + + createConcurrentCleanupFiles(ctx, t, repos, snapshot.ID.String()) // Run multiple cleanup operations concurrently // Note: SQLite has limited support for concurrent writes, so we expect some to fail diff --git a/internal/database/repository_debug_test.go b/internal/database/repository_debug_test.go index 9deedaa..db541ed 100644 --- a/internal/database/repository_debug_test.go +++ b/internal/database/repository_debug_test.go @@ -1,3 +1,4 @@ +//nolint:testpackage // inspects the unexported database connection package database import ( @@ -6,15 +7,50 @@ import ( "time" ) -// TestOrphanedFileCleanupDebug tests orphaned file cleanup with debug output -func TestOrphanedFileCleanupDebug(t *testing.T) { - db, cleanup := setupTestDB(t) - defer cleanup() +// logSnapshotFileIDs logs every file_id present in snapshot_files. +func logSnapshotFileIDs(t *testing.T, db *DB) { + t.Helper() ctx := context.Background() - repos := NewRepositories(db) - // Create files + rows, err := db.conn.QueryContext(ctx, "SELECT file_id FROM snapshot_files") + if err != nil { + t.Fatal(err) + } + defer func() { + err := rows.Close() + if err != nil { + t.Logf("failed to close rows: %v", err) + } + }() + + t.Log("Files in snapshot_files:") + + for rows.Next() { + var fileID string + + err := rows.Scan(&fileID) + if err != nil { + t.Fatal(err) + } + + t.Logf(" - %s", fileID) + } + + err = rows.Err() + if err != nil { + t.Fatal(err) + } +} + +// TestOrphanedFileCleanupDebug tests orphaned file cleanup with debug output +// createOrphanDebugFixtures creates one orphaned file, one referenced +// file, and the snapshot that will reference the latter. +func createOrphanDebugFixtures( + ctx context.Context, t *testing.T, repos *Repositories, +) (*File, *File, *Snapshot) { + t.Helper() + file1 := &File{ Path: "/orphaned.txt", MTime: time.Now().Truncate(time.Second), @@ -48,8 +84,8 @@ func TestOrphanedFileCleanupDebug(t *testing.T) { // Create a snapshot and reference only file2 snapshot := &Snapshot{ - ID: "test-snapshot", - Hostname: "test-host", + ID: internalTestSnapshotID, + Hostname: internalTestHost, StartedAt: time.Now(), } @@ -60,18 +96,26 @@ func TestOrphanedFileCleanupDebug(t *testing.T) { t.Logf("Created snapshot: %s", snapshot.ID) + return file1, file2, snapshot +} + +func TestOrphanedFileCleanupDebug(t *testing.T) { + t.Parallel() + + db, cleanup := setupTestDB(t) + defer cleanup() + + ctx := context.Background() + repos := NewRepositories(db) + + file1, file2, snapshot := createOrphanDebugFixtures(ctx, t, repos) + // Check snapshot_files before adding - var count int - - err = db.conn.QueryRow("SELECT COUNT(*) FROM snapshot_files").Scan(&count) - if err != nil { - t.Fatal(err) - } - + count := countRow(t, db, "SELECT COUNT(*) FROM snapshot_files") t.Logf("snapshot_files count before add: %d", count) // Add file2 to snapshot - err = repos.Snapshots.AddFileByID(ctx, nil, snapshot.ID.String(), file2.ID) + err := repos.Snapshots.AddFileByID(ctx, nil, snapshot.ID.String(), file2.ID) if err != nil { t.Fatalf("failed to add file to snapshot: %v", err) } @@ -79,44 +123,14 @@ func TestOrphanedFileCleanupDebug(t *testing.T) { t.Logf("Added file2 to snapshot") // Check snapshot_files after adding - err = db.conn.QueryRow("SELECT COUNT(*) FROM snapshot_files").Scan(&count) - if err != nil { - t.Fatal(err) - } - + count = countRow(t, db, "SELECT COUNT(*) FROM snapshot_files") t.Logf("snapshot_files count after add: %d", count) // Check which files are referenced - rows, err := db.conn.Query("SELECT file_id FROM snapshot_files") - if err != nil { - t.Fatal(err) - } - defer func() { - err := rows.Close() - if err != nil { - t.Logf("failed to close rows: %v", err) - } - }() - - t.Log("Files in snapshot_files:") - - for rows.Next() { - var fileID string - - err := rows.Scan(&fileID) - if err != nil { - t.Fatal(err) - } - - t.Logf(" - %s", fileID) - } + logSnapshotFileIDs(t, db) // Check files before cleanup - err = db.conn.QueryRow("SELECT COUNT(*) FROM files").Scan(&count) - if err != nil { - t.Fatal(err) - } - + count = countRow(t, db, countFilesQuery) t.Logf("Files count before cleanup: %d", count) // Run orphaned cleanup @@ -128,11 +142,7 @@ func TestOrphanedFileCleanupDebug(t *testing.T) { t.Log("Ran orphaned cleanup") // Check files after cleanup - err = db.conn.QueryRow("SELECT COUNT(*) FROM files").Scan(&count) - if err != nil { - t.Fatal(err) - } - + count = countRow(t, db, countFilesQuery) t.Logf("Files count after cleanup: %d", count) // List remaining files @@ -156,18 +166,12 @@ func TestOrphanedFileCleanupDebug(t *testing.T) { if orphanedFile != nil { t.Error("orphaned file should have been deleted") // Let's check why it wasn't deleted - var exists bool - - err = db.conn.QueryRow(` + stillReferenced := countRow(t, db, ` SELECT EXISTS( - SELECT 1 FROM snapshot_files + SELECT 1 FROM snapshot_files WHERE file_id = ? - )`, file1.ID).Scan(&exists) - if err != nil { - t.Fatal(err) - } - - t.Logf("File1 exists in snapshot_files: %v", exists) + )`, file1.ID) + t.Logf("File1 exists in snapshot_files: %v", stillReferenced != 0) } else { t.Log("Orphaned file was correctly deleted") } diff --git a/internal/database/repository_edge_cases_test.go b/internal/database/repository_edge_cases_test.go index 666f33f..3e8cfb2 100644 --- a/internal/database/repository_edge_cases_test.go +++ b/internal/database/repository_edge_cases_test.go @@ -1,3 +1,4 @@ +//nolint:testpackage // inspects the unexported database connection package database import ( @@ -10,20 +11,17 @@ import ( "sneak.berlin/go/vaultik/internal/types" ) -// TestFileRepositoryEdgeCases tests edge cases for file repository -func TestFileRepositoryEdgeCases(t *testing.T) { - db, cleanup := setupTestDB(t) - defer cleanup() +// fileEdgeCase describes one Create edge-case scenario. +type fileEdgeCase struct { + name string + file *File + wantErr bool + errMsg string +} - ctx := context.Background() - repo := NewFileRepository(db) - - tests := []struct { - name string - file *File - wantErr bool - errMsg string - }{ +// fileEdgeCases returns the Create edge-case table. +func fileEdgeCases() []fileEdgeCase { + return []fileEdgeCase{ { name: "empty path", file: &File{ @@ -51,6 +49,7 @@ func TestFileRepositoryEdgeCases(t *testing.T) { { name: "path with special characters", file: &File{ + //nolint:gosmopolitan // non-ASCII path is deliberate test data Path: "/test/file with spaces and 特殊文字.txt", MTime: time.Now(), Size: 1024, @@ -86,12 +85,26 @@ func TestFileRepositoryEdgeCases(t *testing.T) { wantErr: false, }, } +} - for i, tt := range tests { +// TestFileRepositoryEdgeCases tests edge cases for file repository +func TestFileRepositoryEdgeCases(t *testing.T) { + t.Parallel() + + db, cleanup := setupTestDB(t) + t.Cleanup(cleanup) + + ctx := context.Background() + repo := NewFileRepository(db) + + for i, tt := range fileEdgeCases() { t.Run(tt.name, func(t *testing.T) { + t.Parallel() + // Add a unique suffix to paths to avoid UNIQUE constraint violations if tt.file.Path != "" { - tt.file.Path = types.FilePath(fmt.Sprintf("%s_%d_%d", tt.file.Path, i, time.Now().UnixNano())) + tt.file.Path = types.FilePath(fmt.Sprintf("%s_%d_%d", + tt.file.Path, i, time.Now().UnixNano())) } err := repo.Create(ctx, nil, tt.file) @@ -106,65 +119,128 @@ func TestFileRepositoryEdgeCases(t *testing.T) { } } +// testDuplicateFilePaths exercises the UPSERT behavior for duplicate paths. +func testDuplicateFilePaths(t *testing.T, repos *Repositories) { + t.Helper() + + ctx := context.Background() + + file1 := &File{ + Path: "/duplicate.txt", + MTime: time.Now(), + Size: 1024, + Mode: 0644, + UID: 1000, + GID: 1000, + } + file2 := &File{ + Path: "/duplicate.txt", // Same path + MTime: time.Now().Add(time.Hour), + Size: 2048, + Mode: 0644, + UID: 1000, + GID: 1000, + } + + err := repos.Files.Create(ctx, nil, file1) + if err != nil { + t.Fatalf("failed to create file1: %v", err) + } + + originalID := file1.ID + + // Create with same path should update the existing record (UPSERT behavior) + err = repos.Files.Create(ctx, nil, file2) + if err != nil { + t.Fatalf("failed to create file2: %v", err) + } + + // Verify the file was updated, not duplicated + retrievedFile, err := repos.Files.GetByPath(ctx, "/duplicate.txt") + if err != nil { + t.Fatalf("failed to retrieve file: %v", err) + } + + // The file should have been updated with file2's data + if retrievedFile.Size != 2048 { + t.Errorf("expected size 2048, got %d", retrievedFile.Size) + } + + // ID might be different due to the UPSERT + if retrievedFile.ID != file2.ID { + t.Logf("File ID changed from %s to %s during upsert", + originalID, retrievedFile.ID) + } +} + +// testDuplicateFileChunks exercises idempotent file-chunk mapping creation. +func testDuplicateFileChunks(t *testing.T, repos *Repositories) { + t.Helper() + + ctx := context.Background() + + file := &File{ + Path: "/test-dup-fc.txt", + MTime: time.Now(), + Size: 1024, + Mode: 0644, + UID: 1000, + GID: 1000, + } + + err := repos.Files.Create(ctx, nil, file) + if err != nil { + t.Fatal(err) + } + + chunk := &Chunk{ + ChunkHash: types.ChunkHash("test-chunk-dup"), + Size: 1024, + } + + err = repos.Chunks.Create(ctx, nil, chunk) + if err != nil { + t.Fatal(err) + } + + fc := &FileChunk{ + FileID: file.ID, + Idx: 0, + ChunkHash: chunk.ChunkHash, + } + + err = repos.FileChunks.Create(ctx, nil, fc) + if err != nil { + t.Fatal(err) + } + + // Creating the same mapping again should be idempotent + err = repos.FileChunks.Create(ctx, nil, fc) + if err != nil { + t.Error("file-chunk creation should be idempotent") + } +} + // TestDuplicateHandling tests handling of duplicate entries func TestDuplicateHandling(t *testing.T) { + t.Parallel() + db, cleanup := setupTestDB(t) - defer cleanup() + t.Cleanup(cleanup) ctx := context.Background() repos := NewRepositories(db) // Test duplicate file paths - Create uses UPSERT logic t.Run("duplicate file paths", func(t *testing.T) { - file1 := &File{ - Path: "/duplicate.txt", - MTime: time.Now(), - Size: 1024, - Mode: 0644, - UID: 1000, - GID: 1000, - } - file2 := &File{ - Path: "/duplicate.txt", // Same path - MTime: time.Now().Add(time.Hour), - Size: 2048, - Mode: 0644, - UID: 1000, - GID: 1000, - } - - err := repos.Files.Create(ctx, nil, file1) - if err != nil { - t.Fatalf("failed to create file1: %v", err) - } - - originalID := file1.ID - - // Create with same path should update the existing record (UPSERT behavior) - err = repos.Files.Create(ctx, nil, file2) - if err != nil { - t.Fatalf("failed to create file2: %v", err) - } - - // Verify the file was updated, not duplicated - retrievedFile, err := repos.Files.GetByPath(ctx, "/duplicate.txt") - if err != nil { - t.Fatalf("failed to retrieve file: %v", err) - } - - // The file should have been updated with file2's data - if retrievedFile.Size != 2048 { - t.Errorf("expected size 2048, got %d", retrievedFile.Size) - } - - // ID might be different due to the UPSERT - if retrievedFile.ID != file2.ID { - t.Logf("File ID changed from %s to %s during upsert", originalID, retrievedFile.ID) - } + t.Parallel() + testDuplicateFilePaths(t, repos) }) // Test duplicate chunk hashes t.Run("duplicate chunk hashes", func(t *testing.T) { + t.Parallel() + chunk := &Chunk{ ChunkHash: types.ChunkHash("duplicate-chunk"), Size: 1024, @@ -184,59 +260,25 @@ func TestDuplicateHandling(t *testing.T) { // Test duplicate file-chunk mappings t.Run("duplicate file-chunk mappings", func(t *testing.T) { - file := &File{ - Path: "/test-dup-fc.txt", - MTime: time.Now(), - Size: 1024, - Mode: 0644, - UID: 1000, - GID: 1000, - } - - err := repos.Files.Create(ctx, nil, file) - if err != nil { - t.Fatal(err) - } - - chunk := &Chunk{ - ChunkHash: types.ChunkHash("test-chunk-dup"), - Size: 1024, - } - - err = repos.Chunks.Create(ctx, nil, chunk) - if err != nil { - t.Fatal(err) - } - - fc := &FileChunk{ - FileID: file.ID, - Idx: 0, - ChunkHash: chunk.ChunkHash, - } - - err = repos.FileChunks.Create(ctx, nil, fc) - if err != nil { - t.Fatal(err) - } - - // Creating the same mapping again should be idempotent - err = repos.FileChunks.Create(ctx, nil, fc) - if err != nil { - t.Error("file-chunk creation should be idempotent") - } + t.Parallel() + testDuplicateFileChunks(t, repos) }) } // TestNullHandling tests handling of NULL values func TestNullHandling(t *testing.T) { + t.Parallel() + db, cleanup := setupTestDB(t) - defer cleanup() + t.Cleanup(cleanup) ctx := context.Background() repos := NewRepositories(db) // Test file with no link target t.Run("file without link target", func(t *testing.T) { + t.Parallel() + file := &File{ Path: "/regular.txt", MTime: time.Now(), @@ -264,9 +306,11 @@ func TestNullHandling(t *testing.T) { // Test snapshot with NULL completed_at t.Run("incomplete snapshot", func(t *testing.T) { + t.Parallel() + snapshot := &Snapshot{ ID: "incomplete-test", - Hostname: "test-host", + Hostname: internalTestHost, StartedAt: time.Now(), CompletedAt: nil, // Should remain NULL until completed } @@ -288,31 +332,86 @@ func TestNullHandling(t *testing.T) { // Test blob with NULL uploaded_ts t.Run("blob not uploaded", func(t *testing.T) { - blob := &Blob{ - ID: types.NewBlobID(), - Hash: types.BlobHash("test-hash"), - CreatedTS: time.Now(), - UploadedTS: nil, // Not uploaded yet - } - - err := repos.Blobs.Create(ctx, nil, blob) - if err != nil { - t.Fatal(err) - } - - retrieved, err := repos.Blobs.GetByID(ctx, blob.ID.String()) - if err != nil { - t.Fatal(err) - } - - if retrieved.UploadedTS != nil { - t.Error("expected nil UploadedTS for non-uploaded blob") - } + t.Parallel() + verifyBlobNullUploadTS(ctx, t, repos) }) } +// verifyBlobNullUploadTS checks that a blob created without an upload +// timestamp round-trips with UploadedTS nil. +func verifyBlobNullUploadTS( + ctx context.Context, t *testing.T, repos *Repositories, +) { + t.Helper() + + blob := &Blob{ + ID: types.NewBlobID(), + Hash: types.BlobHash("test-hash"), + CreatedTS: time.Now(), + UploadedTS: nil, // Not uploaded yet + } + + err := repos.Blobs.Create(ctx, nil, blob) + if err != nil { + t.Fatal(err) + } + + retrieved, err := repos.Blobs.GetByID(ctx, blob.ID.String()) + if err != nil { + t.Fatal(err) + } + + if retrieved.UploadedTS != nil { + t.Error("expected nil UploadedTS for non-uploaded blob") + } +} + +// createLargeDatasetFiles creates fileCount files and adds every other +// one to the snapshot. +func createLargeDatasetFiles( + t *testing.T, + repos *Repositories, + snapshotID string, + fileCount int, +) { + t.Helper() + + ctx := context.Background() + start := time.Now() + + for i := range fileCount { + file := &File{ + Path: types.FilePath(fmt.Sprintf("/large/file%05d.txt", i)), + MTime: time.Now(), + Size: int64(i * 1024), + Mode: 0644, + UID: uint32(1000 + (i % 10)), + GID: uint32(1000 + (i % 10)), + } + + err := repos.Files.Create(ctx, nil, file) + if err != nil { + t.Fatalf("failed to create file %d: %v", i, err) + } + + // Add half to snapshot + if i%2 == 0 { + err = repos.Snapshots.AddFileByID(ctx, nil, snapshotID, file.ID) + if err != nil { + t.Fatal(err) + } + } + } + + t.Logf("Created %d files in %v", fileCount, time.Since(start)) +} + // TestLargeDatasets tests operations with large amounts of data +// +//nolint:tparallel // subtests share one database and are order-dependent func TestLargeDatasets(t *testing.T) { + t.Parallel() + if testing.Short() { t.Skip("skipping large dataset test in short mode") } @@ -326,7 +425,7 @@ func TestLargeDatasets(t *testing.T) { // Create a snapshot snapshot := &Snapshot{ ID: "large-dataset-test", - Hostname: "test-host", + Hostname: internalTestHost, StartedAt: time.Now(), } @@ -338,40 +437,13 @@ func TestLargeDatasets(t *testing.T) { // Create many files const fileCount = 1000 - fileIDs := make([]types.FileID, fileCount) - + //nolint:paralleltest // phases share one database and are order-dependent t.Run("create many files", func(t *testing.T) { - start := time.Now() - for i := range fileCount { - file := &File{ - Path: types.FilePath(fmt.Sprintf("/large/file%05d.txt", i)), - MTime: time.Now(), - Size: int64(i * 1024), - Mode: 0644, - UID: uint32(1000 + (i % 10)), - GID: uint32(1000 + (i % 10)), - } - - err := repos.Files.Create(ctx, nil, file) - if err != nil { - t.Fatalf("failed to create file %d: %v", i, err) - } - - fileIDs[i] = file.ID - - // Add half to snapshot - if i%2 == 0 { - err = repos.Snapshots.AddFileByID(ctx, nil, snapshot.ID.String(), file.ID) - if err != nil { - t.Fatal(err) - } - } - } - - t.Logf("Created %d files in %v", fileCount, time.Since(start)) + createLargeDatasetFiles(t, repos, snapshot.ID.String(), fileCount) }) // Test ListByPrefix performance + //nolint:paralleltest // phases share one database and are order-dependent t.Run("list by prefix performance", func(t *testing.T) { start := time.Now() @@ -388,6 +460,7 @@ func TestLargeDatasets(t *testing.T) { }) // Test orphaned cleanup performance + //nolint:paralleltest // phases share one database and are order-dependent t.Run("orphaned cleanup performance", func(t *testing.T) { start := time.Now() @@ -405,21 +478,26 @@ func TestLargeDatasets(t *testing.T) { } if len(files) != fileCount/2 { - t.Errorf("expected %d files after cleanup, got %d", fileCount/2, len(files)) + t.Errorf("expected %d files after cleanup, got %d", + fileCount/2, len(files)) } }) } // TestErrorPropagation tests that errors are properly propagated func TestErrorPropagation(t *testing.T) { + t.Parallel() + db, cleanup := setupTestDB(t) - defer cleanup() + t.Cleanup(cleanup) ctx := context.Background() repos := NewRepositories(db) // Test GetByID with non-existent ID t.Run("GetByID non-existent", func(t *testing.T) { + t.Parallel() + file, err := repos.Files.GetByID(ctx, types.NewFileID()) if err != nil { t.Errorf("GetByID should not return error for non-existent ID, got: %v", err) @@ -432,9 +510,12 @@ func TestErrorPropagation(t *testing.T) { // Test GetByPath with non-existent path t.Run("GetByPath non-existent", func(t *testing.T) { + t.Parallel() + file, err := repos.Files.GetByPath(ctx, "/non/existent/path.txt") if err != nil { - t.Errorf("GetByPath should not return error for non-existent path, got: %v", err) + t.Errorf("GetByPath should not return error for non-existent path, got: %v", + err) } if file != nil { @@ -444,6 +525,8 @@ func TestErrorPropagation(t *testing.T) { // Test invalid foreign key reference t.Run("invalid foreign key", func(t *testing.T) { + t.Parallel() + fc := &FileChunk{ FileID: types.NewFileID(), Idx: 0, @@ -463,8 +546,10 @@ func TestErrorPropagation(t *testing.T) { // TestQueryInjection tests that the system is safe from SQL injection func TestQueryInjection(t *testing.T) { + t.Parallel() + db, cleanup := setupTestDB(t) - defer cleanup() + t.Cleanup(cleanup) ctx := context.Background() repos := NewRepositories(db) @@ -479,6 +564,8 @@ func TestQueryInjection(t *testing.T) { for _, injection := range injectionTests { t.Run("injection attempt", func(t *testing.T) { + t.Parallel() + // Try injection in file path file := &File{ Path: types.FilePath(injection), @@ -495,7 +582,7 @@ func TestQueryInjection(t *testing.T) { // Verify tables still exist var count int - err := db.conn.QueryRow("SELECT COUNT(*) FROM files").Scan(&count) + err := db.conn.QueryRowContext(ctx, countFilesQuery).Scan(&count) if err != nil { t.Fatal("files table was damaged by injection") } @@ -505,6 +592,8 @@ func TestQueryInjection(t *testing.T) { // TestTimezoneHandling tests that times are properly handled in UTC func TestTimezoneHandling(t *testing.T) { + t.Parallel() + db, cleanup := setupTestDB(t) defer cleanup() diff --git a/internal/database/snapshots.go b/internal/database/snapshots.go index 7b620a2..7bc550a 100644 --- a/internal/database/snapshots.go +++ b/internal/database/snapshots.go @@ -3,6 +3,7 @@ package database import ( "context" "database/sql" + "errors" "fmt" "strings" "time" @@ -10,19 +11,27 @@ import ( "sneak.berlin/go/vaultik/internal/types" ) +// SnapshotRepository provides access to the snapshots table and its +// snapshot_files / snapshot_blobs association tables. type SnapshotRepository struct { db *DB } +// NewSnapshotRepository creates a SnapshotRepository backed by db. func NewSnapshotRepository(db *DB) *SnapshotRepository { return &SnapshotRepository{db: db} } -func (r *SnapshotRepository) Create(ctx context.Context, tx *sql.Tx, snapshot *Snapshot) error { +// Create inserts a snapshot row, using tx when non-nil. +func (r *SnapshotRepository) Create( + ctx context.Context, tx *sql.Tx, snapshot *Snapshot, +) error { query := ` - INSERT INTO snapshots (id, hostname, vaultik_version, vaultik_git_revision, started_at, completed_at, - file_count, chunk_count, blob_count, total_size, blob_size, blob_uncompressed_size, - compression_ratio, compression_level, upload_bytes, upload_duration_ms) + INSERT INTO snapshots (id, hostname, vaultik_version, + vaultik_git_revision, started_at, completed_at, + file_count, chunk_count, blob_count, total_size, blob_size, + blob_uncompressed_size, compression_ratio, compression_level, + upload_bytes, upload_duration_ms) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ` @@ -33,15 +42,21 @@ func (r *SnapshotRepository) Create(ctx context.Context, tx *sql.Tx, snapshot *S completedAt = &ts } + args := []any{ + snapshot.ID, snapshot.Hostname, snapshot.VaultikVersion, + snapshot.VaultikGitRevision, snapshot.StartedAt.Unix(), + completedAt, snapshot.FileCount, snapshot.ChunkCount, + snapshot.BlobCount, snapshot.TotalSize, snapshot.BlobSize, + snapshot.BlobUncompressedSize, snapshot.CompressionRatio, + snapshot.CompressionLevel, snapshot.UploadBytes, + snapshot.UploadDurationMs, + } + var err error if tx != nil { - _, err = tx.ExecContext(ctx, query, snapshot.ID, snapshot.Hostname, snapshot.VaultikVersion, snapshot.VaultikGitRevision, snapshot.StartedAt.Unix(), - completedAt, snapshot.FileCount, snapshot.ChunkCount, snapshot.BlobCount, snapshot.TotalSize, snapshot.BlobSize, snapshot.BlobUncompressedSize, - snapshot.CompressionRatio, snapshot.CompressionLevel, snapshot.UploadBytes, snapshot.UploadDurationMs) + _, err = tx.ExecContext(ctx, query, args...) } else { - _, err = r.db.ExecWithLog(ctx, query, snapshot.ID, snapshot.Hostname, snapshot.VaultikVersion, snapshot.VaultikGitRevision, snapshot.StartedAt.Unix(), - completedAt, snapshot.FileCount, snapshot.ChunkCount, snapshot.BlobCount, snapshot.TotalSize, snapshot.BlobSize, snapshot.BlobUncompressedSize, - snapshot.CompressionRatio, snapshot.CompressionLevel, snapshot.UploadBytes, snapshot.UploadDurationMs) + _, err = r.db.ExecWithLog(ctx, query, args...) } if err != nil { @@ -51,7 +66,14 @@ func (r *SnapshotRepository) Create(ctx context.Context, tx *sql.Tx, snapshot *S return nil } -func (r *SnapshotRepository) UpdateCounts(ctx context.Context, tx *sql.Tx, snapshotID string, fileCount, chunkCount, blobCount, totalSize, blobSize int64) error { +// UpdateCounts updates a snapshot's file/chunk/blob counters and sizes, +// recomputing the compression ratio, using tx when non-nil. +func (r *SnapshotRepository) UpdateCounts( + ctx context.Context, + tx *sql.Tx, + snapshotID string, + fileCount, chunkCount, blobCount, totalSize, blobSize int64, +) error { compressionRatio := 1.0 if totalSize > 0 { compressionRatio = float64(blobSize) / float64(totalSize) @@ -70,9 +92,13 @@ func (r *SnapshotRepository) UpdateCounts(ctx context.Context, tx *sql.Tx, snaps var err error if tx != nil { - _, err = tx.ExecContext(ctx, query, fileCount, chunkCount, blobCount, totalSize, blobSize, compressionRatio, snapshotID) + _, err = tx.ExecContext(ctx, query, + fileCount, chunkCount, blobCount, totalSize, blobSize, + compressionRatio, snapshotID) } else { - _, err = r.db.ExecWithLog(ctx, query, fileCount, chunkCount, blobCount, totalSize, blobSize, compressionRatio, snapshotID) + _, err = r.db.ExecWithLog(ctx, query, + fileCount, chunkCount, blobCount, totalSize, blobSize, + compressionRatio, snapshotID) } if err != nil { @@ -83,34 +109,23 @@ func (r *SnapshotRepository) UpdateCounts(ctx context.Context, tx *sql.Tx, snaps } // UpdateExtendedStats updates extended statistics for a snapshot -func (r *SnapshotRepository) UpdateExtendedStats(ctx context.Context, tx *sql.Tx, snapshotID string, blobUncompressedSize int64, compressionLevel int, uploadDurationMs int64) error { - // Calculate compression ratio based on uncompressed vs compressed sizes - var compressionRatio float64 - - if blobUncompressedSize > 0 { - // Get current blob_size from DB to calculate ratio - var blobSize int64 - - queryGet := `SELECT blob_size FROM snapshots WHERE id = ?` - if tx != nil { - err := tx.QueryRowContext(ctx, queryGet, snapshotID).Scan(&blobSize) - if err != nil { - return fmt.Errorf("getting blob size: %w", err) - } - } else { - err := r.db.conn.QueryRowContext(ctx, queryGet, snapshotID).Scan(&blobSize) - if err != nil { - return fmt.Errorf("getting blob size: %w", err) - } - } - - compressionRatio = float64(blobSize) / float64(blobUncompressedSize) - } else { - compressionRatio = 1.0 +func (r *SnapshotRepository) UpdateExtendedStats( + ctx context.Context, + tx *sql.Tx, + snapshotID string, + blobUncompressedSize int64, + compressionLevel int, + uploadDurationMs int64, +) error { + compressionRatio, err := r.extendedCompressionRatio( + ctx, tx, snapshotID, blobUncompressedSize, + ) + if err != nil { + return err } query := ` - UPDATE snapshots + UPDATE snapshots SET blob_uncompressed_size = ?, compression_ratio = ?, compression_level = ?, @@ -119,11 +134,14 @@ func (r *SnapshotRepository) UpdateExtendedStats(ctx context.Context, tx *sql.Tx WHERE id = ? ` - var err error if tx != nil { - _, err = tx.ExecContext(ctx, query, blobUncompressedSize, compressionRatio, compressionLevel, uploadDurationMs, snapshotID) + _, err = tx.ExecContext(ctx, query, + blobUncompressedSize, compressionRatio, compressionLevel, + uploadDurationMs, snapshotID) } else { - _, err = r.db.ExecWithLog(ctx, query, blobUncompressedSize, compressionRatio, compressionLevel, uploadDurationMs, snapshotID) + _, err = r.db.ExecWithLog(ctx, query, + blobUncompressedSize, compressionRatio, compressionLevel, + uploadDurationMs, snapshotID) } if err != nil { @@ -133,7 +151,11 @@ func (r *SnapshotRepository) UpdateExtendedStats(ctx context.Context, tx *sql.Tx return nil } -func (r *SnapshotRepository) GetByID(ctx context.Context, snapshotID string) (*Snapshot, error) { +// GetByID returns the snapshot with the given ID, or nil if no such +// snapshot exists. +func (r *SnapshotRepository) GetByID( + ctx context.Context, snapshotID string, +) (*Snapshot, error) { query := ` SELECT id, hostname, vaultik_version, vaultik_git_revision, started_at, completed_at, file_count, chunk_count, blob_count, total_size, blob_size, blob_uncompressed_size, @@ -167,8 +189,8 @@ func (r *SnapshotRepository) GetByID(ctx context.Context, snapshotID string) (*S &snapshot.UploadDurationMs, ) - if err == sql.ErrNoRows { - return nil, nil + if errors.Is(err, sql.ErrNoRows) { + return nil, nil //nolint:nilnil // nil,nil signals not-found; callers check nil } if err != nil { @@ -184,9 +206,14 @@ func (r *SnapshotRepository) GetByID(ctx context.Context, snapshotID string) (*S return &snapshot, nil } -func (r *SnapshotRepository) ListRecent(ctx context.Context, limit int) ([]*Snapshot, error) { +// ListRecent returns up to limit snapshots, most recently started first. +func (r *SnapshotRepository) ListRecent( + ctx context.Context, limit int, +) ([]*Snapshot, error) { query := ` - SELECT id, hostname, vaultik_version, vaultik_git_revision, started_at, completed_at, file_count, chunk_count, blob_count, total_size, blob_size, compression_ratio + SELECT id, hostname, vaultik_version, vaultik_git_revision, + started_at, completed_at, file_count, chunk_count, blob_count, + total_size, blob_size, compression_ratio FROM snapshots ORDER BY started_at DESC LIMIT ? @@ -198,47 +225,13 @@ func (r *SnapshotRepository) ListRecent(ctx context.Context, limit int) ([]*Snap } defer CloseRows(rows) - var snapshots []*Snapshot - - for rows.Next() { - var ( - snapshot Snapshot - startedAtUnix int64 - completedAtUnix *int64 - ) - - err := rows.Scan( - &snapshot.ID, - &snapshot.Hostname, - &snapshot.VaultikVersion, - &snapshot.VaultikGitRevision, - &startedAtUnix, - &completedAtUnix, - &snapshot.FileCount, - &snapshot.ChunkCount, - &snapshot.BlobCount, - &snapshot.TotalSize, - &snapshot.BlobSize, - &snapshot.CompressionRatio, - ) - if err != nil { - return nil, fmt.Errorf("scanning snapshot: %w", err) - } - - snapshot.StartedAt = time.Unix(startedAtUnix, 0) - if completedAtUnix != nil { - t := time.Unix(*completedAtUnix, 0) - snapshot.CompletedAt = &t - } - - snapshots = append(snapshots, &snapshot) - } - - return snapshots, rows.Err() + return r.scanSnapshotRows(rows) } // MarkComplete marks a snapshot as completed with the current timestamp -func (r *SnapshotRepository) MarkComplete(ctx context.Context, tx *sql.Tx, snapshotID string) error { +func (r *SnapshotRepository) MarkComplete( + ctx context.Context, tx *sql.Tx, snapshotID string, +) error { query := ` UPDATE snapshots SET completed_at = ? @@ -262,7 +255,9 @@ func (r *SnapshotRepository) MarkComplete(ctx context.Context, tx *sql.Tx, snaps } // AddFile adds a file to a snapshot -func (r *SnapshotRepository) AddFile(ctx context.Context, tx *sql.Tx, snapshotID string, filePath string) error { +func (r *SnapshotRepository) AddFile( + ctx context.Context, tx *sql.Tx, snapshotID string, filePath string, +) error { query := ` INSERT OR IGNORE INTO snapshot_files (snapshot_id, file_id) SELECT ?, id FROM files WHERE path = ? @@ -283,7 +278,9 @@ func (r *SnapshotRepository) AddFile(ctx context.Context, tx *sql.Tx, snapshotID } // AddFileByID adds a file to a snapshot by file ID -func (r *SnapshotRepository) AddFileByID(ctx context.Context, tx *sql.Tx, snapshotID string, fileID types.FileID) error { +func (r *SnapshotRepository) AddFileByID( + ctx context.Context, tx *sql.Tx, snapshotID string, fileID types.FileID, +) error { query := ` INSERT OR IGNORE INTO snapshot_files (snapshot_id, file_id) VALUES (?, ?) @@ -304,12 +301,17 @@ func (r *SnapshotRepository) AddFileByID(ctx context.Context, tx *sql.Tx, snapsh } // AddFilesByIDBatch adds multiple files to a snapshot in batched inserts -func (r *SnapshotRepository) AddFilesByIDBatch(ctx context.Context, tx *sql.Tx, snapshotID string, fileIDs []types.FileID) error { +func (r *SnapshotRepository) AddFilesByIDBatch( + ctx context.Context, tx *sql.Tx, snapshotID string, fileIDs []types.FileID, +) error { if len(fileIDs) == 0 { return nil } - // Each entry has 2 values, so batch at 400 to be safe + // Each snapshot_files row binds this many SQL variables. + const snapshotFileCols = 2 + + // Batch at 400 rows to be safe with SQLite's variable limit. const batchSize = 400 for i := 0; i < len(fileIDs); i += batchSize { @@ -319,7 +321,7 @@ func (r *SnapshotRepository) AddFilesByIDBatch(ctx context.Context, tx *sql.Tx, query := "INSERT OR IGNORE INTO snapshot_files (snapshot_id, file_id) VALUES " - args := make([]any, 0, len(batch)*2) + args := make([]any, 0, len(batch)*snapshotFileCols) var querySb312 strings.Builder @@ -333,7 +335,7 @@ func (r *SnapshotRepository) AddFilesByIDBatch(ctx context.Context, tx *sql.Tx, args = append(args, snapshotID, fileID.String()) } - query += querySb312.String() + query += querySb312.String() //nolint:gosec // G202: appends "?" placeholders only var err error if tx != nil { @@ -360,7 +362,9 @@ func (r *SnapshotRepository) AddFilesByIDBatch(ctx context.Context, tx *sql.Tx, // Returns the number of rows inserted (i.e. blobs that were previously // referenced indirectly via file_chunks but not yet recorded in // snapshot_blobs for this snapshot). -func (r *SnapshotRepository) PopulateReferencedBlobs(ctx context.Context, tx *sql.Tx, snapshotID string) (int64, error) { +func (r *SnapshotRepository) PopulateReferencedBlobs( + ctx context.Context, tx *sql.Tx, snapshotID string, +) (int64, error) { query := ` INSERT OR IGNORE INTO snapshot_blobs (snapshot_id, blob_id, blob_hash) SELECT DISTINCT ?, blobs.id, blobs.blob_hash @@ -392,7 +396,13 @@ func (r *SnapshotRepository) PopulateReferencedBlobs(ctx context.Context, tx *sq } // AddBlob adds a blob to a snapshot -func (r *SnapshotRepository) AddBlob(ctx context.Context, tx *sql.Tx, snapshotID string, blobID types.BlobID, blobHash types.BlobHash) error { +func (r *SnapshotRepository) AddBlob( + ctx context.Context, + tx *sql.Tx, + snapshotID string, + blobID types.BlobID, + blobHash types.BlobHash, +) error { query := ` INSERT OR IGNORE INTO snapshot_blobs (snapshot_id, blob_id, blob_hash) VALUES (?, ?, ?) @@ -413,7 +423,9 @@ func (r *SnapshotRepository) AddBlob(ctx context.Context, tx *sql.Tx, snapshotID } // GetBlobHashes returns all blob hashes for a snapshot -func (r *SnapshotRepository) GetBlobHashes(ctx context.Context, snapshotID string) ([]string, error) { +func (r *SnapshotRepository) GetBlobHashes( + ctx context.Context, snapshotID string, +) ([]string, error) { query := ` SELECT sb.blob_hash FROM snapshot_blobs sb @@ -443,8 +455,11 @@ func (r *SnapshotRepository) GetBlobHashes(ctx context.Context, snapshotID strin return blobs, rows.Err() } -// GetSnapshotTotalCompressedSize returns the total compressed size of all blobs referenced by a snapshot -func (r *SnapshotRepository) GetSnapshotTotalCompressedSize(ctx context.Context, snapshotID string) (int64, error) { +// GetSnapshotTotalCompressedSize returns the total compressed size of all +// blobs referenced by a snapshot. +func (r *SnapshotRepository) GetSnapshotTotalCompressedSize( + ctx context.Context, snapshotID string, +) (int64, error) { query := ` SELECT COALESCE(SUM(b.compressed_size), 0) FROM snapshot_blobs sb @@ -464,7 +479,9 @@ func (r *SnapshotRepository) GetSnapshotTotalCompressedSize(ctx context.Context, // GetSnapshotUncompressedChunkSize returns the sum of plaintext sizes of all unique // chunks referenced by a snapshot (via snapshot_files → file_chunks → chunks). -func (r *SnapshotRepository) GetSnapshotUncompressedChunkSize(ctx context.Context, snapshotID string) (int64, error) { +func (r *SnapshotRepository) GetSnapshotUncompressedChunkSize( + ctx context.Context, snapshotID string, +) (int64, error) { query := ` SELECT COALESCE(SUM(c.size), 0) FROM ( @@ -490,7 +507,9 @@ func (r *SnapshotRepository) GetSnapshotUncompressedChunkSize(ctx context.Contex // referenced by this snapshot but not by any earlier completed snapshot known to // the local database. The result is the marginal uncompressed data this snapshot // added to the dedup pool — i.e., the delta from prior snapshots. -func (r *SnapshotRepository) GetSnapshotNewChunkSize(ctx context.Context, snapshotID string) (int64, error) { +func (r *SnapshotRepository) GetSnapshotNewChunkSize( + ctx context.Context, snapshotID string, +) (int64, error) { query := ` WITH this_snap_chunks AS ( SELECT DISTINCT fc.chunk_hash @@ -515,7 +534,9 @@ func (r *SnapshotRepository) GetSnapshotNewChunkSize(ctx context.Context, snapsh var totalSize int64 - err := r.db.conn.QueryRowContext(ctx, query, snapshotID, snapshotID, snapshotID).Scan(&totalSize) + err := r.db.conn.QueryRowContext( + ctx, query, snapshotID, snapshotID, snapshotID, + ).Scan(&totalSize) if err != nil { return 0, fmt.Errorf("querying new chunk size: %w", err) } @@ -524,9 +545,13 @@ func (r *SnapshotRepository) GetSnapshotNewChunkSize(ctx context.Context, snapsh } // GetIncompleteSnapshots returns all snapshots that haven't been completed -func (r *SnapshotRepository) GetIncompleteSnapshots(ctx context.Context) ([]*Snapshot, error) { +func (r *SnapshotRepository) GetIncompleteSnapshots( + ctx context.Context, +) ([]*Snapshot, error) { query := ` - SELECT id, hostname, vaultik_version, vaultik_git_revision, started_at, completed_at, file_count, chunk_count, blob_count, total_size, blob_size, compression_ratio + SELECT id, hostname, vaultik_version, vaultik_git_revision, + started_at, completed_at, file_count, chunk_count, blob_count, + total_size, blob_size, compression_ratio FROM snapshots WHERE completed_at IS NULL ORDER BY started_at DESC @@ -538,49 +563,17 @@ func (r *SnapshotRepository) GetIncompleteSnapshots(ctx context.Context) ([]*Sna } defer CloseRows(rows) - var snapshots []*Snapshot - - for rows.Next() { - var ( - snapshot Snapshot - startedAtUnix int64 - completedAtUnix *int64 - ) - - err := rows.Scan( - &snapshot.ID, - &snapshot.Hostname, - &snapshot.VaultikVersion, - &snapshot.VaultikGitRevision, - &startedAtUnix, - &completedAtUnix, - &snapshot.FileCount, - &snapshot.ChunkCount, - &snapshot.BlobCount, - &snapshot.TotalSize, - &snapshot.BlobSize, - &snapshot.CompressionRatio, - ) - if err != nil { - return nil, fmt.Errorf("scanning snapshot: %w", err) - } - - snapshot.StartedAt = time.Unix(startedAtUnix, 0) - if completedAtUnix != nil { - t := time.Unix(*completedAtUnix, 0) - snapshot.CompletedAt = &t - } - - snapshots = append(snapshots, &snapshot) - } - - return snapshots, rows.Err() + return r.scanSnapshotRows(rows) } // GetIncompleteByHostname returns all incomplete snapshots for a specific hostname -func (r *SnapshotRepository) GetIncompleteByHostname(ctx context.Context, hostname string) ([]*Snapshot, error) { +func (r *SnapshotRepository) GetIncompleteByHostname( + ctx context.Context, hostname string, +) ([]*Snapshot, error) { query := ` - SELECT id, hostname, vaultik_version, vaultik_git_revision, started_at, completed_at, file_count, chunk_count, blob_count, total_size, blob_size, compression_ratio + SELECT id, hostname, vaultik_version, vaultik_git_revision, + started_at, completed_at, file_count, chunk_count, blob_count, + total_size, blob_size, compression_ratio FROM snapshots WHERE completed_at IS NULL AND hostname = ? ORDER BY started_at DESC @@ -644,7 +637,9 @@ func (r *SnapshotRepository) Delete(ctx context.Context, snapshotID string) erro } // DeleteSnapshotFiles removes all snapshot_files entries for a snapshot -func (r *SnapshotRepository) DeleteSnapshotFiles(ctx context.Context, snapshotID string) error { +func (r *SnapshotRepository) DeleteSnapshotFiles( + ctx context.Context, snapshotID string, +) error { query := `DELETE FROM snapshot_files WHERE snapshot_id = ?` _, err := r.db.ExecWithLog(ctx, query, snapshotID) @@ -656,7 +651,9 @@ func (r *SnapshotRepository) DeleteSnapshotFiles(ctx context.Context, snapshotID } // DeleteSnapshotBlobs removes all snapshot_blobs entries for a snapshot -func (r *SnapshotRepository) DeleteSnapshotBlobs(ctx context.Context, snapshotID string) error { +func (r *SnapshotRepository) DeleteSnapshotBlobs( + ctx context.Context, snapshotID string, +) error { query := `DELETE FROM snapshot_blobs WHERE snapshot_id = ?` _, err := r.db.ExecWithLog(ctx, query, snapshotID) @@ -668,7 +665,9 @@ func (r *SnapshotRepository) DeleteSnapshotBlobs(ctx context.Context, snapshotID } // DeleteSnapshotUploads removes all uploads entries for a snapshot -func (r *SnapshotRepository) DeleteSnapshotUploads(ctx context.Context, snapshotID string) error { +func (r *SnapshotRepository) DeleteSnapshotUploads( + ctx context.Context, snapshotID string, +) error { query := `DELETE FROM uploads WHERE snapshot_id = ?` _, err := r.db.ExecWithLog(ctx, query, snapshotID) @@ -678,3 +677,77 @@ func (r *SnapshotRepository) DeleteSnapshotUploads(ctx context.Context, snapshot return nil } + +// extendedCompressionRatio computes the compression ratio for a snapshot +// from its stored blob_size and the given uncompressed size. Returns 1.0 +// when the uncompressed size is zero. +func (r *SnapshotRepository) extendedCompressionRatio( + ctx context.Context, + tx *sql.Tx, + snapshotID string, + blobUncompressedSize int64, +) (float64, error) { + if blobUncompressedSize <= 0 { + return 1.0, nil + } + + // Get current blob_size from DB to calculate ratio + var blobSize int64 + + queryGet := `SELECT blob_size FROM snapshots WHERE id = ?` + + var err error + if tx != nil { + err = tx.QueryRowContext(ctx, queryGet, snapshotID).Scan(&blobSize) + } else { + err = r.db.conn.QueryRowContext(ctx, queryGet, snapshotID).Scan(&blobSize) + } + + if err != nil { + return 0, fmt.Errorf("getting blob size: %w", err) + } + + return float64(blobSize) / float64(blobUncompressedSize), nil +} + +// scanSnapshotRows scans the standard snapshot column set from a rows +// iterator into Snapshot records. +func (r *SnapshotRepository) scanSnapshotRows(rows *sql.Rows) ([]*Snapshot, error) { + var snapshots []*Snapshot + + for rows.Next() { + var ( + snapshot Snapshot + startedAtUnix int64 + completedAtUnix *int64 + ) + + err := rows.Scan( + &snapshot.ID, + &snapshot.Hostname, + &snapshot.VaultikVersion, + &snapshot.VaultikGitRevision, + &startedAtUnix, + &completedAtUnix, + &snapshot.FileCount, + &snapshot.ChunkCount, + &snapshot.BlobCount, + &snapshot.TotalSize, + &snapshot.BlobSize, + &snapshot.CompressionRatio, + ) + if err != nil { + return nil, fmt.Errorf("scanning snapshot: %w", err) + } + + snapshot.StartedAt = time.Unix(startedAtUnix, 0) + if completedAtUnix != nil { + t := time.Unix(*completedAtUnix, 0) + snapshot.CompletedAt = &t + } + + snapshots = append(snapshots, &snapshot) + } + + return snapshots, rows.Err() +} diff --git a/internal/database/snapshots_test.go b/internal/database/snapshots_test.go index a1efaa1..5204a61 100644 --- a/internal/database/snapshots_test.go +++ b/internal/database/snapshots_test.go @@ -1,4 +1,4 @@ -package database +package database_test import ( "context" @@ -7,6 +7,7 @@ import ( "testing" "time" + "sneak.berlin/go/vaultik/internal/database" "sneak.berlin/go/vaultik/internal/types" ) @@ -21,17 +22,19 @@ const ( ) func TestSnapshotRepository(t *testing.T) { + t.Parallel() + db, cleanup := setupTestDB(t) defer cleanup() ctx := context.Background() - repo := NewSnapshotRepository(db) + repo := database.NewSnapshotRepository(db) // Test Create - snapshot := &Snapshot{ + snapshot := &database.Snapshot{ ID: "2024-01-01T12:00:00Z", - Hostname: "test-host", - VaultikVersion: "1.0.0", + Hostname: testHostname, + VaultikVersion: testVersion, StartedAt: time.Now().Truncate(time.Second), CompletedAt: nil, FileCount: 100, @@ -62,20 +65,52 @@ func TestSnapshotRepository(t *testing.T) { } if retrieved.Hostname != snapshot.Hostname { - t.Errorf("hostname mismatch: got %s, want %s", retrieved.Hostname, snapshot.Hostname) + t.Errorf("hostname mismatch: got %s, want %s", + retrieved.Hostname, snapshot.Hostname) } if retrieved.FileCount != snapshot.FileCount { - t.Errorf("file count mismatch: got %d, want %d", retrieved.FileCount, snapshot.FileCount) + t.Errorf("file count mismatch: got %d, want %d", + retrieved.FileCount, snapshot.FileCount) + } +} + +func TestSnapshotRepositoryUpdateCounts(t *testing.T) { + t.Parallel() + + db, cleanup := setupTestDB(t) + defer cleanup() + + ctx := context.Background() + repo := database.NewSnapshotRepository(db) + + snapshot := &database.Snapshot{ + ID: "2024-01-02T12:00:00Z", + Hostname: testHostname, + VaultikVersion: testVersion, + StartedAt: time.Now().Truncate(time.Second), + CompletedAt: nil, + FileCount: 100, + ChunkCount: 500, + BlobCount: 10, + TotalSize: oneHundredMebibytes, + BlobSize: fortyMebibytes, + CompressionRatio: compressionRatioPoint4, + } + + err := repo.Create(ctx, nil, snapshot) + if err != nil { + t.Fatalf("failed to create snapshot: %v", err) } // Test UpdateCounts - err = repo.UpdateCounts(ctx, nil, snapshot.ID.String(), 200, 1000, 20, twoHundredMebibytes, sixtyMebibytes) + err = repo.UpdateCounts(ctx, nil, snapshot.ID.String(), + 200, 1000, 20, twoHundredMebibytes, sixtyMebibytes) if err != nil { t.Fatalf("failed to update counts: %v", err) } - retrieved, err = repo.GetByID(ctx, snapshot.ID.String()) + retrieved, err := repo.GetByID(ctx, snapshot.ID.String()) if err != nil { t.Fatalf("failed to get updated snapshot: %v", err) } @@ -85,7 +120,8 @@ func TestSnapshotRepository(t *testing.T) { } if retrieved.ChunkCount != 1000 { - t.Errorf("chunk count not updated: got %d, want %d", retrieved.ChunkCount, 1000) + t.Errorf("chunk count not updated: got %d, want %d", + retrieved.ChunkCount, 1000) } if retrieved.BlobCount != 20 { @@ -93,25 +129,37 @@ func TestSnapshotRepository(t *testing.T) { } if retrieved.TotalSize != twoHundredMebibytes { - t.Errorf("total size not updated: got %d, want %d", retrieved.TotalSize, twoHundredMebibytes) + t.Errorf("total size not updated: got %d, want %d", + retrieved.TotalSize, twoHundredMebibytes) } if retrieved.BlobSize != sixtyMebibytes { - t.Errorf("blob size not updated: got %d, want %d", retrieved.BlobSize, sixtyMebibytes) + t.Errorf("blob size not updated: got %d, want %d", + retrieved.BlobSize, sixtyMebibytes) } expectedRatio := compressionRatioPoint3 // 0.3 if math.Abs(retrieved.CompressionRatio-expectedRatio) > 0.001 { - t.Errorf("compression ratio not updated: got %f, want %f", retrieved.CompressionRatio, expectedRatio) + t.Errorf("compression ratio not updated: got %f, want %f", + retrieved.CompressionRatio, expectedRatio) } +} - // Test ListRecent - // Add more snapshots - for i := 2; i <= 5; i++ { - s := &Snapshot{ +func TestSnapshotRepositoryListRecent(t *testing.T) { + t.Parallel() + + db, cleanup := setupTestDB(t) + defer cleanup() + + ctx := context.Background() + repo := database.NewSnapshotRepository(db) + + // Add snapshots + for i := 1; i <= 5; i++ { + s := &database.Snapshot{ ID: types.SnapshotID(fmt.Sprintf("2024-01-0%dT12:00:00Z", i)), - Hostname: "test-host", - VaultikVersion: "1.0.0", + Hostname: testHostname, + VaultikVersion: testVersion, StartedAt: time.Now().Add(time.Duration(i) * time.Hour).Truncate(time.Second), CompletedAt: nil, FileCount: int64(100 * i), @@ -144,11 +192,13 @@ func TestSnapshotRepository(t *testing.T) { } func TestSnapshotRepositoryNotFound(t *testing.T) { + t.Parallel() + db, cleanup := setupTestDB(t) defer cleanup() ctx := context.Background() - repo := NewSnapshotRepository(db) + repo := database.NewSnapshotRepository(db) // Test GetByID with non-existent ID snapshot, err := repo.GetByID(ctx, "nonexistent") @@ -161,7 +211,8 @@ func TestSnapshotRepositoryNotFound(t *testing.T) { } // Test UpdateCounts on non-existent snapshot - err = repo.UpdateCounts(ctx, nil, "nonexistent", 100, 200, 10, oneHundredMebibytes, fortyMebibytes) + err = repo.UpdateCounts(ctx, nil, "nonexistent", + 100, 200, 10, oneHundredMebibytes, fortyMebibytes) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -169,16 +220,18 @@ func TestSnapshotRepositoryNotFound(t *testing.T) { } func TestSnapshotRepositoryDuplicate(t *testing.T) { + t.Parallel() + db, cleanup := setupTestDB(t) defer cleanup() ctx := context.Background() - repo := NewSnapshotRepository(db) + repo := database.NewSnapshotRepository(db) - snapshot := &Snapshot{ + snapshot := &database.Snapshot{ ID: "2024-01-01T12:00:00Z", - Hostname: "test-host", - VaultikVersion: "1.0.0", + Hostname: testHostname, + VaultikVersion: testVersion, StartedAt: time.Now().Truncate(time.Second), CompletedAt: nil, FileCount: 100, diff --git a/internal/database/uploads.go b/internal/database/uploads.go index 154cd09..1070550 100644 --- a/internal/database/uploads.go +++ b/internal/database/uploads.go @@ -3,6 +3,7 @@ package database import ( "context" "database/sql" + "errors" "time" "sneak.berlin/go/vaultik/internal/log" @@ -28,7 +29,9 @@ func NewUploadRepository(conn *sql.DB) *UploadRepository { } // Create inserts a new upload record -func (r *UploadRepository) Create(ctx context.Context, tx *sql.Tx, upload *Upload) error { +func (r *UploadRepository) Create( + ctx context.Context, tx *sql.Tx, upload *Upload, +) error { query := ` INSERT INTO uploads (blob_hash, snapshot_id, uploaded_at, size, duration_ms) VALUES (?, ?, ?, ?, ?) @@ -36,16 +39,22 @@ func (r *UploadRepository) Create(ctx context.Context, tx *sql.Tx, upload *Uploa var err error if tx != nil { - _, err = tx.ExecContext(ctx, query, upload.BlobHash, upload.SnapshotID, upload.UploadedAt, upload.Size, upload.DurationMs) + _, err = tx.ExecContext(ctx, query, + upload.BlobHash, upload.SnapshotID, upload.UploadedAt, + upload.Size, upload.DurationMs) } else { - _, err = r.conn.ExecContext(ctx, query, upload.BlobHash, upload.SnapshotID, upload.UploadedAt, upload.Size, upload.DurationMs) + _, err = r.conn.ExecContext(ctx, query, + upload.BlobHash, upload.SnapshotID, upload.UploadedAt, + upload.Size, upload.DurationMs) } return err } // GetByBlobHash retrieves an upload record by blob hash -func (r *UploadRepository) GetByBlobHash(ctx context.Context, blobHash string) (*Upload, error) { +func (r *UploadRepository) GetByBlobHash( + ctx context.Context, blobHash string, +) (*Upload, error) { query := ` SELECT blob_hash, uploaded_at, size, duration_ms FROM uploads @@ -61,8 +70,8 @@ func (r *UploadRepository) GetByBlobHash(ctx context.Context, blobHash string) ( &upload.DurationMs, ) - if err == sql.ErrNoRows { - return nil, nil + if errors.Is(err, sql.ErrNoRows) { + return nil, nil //nolint:nilnil // nil,nil signals not-found; callers check nil } if err != nil { @@ -73,7 +82,9 @@ func (r *UploadRepository) GetByBlobHash(ctx context.Context, blobHash string) ( } // GetRecentUploads retrieves recent uploads ordered by upload time -func (r *UploadRepository) GetRecentUploads(ctx context.Context, limit int) ([]*Upload, error) { +func (r *UploadRepository) GetRecentUploads( + ctx context.Context, limit int, +) ([]*Upload, error) { query := ` SELECT blob_hash, uploaded_at, size, duration_ms FROM uploads @@ -97,7 +108,9 @@ func (r *UploadRepository) GetRecentUploads(ctx context.Context, limit int) ([]* for rows.Next() { var upload Upload - err := rows.Scan(&upload.BlobHash, &upload.UploadedAt, &upload.Size, &upload.DurationMs) + err := rows.Scan( + &upload.BlobHash, &upload.UploadedAt, &upload.Size, &upload.DurationMs, + ) if err != nil { return nil, err } @@ -109,9 +122,11 @@ func (r *UploadRepository) GetRecentUploads(ctx context.Context, limit int) ([]* } // GetUploadStats returns aggregate statistics for uploads -func (r *UploadRepository) GetUploadStats(ctx context.Context, since time.Time) (*UploadStats, error) { +func (r *UploadRepository) GetUploadStats( + ctx context.Context, since time.Time, +) (*UploadStats, error) { query := ` - SELECT + SELECT COUNT(*) as count, COALESCE(SUM(size), 0) as total_size, COALESCE(AVG(duration_ms), 0) as avg_duration_ms, @@ -144,7 +159,9 @@ type UploadStats struct { } // GetCountBySnapshot returns the count of uploads for a specific snapshot -func (r *UploadRepository) GetCountBySnapshot(ctx context.Context, snapshotID string) (int64, error) { +func (r *UploadRepository) GetCountBySnapshot( + ctx context.Context, snapshotID string, +) (int64, error) { query := `SELECT COUNT(*) FROM uploads WHERE snapshot_id = ?` var count int64 diff --git a/internal/globals/globals.go b/internal/globals/globals.go index 21edab6..bbd2843 100644 --- a/internal/globals/globals.go +++ b/internal/globals/globals.go @@ -1,3 +1,5 @@ +// Package globals holds application-wide metadata (name, version, +// commit) that is populated at build time via linker flags. package globals import ( @@ -5,16 +7,16 @@ import ( ) // Appname is the application name, populated from main(). -var Appname string = "vaultik" +var Appname = "vaultik" //nolint:gochecknoglobals // set via -ldflags at build time // Version is the application version, populated from main(). -var Version string = "dev" +var Version = "dev" //nolint:gochecknoglobals // set via -ldflags at build time // Commit is the git commit hash, populated from main(). -var Commit string = "unknown" +var Commit = "unknown" //nolint:gochecknoglobals // set via -ldflags at build time // CommitDate is the ISO-8601 date of the commit, populated from main(). -var CommitDate string = "unknown" +var CommitDate = "unknown" //nolint:gochecknoglobals // set via -ldflags at build time // Author identifies the upstream author of vaultik. const Author = "Jeffrey Paul " @@ -34,7 +36,8 @@ type Globals struct { StartTime time.Time } -// New creates and returns a new Globals instance initialized with the package-level variables. +// New creates and returns a new Globals instance initialized with the +// package-level variables. func New() (*Globals, error) { return &Globals{ Appname: Appname, @@ -44,11 +47,14 @@ func New() (*Globals, error) { }, nil } +// shortCommitLen is the number of commit-hash characters ShortCommit keeps. +const shortCommitLen = 12 + // ShortCommit returns the first 12 chars of the commit hash, or the // whole string if it's shorter (e.g. "unknown"). func (g *Globals) ShortCommit() string { - if len(g.Commit) > 12 { - return g.Commit[:12] + if len(g.Commit) > shortCommitLen { + return g.Commit[:shortCommitLen] } return g.Commit diff --git a/internal/globals/globals_test.go b/internal/globals/globals_test.go index 164602c..2784428 100644 --- a/internal/globals/globals_test.go +++ b/internal/globals/globals_test.go @@ -1,12 +1,16 @@ -package globals +package globals_test import ( "testing" + + "sneak.berlin/go/vaultik/internal/globals" ) // TestGlobalsNew ensures the globals package initializes correctly func TestGlobalsNew(t *testing.T) { - g, err := New() + t.Parallel() + + g, err := globals.New() if err != nil { t.Fatalf("Failed to create Globals: %v", err) } diff --git a/internal/log/log.go b/internal/log/log.go index c9867ee..17025ca 100644 --- a/internal/log/log.go +++ b/internal/log/log.go @@ -1,3 +1,5 @@ +// Package log provides the application-wide structured logger: slog +// with a colorized TTY handler on terminals and JSON output otherwise. package log import ( @@ -12,12 +14,12 @@ import ( "golang.org/x/term" ) -// LogLevel represents the logging level. -type LogLevel int +// Level represents the logging level. +type Level int const ( // LevelFatal represents a fatal error level that will exit the program. - LevelFatal LogLevel = iota + LevelFatal Level = iota // LevelError represents an error level. LevelError // LevelWarn represents a warning level. @@ -38,6 +40,7 @@ type Config struct { Quiet bool } +//nolint:gochecknoglobals // package-level logger is the package's purpose var logger *slog.Logger // Initialize sets up the global logger based on the provided configuration. @@ -45,18 +48,19 @@ func Initialize(cfg Config) { // Determine log level based on configuration var level slog.Level - if cfg.Cron || cfg.Quiet { + switch { + case cfg.Cron || cfg.Quiet: // In cron/quiet mode keep warnings and errors visible — the // whole point of --cron is to stay silent only on total // success, so that anything cron emails to root is genuinely // "something went wrong, look at it." A backup with stuck // permission errors or skipped files should NOT be silent. level = slog.LevelWarn - } else if cfg.Debug || strings.Contains(os.Getenv("GODEBUG"), "vaultik") { + case cfg.Debug || strings.Contains(os.Getenv("GODEBUG"), "vaultik"): level = slog.LevelDebug - } else if cfg.Verbose { + case cfg.Verbose: level = slog.LevelInfo - } else { + default: level = slog.LevelWarn } @@ -78,9 +82,13 @@ func Initialize(cfg Config) { slog.SetDefault(logger) } +// callerSkipFrames is the number of stack frames between runtime.Caller +// and the code that invoked the package-level logging function. +const callerSkipFrames = 2 + // getCaller returns the caller information as a string -func getCaller(skip int) string { - _, file, line, ok := runtime.Caller(skip) +func getCaller() string { + _, file, line, ok := runtime.Caller(callerSkipFrames) if !ok { return "unknown" } @@ -92,7 +100,7 @@ func getCaller(skip int) string { func Fatal(msg string, args ...any) { if logger != nil { // Add caller info to args - args = append(args, "caller", getCaller(2)) + args = append(args, "caller", getCaller()) logger.Error(msg, args...) } @@ -107,7 +115,7 @@ func Fatalf(format string, args ...any) { // Error logs an error message. func Error(msg string, args ...any) { if logger != nil { - args = append(args, "caller", getCaller(2)) + args = append(args, "caller", getCaller()) logger.Error(msg, args...) } } @@ -120,7 +128,7 @@ func Errorf(format string, args ...any) { // Warn logs a warning message. func Warn(msg string, args ...any) { if logger != nil { - args = append(args, "caller", getCaller(2)) + args = append(args, "caller", getCaller()) logger.Warn(msg, args...) } } @@ -133,7 +141,7 @@ func Warnf(format string, args ...any) { // Notice logs a notice message (mapped to Info level). func Notice(msg string, args ...any) { if logger != nil { - args = append(args, "caller", getCaller(2)) + args = append(args, "caller", getCaller()) logger.Info(msg, args...) } } @@ -146,7 +154,7 @@ func Noticef(format string, args ...any) { // Info logs an informational message. func Info(msg string, args ...any) { if logger != nil { - args = append(args, "caller", getCaller(2)) + args = append(args, "caller", getCaller()) logger.Info(msg, args...) } } @@ -159,7 +167,7 @@ func Infof(format string, args ...any) { // Debug logs a debug message. func Debug(msg string, args ...any) { if logger != nil { - args = append(args, "caller", getCaller(2)) + args = append(args, "caller", getCaller()) logger.Debug(msg, args...) } } @@ -179,7 +187,7 @@ func With(args ...any) *slog.Logger { } // WithContext returns a logger with the provided context. -func WithContext(ctx context.Context) *slog.Logger { +func WithContext(_ context.Context) *slog.Logger { return logger } diff --git a/internal/log/module.go b/internal/log/module.go index 1b65128..525f969 100644 --- a/internal/log/module.go +++ b/internal/log/module.go @@ -5,6 +5,8 @@ import ( ) // Module exports logging functionality for dependency injection. +// +//nolint:gochecknoglobals // fx module definitions are package globals var Module = fx.Module("log", fx.Invoke(func(cfg Config) { Initialize(cfg) @@ -12,12 +14,12 @@ var Module = fx.Module("log", ) // New creates a new logger configuration from provided options. -func New(opts LogOptions) Config { +func New(opts Options) Config { return Config(opts) } -// LogOptions are provided by the CLI. -type LogOptions struct { +// Options are provided by the CLI. +type Options struct { Verbose bool Debug bool Cron bool diff --git a/internal/log/tty_handler.go b/internal/log/tty_handler.go index 81c1fe6..e787de8 100644 --- a/internal/log/tty_handler.go +++ b/internal/log/tty_handler.go @@ -94,6 +94,11 @@ func (h *TTYHandler) Handle(_ context.Context, r slog.Record) error { if a.Key == "bytes" { value = formatBytes(a.Value.Int64()) } + case slog.KindAny, slog.KindBool, slog.KindFloat64, slog.KindString, + slog.KindTime, slog.KindUint64, slog.KindGroup, slog.KindLogValuer: + // Plain string form above is already correct for these kinds. + default: + // Future kinds also use the plain string form. } _, _ = fmt.Fprintf(h.out, " %s%s%s=%s%s%s", @@ -109,26 +114,27 @@ func (h *TTYHandler) Handle(_ context.Context, r slog.Record) error { } // WithAttrs returns a new handler with the given attributes. -func (h *TTYHandler) WithAttrs(attrs []slog.Attr) slog.Handler { +func (h *TTYHandler) WithAttrs(_ []slog.Attr) slog.Handler { return h // Simplified for now } // WithGroup returns a new handler with the given group name. -func (h *TTYHandler) WithGroup(name string) slog.Handler { +func (h *TTYHandler) WithGroup(_ string) slog.Handler { return h // Simplified for now } // formatDuration formats a duration in a human-readable way func formatDuration(d time.Duration) string { - if d < time.Millisecond { + switch { + case d < time.Millisecond: return fmt.Sprintf("%dµs", d.Microseconds()) - } else if d < time.Second { + case d < time.Second: return fmt.Sprintf("%dms", d.Milliseconds()) - } else if d < time.Minute { + case d < time.Minute: return fmt.Sprintf("%.1fs", d.Seconds()) + default: + return d.String() } - - return d.String() } // formatBytes formats bytes in a human-readable way diff --git a/internal/models/models.go b/internal/models/models.go index f16e17a..819402a 100644 --- a/internal/models/models.go +++ b/internal/models/models.go @@ -1,3 +1,5 @@ +// Package models defines shared value types describing files, chunks, +// blobs, and snapshots as they move through the backup pipeline. package models import ( diff --git a/internal/models/models_test.go b/internal/models/models_test.go index 1f44ac9..b5c4cab 100644 --- a/internal/models/models_test.go +++ b/internal/models/models_test.go @@ -1,17 +1,21 @@ -package models +package models_test import ( "testing" "time" + + "sneak.berlin/go/vaultik/internal/models" ) // TestModelsCompilation ensures all model types can be instantiated func TestModelsCompilation(t *testing.T) { + t.Parallel() + // This test primarily serves as a compilation test // to ensure all types are properly defined // Test FileInfo - fi := &FileInfo{ + fi := &models.FileInfo{ Path: "/test/file.txt", MTime: time.Now(), Size: 1024, @@ -21,7 +25,7 @@ func TestModelsCompilation(t *testing.T) { } // Test ChunkInfo - ci := &ChunkInfo{ + ci := &models.ChunkInfo{ Hash: "abc123", Size: 512, Offset: 0, @@ -31,7 +35,7 @@ func TestModelsCompilation(t *testing.T) { } // Test BlobInfo - bi := &BlobInfo{ + bi := &models.BlobInfo{ Hash: "blob123", CreatedAt: time.Now(), Size: 1024, @@ -42,7 +46,7 @@ func TestModelsCompilation(t *testing.T) { } // Test Snapshot - s := &Snapshot{ + s := &models.Snapshot{ ID: "2024-01-01T00:00:00Z", Hostname: "test-host", Version: "1.0.0", diff --git a/internal/pidlock/pidlock.go b/internal/pidlock/pidlock.go index e576970..ea15e77 100644 --- a/internal/pidlock/pidlock.go +++ b/internal/pidlock/pidlock.go @@ -21,6 +21,13 @@ type Lock struct { path string } +const ( + // lockDirPerm is the mode for the lock directory (owner-only). + lockDirPerm = 0o700 + // pidFilePerm is the mode for the PID file (owner-only). + pidFilePerm = 0o600 +) + // Acquire attempts to acquire a PID lock in the specified directory. // If the lock file exists and the process is still running, it returns // ErrAlreadyRunning with details about the existing process. @@ -28,7 +35,8 @@ type Lock struct { // a Lock that must be released with Release(). func Acquire(lockDir string) (*Lock, error) { // Ensure lock directory exists - if err := os.MkdirAll(lockDir, 0700); err != nil { + err := os.MkdirAll(lockDir, lockDirPerm) + if err != nil { return nil, fmt.Errorf("creating lock directory: %w", err) } @@ -46,7 +54,9 @@ func Acquire(lockDir string) (*Lock, error) { // Write our PID pid := os.Getpid() - if err := os.WriteFile(lockPath, []byte(strconv.Itoa(pid)), 0600); err != nil { + + err = os.WriteFile(lockPath, []byte(strconv.Itoa(pid)), pidFilePerm) + if err != nil { return nil, fmt.Errorf("writing PID file: %w", err) } @@ -64,7 +74,7 @@ func (l *Lock) Release() error { existingPID, err := readPIDFile(l.path) if err != nil { // File already gone or unreadable - that's fine - return nil + return nil //nolint:nilerr // unreadable lock file means nothing to release } if existingPID != os.Getpid() { @@ -72,7 +82,8 @@ func (l *Lock) Release() error { return nil } - if err := os.Remove(l.path); err != nil && !os.IsNotExist(err) { + err = os.Remove(l.path) + if err != nil && !os.IsNotExist(err) { return fmt.Errorf("removing PID file: %w", err) } @@ -83,7 +94,7 @@ func (l *Lock) Release() error { // readPIDFile reads and parses the PID from a lock file. func readPIDFile(path string) (int, error) { - data, err := os.ReadFile(path) + data, err := os.ReadFile(path) //nolint:gosec // G304: path is our own lock file if err != nil { return 0, err } diff --git a/internal/pidlock/pidlock_test.go b/internal/pidlock/pidlock_test.go index dce224d..3adc2e0 100644 --- a/internal/pidlock/pidlock_test.go +++ b/internal/pidlock/pidlock_test.go @@ -1,4 +1,4 @@ -package pidlock +package pidlock_test import ( "os" @@ -8,18 +8,22 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "sneak.berlin/go/vaultik/internal/pidlock" ) func TestAcquireAndRelease(t *testing.T) { + t.Parallel() + tmpDir := t.TempDir() // Acquire lock - lock, err := Acquire(tmpDir) + lock, err := pidlock.Acquire(tmpDir) require.NoError(t, err) require.NotNil(t, lock) // Verify PID file exists with our PID - data, err := os.ReadFile(filepath.Join(tmpDir, "vaultik.pid")) + pidPath := filepath.Join(tmpDir, "vaultik.pid") + data, err := os.ReadFile(pidPath) //nolint:gosec // G304: test's own temp file require.NoError(t, err) pid, err := strconv.Atoi(string(data)) require.NoError(t, err) @@ -30,27 +34,31 @@ func TestAcquireAndRelease(t *testing.T) { require.NoError(t, err) // Verify PID file is gone - _, err = os.Stat(filepath.Join(tmpDir, "vaultik.pid")) + _, err = os.Stat(pidPath) assert.True(t, os.IsNotExist(err)) } func TestAcquireBlocksSecondInstance(t *testing.T) { + t.Parallel() + tmpDir := t.TempDir() // Acquire first lock - lock1, err := Acquire(tmpDir) + lock1, err := pidlock.Acquire(tmpDir) require.NoError(t, err) require.NotNil(t, lock1) defer func() { _ = lock1.Release() }() // Try to acquire second lock - should fail - lock2, err := Acquire(tmpDir) - assert.ErrorIs(t, err, ErrAlreadyRunning) + lock2, err := pidlock.Acquire(tmpDir) + require.ErrorIs(t, err, pidlock.ErrAlreadyRunning) assert.Nil(t, lock2) } func TestAcquireWithStaleLock(t *testing.T) { + t.Parallel() + tmpDir := t.TempDir() // Write a stale PID file (PID that doesn't exist) @@ -60,14 +68,14 @@ func TestAcquireWithStaleLock(t *testing.T) { require.NoError(t, err) // Should be able to acquire lock (stale lock is cleaned up) - lock, err := Acquire(tmpDir) + lock, err := pidlock.Acquire(tmpDir) require.NoError(t, err) require.NotNil(t, lock) defer func() { _ = lock.Release() }() // Verify our PID is now in the file - data, err := os.ReadFile(pidPath) + data, err := os.ReadFile(pidPath) //nolint:gosec // G304: test's own temp file require.NoError(t, err) pid, err := strconv.Atoi(string(data)) require.NoError(t, err) @@ -75,9 +83,11 @@ func TestAcquireWithStaleLock(t *testing.T) { } func TestReleaseIsIdempotent(t *testing.T) { + t.Parallel() + tmpDir := t.TempDir() - lock, err := Acquire(tmpDir) + lock, err := pidlock.Acquire(tmpDir) require.NoError(t, err) // Release multiple times - should not error @@ -89,17 +99,21 @@ func TestReleaseIsIdempotent(t *testing.T) { } func TestReleaseNilLock(t *testing.T) { - var lock *Lock + t.Parallel() + + var lock *pidlock.Lock err := lock.Release() - assert.NoError(t, err) + require.NoError(t, err) } func TestAcquireCreatesDirectory(t *testing.T) { + t.Parallel() + tmpDir := t.TempDir() nestedDir := filepath.Join(tmpDir, "nested", "dir") - lock, err := Acquire(nestedDir) + lock, err := pidlock.Acquire(nestedDir) require.NoError(t, err) require.NotNil(t, lock) diff --git a/internal/s3/client.go b/internal/s3/client.go index 1079b8f..f043ba3 100644 --- a/internal/s3/client.go +++ b/internal/s3/client.go @@ -1,3 +1,5 @@ +// Package s3 wraps the AWS S3 SDK with a simplified client for vaultik's +// bucket-and-prefix scoped object operations. package s3 import ( @@ -42,7 +44,7 @@ type Config struct { // Used to suppress SDK warnings about checksums. type nopLogger struct{} -func (nopLogger) Logf(classification logging.Classification, format string, v ...any) {} +func (nopLogger) Logf(_ logging.Classification, _ string, _ ...any) {} // NewClient creates a new S3 client with the provided configuration. // It establishes a connection to the S3-compatible storage service and @@ -105,13 +107,18 @@ type ProgressCallback func(bytesUploaded int64) error // The size parameter must be the exact size of the data to upload. // The progress callback is called periodically with the number of bytes uploaded. // Returns an error if the upload fails. -func (c *Client) PutObjectWithProgress(ctx context.Context, key string, data io.Reader, size int64, progress ProgressCallback) error { +func (c *Client) PutObjectWithProgress( + ctx context.Context, key string, data io.Reader, + size int64, progress ProgressCallback, +) error { fullKey := c.prefix + key + // uploadPartSize is 10MB for better progress granularity. + const uploadPartSize = 10 * 1024 * 1024 + // Create an uploader with the S3 client uploader := manager.NewUploader(c.s3Client, func(u *manager.Uploader) { - // Set part size to 10MB for better progress granularity - u.PartSize = 10 * 1024 * 1024 + u.PartSize = uploadPartSize }) // Create a progress reader that tracks upload progress @@ -241,7 +248,9 @@ type ObjectInfo struct { // listing is complete or an error occurs. If an error occurs, it will be // sent as the last item with the Err field set. The recursive parameter // is currently unused but reserved for future use. -func (c *Client) ListObjectsStream(ctx context.Context, prefix string, recursive bool) <-chan ObjectInfo { +func (c *Client) ListObjectsStream( + ctx context.Context, prefix string, _ bool, +) <-chan ObjectInfo { ch := make(chan ObjectInfo) go func() { diff --git a/internal/s3/client_test.go b/internal/s3/client_test.go index 7ebab27..81f986f 100644 --- a/internal/s3/client_test.go +++ b/internal/s3/client_test.go @@ -9,6 +9,7 @@ import ( "sneak.berlin/go/vaultik/internal/s3" ) +//nolint:paralleltest // test servers share a fixed localhost port func TestClient(t *testing.T) { ts := NewTestServer(t) defer func() { @@ -33,11 +34,21 @@ func TestClient(t *testing.T) { t.Fatalf("failed to create client: %v", err) } - // Test PutObject testKey := "foo/bar.txt" testData := []byte("test data") - err = client.PutObject(ctx, testKey, bytes.NewReader(testData)) + verifyPutGetHead(ctx, t, client, testKey, testData) + verifyListAndDelete(ctx, t, client, testKey) +} + +// verifyPutGetHead uploads an object, reads it back, and checks existence. +func verifyPutGetHead( + ctx context.Context, t *testing.T, client *s3.Client, + testKey string, testData []byte, +) { + t.Helper() + + err := client.PutObject(ctx, testKey, bytes.NewReader(testData)) if err != nil { t.Fatalf("failed to put object: %v", err) } @@ -72,8 +83,15 @@ func TestClient(t *testing.T) { if !exists { t.Error("expected object to exist") } +} + +// verifyListAndDelete lists the object's prefix, deletes it, and checks +// it is gone. +func verifyListAndDelete( + ctx context.Context, t *testing.T, client *s3.Client, testKey string, +) { + t.Helper() - // Test ListObjects keys, err := client.ListObjects(ctx, "foo/") if err != nil { t.Fatalf("failed to list objects: %v", err) @@ -94,7 +112,7 @@ func TestClient(t *testing.T) { } // Verify deletion - exists, err = client.HeadObject(ctx, testKey) + exists, err := client.HeadObject(ctx, testKey) if err != nil { t.Fatalf("failed to head object after deletion: %v", err) } diff --git a/internal/s3/module.go b/internal/s3/module.go index 8839ec7..33b24ba 100644 --- a/internal/s3/module.go +++ b/internal/s3/module.go @@ -10,6 +10,8 @@ import ( // Module exports S3 functionality as an fx module. // It provides automatic dependency injection for the S3 client, // configuring it based on the application's configuration settings. +// +//nolint:gochecknoglobals // fx module definitions are package globals var Module = fx.Module("s3", fx.Provide( provideClient, @@ -32,7 +34,7 @@ func provideClient(lc fx.Lifecycle, cfg *config.Config) (*Client, error) { } lc.Append(fx.Hook{ - OnStop: func(ctx context.Context) error { + OnStop: func(_ context.Context) error { // S3 client doesn't need explicit cleanup return nil }, diff --git a/internal/s3/s3_test.go b/internal/s3/s3_test.go index 57fca9a..38dc911 100644 --- a/internal/s3/s3_test.go +++ b/internal/s3/s3_test.go @@ -3,10 +3,10 @@ package s3_test import ( "bytes" "context" + "errors" "fmt" "io" "net/http" - "os" "path/filepath" "testing" "time" @@ -37,13 +37,16 @@ type TestServer struct { logBuf *bytes.Buffer } +// testServerReadHeaderTimeout bounds header reads on the in-process +// test server (gosec G112). +const testServerReadHeaderTimeout = 5 * time.Second + // NewTestServer creates and starts a new test server func NewTestServer(t *testing.T) *TestServer { + t.Helper() + // Create temp directory for any file operations - tempDir, err := os.MkdirTemp("", "vaultik-s3-test-*") - if err != nil { - t.Fatalf("failed to create temp dir: %v", err) - } + tempDir := t.TempDir() // Create in-memory backend backend := s3mem.New() @@ -51,14 +54,15 @@ func NewTestServer(t *testing.T) *TestServer { // Create HTTP server server := &http.Server{ - Addr: "localhost:9999", - Handler: faker.Server(), + Addr: "localhost:9999", + Handler: faker.Server(), + ReadHeaderTimeout: testServerReadHeaderTimeout, } // Start server in background go func() { err := server.ListenAndServe() - if err != nil && err != http.ErrServerClosed { + if err != nil && !errors.Is(err, http.ErrServerClosed) { t.Logf("test server error: %v", err) } }() @@ -70,6 +74,14 @@ func NewTestServer(t *testing.T) *TestServer { logBuf := &bytes.Buffer{} // Create S3 client with custom logger + logFn := func(classification logging.Classification, format string, v ...any) { + // Capture logs to buffer instead of stdout + fmt.Fprintf(logBuf, "SDK %s %s %s\n", + time.Now().Format("2006/01/02 15:04:05"), + string(classification), + fmt.Sprintf(format, v...)) + } + cfg, err := config.LoadDefaultConfig(context.Background(), config.WithRegion(testRegion), config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider( @@ -77,14 +89,9 @@ func NewTestServer(t *testing.T) *TestServer { testSecretKey, "", )), - config.WithClientLogMode(aws.LogRetries|aws.LogRequestWithBody|aws.LogResponseWithBody), - config.WithLogger(logging.LoggerFunc(func(classification logging.Classification, format string, v ...any) { - // Capture logs to buffer instead of stdout - fmt.Fprintf(logBuf, "SDK %s %s %s\n", - time.Now().Format("2006/01/02 15:04:05"), - string(classification), - fmt.Sprintf(format, v...)) - })), + config.WithClientLogMode( + aws.LogRetries|aws.LogRequestWithBody|aws.LogResponseWithBody), + config.WithLogger(logging.LoggerFunc(logFn)), ) if err != nil { t.Fatalf("failed to create AWS config: %v", err) @@ -121,17 +128,13 @@ func NewTestServer(t *testing.T) *TestServer { return ts } -// Cleanup shuts down the server and removes temp directory +// Cleanup shuts down the server. The temp directory is removed +// automatically by t.TempDir. func (ts *TestServer) Cleanup() error { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - err := ts.server.Shutdown(ctx) - if err != nil { - return err - } - - return os.RemoveAll(ts.tempDir) + return ts.server.Shutdown(ctx) } // Client returns the S3 client configured for the test server @@ -140,6 +143,8 @@ func (ts *TestServer) Client() *s3.Client { } // TestBasicS3Operations tests basic store and retrieve operations +// +//nolint:paralleltest // test servers share a fixed localhost port func TestBasicS3Operations(t *testing.T) { ts := NewTestServer(t) defer func() { @@ -193,6 +198,8 @@ func TestBasicS3Operations(t *testing.T) { } // TestBlobOperations tests blob storage patterns for vaultik +// +//nolint:paralleltest // test servers share a fixed localhost port func TestBlobOperations(t *testing.T) { ts := NewTestServer(t) defer func() { @@ -257,6 +264,8 @@ func TestBlobOperations(t *testing.T) { } // TestMetadataOperations tests metadata storage patterns +// +//nolint:paralleltest // test servers share a fixed localhost port func TestMetadataOperations(t *testing.T) { ts := NewTestServer(t) defer func() { @@ -286,7 +295,8 @@ func TestMetadataOperations(t *testing.T) { // Store manifest manifestKey := filepath.Join("metadata", snapshotID+".manifest.json.zst") - manifestData := []byte(`{"snapshot_id":"2024-01-01T12:00:00Z","blob_hashes":["hash1","hash2"]}`) + manifestData := []byte(`{"snapshot_id":"2024-01-01T12:00:00Z",` + + `"blob_hashes":["hash1","hash2"]}`) _, err = client.PutObject(ctx, &s3.PutObjectInput{ Bucket: aws.String(testBucket), diff --git a/internal/snapshot/backup_test.go b/internal/snapshot/backup_test.go index b310fe7..a6e2a80 100644 --- a/internal/snapshot/backup_test.go +++ b/internal/snapshot/backup_test.go @@ -1,4 +1,4 @@ -package snapshot +package snapshot_test import ( "context" @@ -19,6 +19,12 @@ import ( "sneak.berlin/go/vaultik/internal/types" ) +// errBlobNotFound is returned by the mock S3 client for unknown hashes. +var errBlobNotFound = errors.New("blob not found") + +// testFile1Name is the shared fixture filename used across backup tests. +const testFile1Name = "file1.txt" + // MockS3Client is a mock implementation of S3 operations for testing type MockS3Client struct { storage map[string][]byte @@ -30,39 +36,149 @@ func NewMockS3Client() *MockS3Client { } } -func (m *MockS3Client) PutBlob(ctx context.Context, hash string, data []byte) error { +func (m *MockS3Client) PutBlob(_ context.Context, hash string, data []byte) error { m.storage[hash] = data return nil } -func (m *MockS3Client) GetBlob(ctx context.Context, hash string) ([]byte, error) { +func (m *MockS3Client) GetBlob(_ context.Context, hash string) ([]byte, error) { data, ok := m.storage[hash] if !ok { - return nil, fmt.Errorf("blob not found: %s", hash) + return nil, fmt.Errorf("%w: %s", errBlobNotFound, hash) } return data, nil } -func (m *MockS3Client) BlobExists(ctx context.Context, hash string) (bool, error) { +func (m *MockS3Client) BlobExists(_ context.Context, hash string) (bool, error) { _, ok := m.storage[hash] return ok, nil } -func (m *MockS3Client) CreateBucket(ctx context.Context, bucket string) error { +func (m *MockS3Client) CreateBucket(_ context.Context, _ string) error { return nil } +// verifyBackupFiles checks the file records created by a backup against +// the fixture filesystem. +func verifyBackupFiles( + ctx context.Context, + t *testing.T, + repos *database.Repositories, + testFS fstest.MapFS, +) { + t.Helper() + + files, err := repos.Files.ListByPrefix(ctx, "") + if err != nil { + t.Fatalf("Failed to list files: %v", err) + } + + expectedFiles := map[string]bool{ + testFile1Name: true, + "dir1/file2.txt": true, + "dir1/subdir/file3.txt": true, + "largefile.bin": true, + } + + if len(files) != len(expectedFiles) { + t.Errorf("Expected %d files, got %d", len(expectedFiles), len(files)) + } + + for _, file := range files { + if !expectedFiles[file.Path.String()] { + t.Errorf("Unexpected file in database: %s", file.Path) + } + + delete(expectedFiles, file.Path.String()) + + // Verify file metadata + fsFile := testFS[file.Path.String()] + if fsFile == nil { + t.Errorf("File %s not found in test filesystem", file.Path) + + continue + } + + if file.Size != int64(len(fsFile.Data)) { + t.Errorf("File %s: expected size %d, got %d", + file.Path, len(fsFile.Data), file.Size) + } + + if file.Mode != uint32(fsFile.Mode) { + t.Errorf("File %s: expected mode %o, got %o", + file.Path, fsFile.Mode, file.Mode) + } + } + + if len(expectedFiles) > 0 { + t.Errorf("Files not found in database: %v", expectedFiles) + } +} + +// verifyBackupChunksAndBlobs checks that chunking produced the expected +// records and every referenced blob exists in the mock S3 store. +func verifyBackupChunksAndBlobs( + ctx context.Context, + t *testing.T, + repos *database.Repositories, + s3Client *MockS3Client, + snapshotID string, +) { + t.Helper() + + chunks, err := repos.Chunks.List(ctx) + if err != nil { + t.Fatalf("Failed to list chunks: %v", err) + } + + if len(chunks) == 0 { + t.Error("No chunks found in database") + } + + // The large file should create 10 chunks (10MB / 1MB chunk size) + // Plus the small files + minExpectedChunks := 10 + 3 + if len(chunks) < minExpectedChunks { + t.Errorf("Expected at least %d chunks, got %d", minExpectedChunks, len(chunks)) + } + + // Verify at least one blob was created and uploaded + // We can't list blobs directly, but we can check via snapshot blobs + blobHashes, err := repos.Snapshots.GetBlobHashes(ctx, snapshotID) + if err != nil { + t.Fatalf("Failed to get blob hashes: %v", err) + } + + if len(blobHashes) == 0 { + t.Error("Expected at least one blob to be created") + } + + for _, blobHash := range blobHashes { + // Check blob exists in mock S3 + exists, err := s3Client.BlobExists(ctx, blobHash) + if err != nil { + t.Errorf("Failed to check blob %s: %v", blobHash, err) + } + + if !exists { + t.Errorf("Blob %s not found in S3", blobHash) + } + } +} + func TestBackupWithInMemoryFS(t *testing.T) { + t.Parallel() + // Create a temporary directory for the database tempDir := t.TempDir() dbPath := filepath.Join(tempDir, "test.db") // Create test filesystem testFS := fstest.MapFS{ - "file1.txt": &fstest.MapFile{ + testFile1Name: &fstest.MapFile{ Data: []byte("Hello, World!"), Mode: 0644, ModTime: time.Now(), @@ -129,100 +245,21 @@ func TestBackupWithInMemoryFS(t *testing.T) { t.Error("Expected snapshot to have files") } - // Verify files in database - files, err := repos.Files.ListByPrefix(ctx, "") - if err != nil { - t.Fatalf("Failed to list files: %v", err) - } - - expectedFiles := map[string]bool{ - "file1.txt": true, - "dir1/file2.txt": true, - "dir1/subdir/file3.txt": true, - "largefile.bin": true, - } - - if len(files) != len(expectedFiles) { - t.Errorf("Expected %d files, got %d", len(expectedFiles), len(files)) - } - - for _, file := range files { - if !expectedFiles[file.Path.String()] { - t.Errorf("Unexpected file in database: %s", file.Path) - } - - delete(expectedFiles, file.Path.String()) - - // Verify file metadata - fsFile := testFS[file.Path.String()] - if fsFile == nil { - t.Errorf("File %s not found in test filesystem", file.Path) - - continue - } - - if file.Size != int64(len(fsFile.Data)) { - t.Errorf("File %s: expected size %d, got %d", file.Path, len(fsFile.Data), file.Size) - } - - if file.Mode != uint32(fsFile.Mode) { - t.Errorf("File %s: expected mode %o, got %o", file.Path, fsFile.Mode, file.Mode) - } - } - - if len(expectedFiles) > 0 { - t.Errorf("Files not found in database: %v", expectedFiles) - } - - // Verify chunks - chunks, err := repos.Chunks.List(ctx) - if err != nil { - t.Fatalf("Failed to list chunks: %v", err) - } - - if len(chunks) == 0 { - t.Error("No chunks found in database") - } - - // The large file should create 10 chunks (10MB / 1MB chunk size) - // Plus the small files - minExpectedChunks := 10 + 3 - if len(chunks) < minExpectedChunks { - t.Errorf("Expected at least %d chunks, got %d", minExpectedChunks, len(chunks)) - } - - // Verify at least one blob was created and uploaded - // We can't list blobs directly, but we can check via snapshot blobs - blobHashes, err := repos.Snapshots.GetBlobHashes(ctx, snapshotID) - if err != nil { - t.Fatalf("Failed to get blob hashes: %v", err) - } - - if len(blobHashes) == 0 { - t.Error("Expected at least one blob to be created") - } - - for _, blobHash := range blobHashes { - // Check blob exists in mock S3 - exists, err := s3Client.BlobExists(ctx, blobHash) - if err != nil { - t.Errorf("Failed to check blob %s: %v", blobHash, err) - } - - if !exists { - t.Errorf("Blob %s not found in S3", blobHash) - } - } + // Verify files, chunks, and blob records + verifyBackupFiles(ctx, t, repos, testFS) + verifyBackupChunksAndBlobs(ctx, t, repos, s3Client, snapshotID) } func TestBackupDeduplication(t *testing.T) { + t.Parallel() + // Create a temporary directory for the database tempDir := t.TempDir() dbPath := filepath.Join(tempDir, "test.db") // Create test filesystem with duplicate content testFS := fstest.MapFS{ - "file1.txt": &fstest.MapFile{ + testFile1Name: &fstest.MapFile{ Data: []byte("Duplicate content"), Mode: 0644, ModTime: time.Now(), @@ -290,7 +327,8 @@ func TestBackupDeduplication(t *testing.T) { // The duplicate content chunk should be referenced by 2 files if chunk.Size == int64(len("Duplicate content")) && len(files) != 2 { - t.Errorf("Expected duplicate chunk to be referenced by 2 files, got %d", len(files)) + t.Errorf("Expected duplicate chunk to be referenced by 2 files, got %d", + len(files)) } } } @@ -304,8 +342,19 @@ type BackupEngine struct { } } +// backupCounters accumulates statistics across a test backup run. +type backupCounters struct { + fileCount int64 + chunkCount int64 + blobCount int64 + totalSize int64 + blobSize int64 +} + // Backup performs a backup of the given filesystem -func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (string, error) { +func (b *BackupEngine) Backup( + ctx context.Context, fsys fs.FS, root string, +) (string, error) { // Create a new snapshot hostname, _ := os.Hostname() snapshotID := time.Now().Format(time.RFC3339) @@ -325,8 +374,7 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str return "", err } - // Track counters - var fileCount, chunkCount, blobCount, totalSize, blobSize int64 + counters := &backupCounters{} // Track which chunks we've seen to handle deduplication processedChunks := make(map[string]bool) @@ -354,122 +402,170 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str return nil } - // Create file record in a short transaction - file := &database.File{ - Path: types.FilePath(path), - Size: info.Size(), - Mode: uint32(info.Mode()), - MTime: info.ModTime(), - UID: 1000, // Default UID for test - GID: 1000, // Default GID for test - } - - err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { - return b.repos.Files.Create(ctx, tx, file) - }) - if err != nil { - return err - } - - fileCount++ - totalSize += info.Size() - - // Read and process file in chunks - f, err := fsys.Open(path) - if err != nil { - return err - } - defer func() { - err := f.Close() - if err != nil { - // Log but don't fail since we're already in an error path potentially - fmt.Fprintf(os.Stderr, "Failed to close file: %v\n", err) - } - }() - - // Process file in chunks - chunkIndex := 0 - buffer := make([]byte, defaultChunkSize) - - for { - n, err := f.Read(buffer) - if err != nil && !errors.Is(err, io.EOF) { - return err - } - - if n == 0 { - break - } - - chunkData := buffer[:n] - chunkHash := calculateHash(chunkData) - - // Check if chunk already exists (outside of transaction) - existingChunk, _ := b.repos.Chunks.GetByHash(ctx, chunkHash) - if existingChunk == nil { - // Create new chunk in a short transaction - err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { - chunk := &database.Chunk{ - ChunkHash: types.ChunkHash(chunkHash), - Size: int64(n), - } - - return b.repos.Chunks.Create(ctx, tx, chunk) - }) - if err != nil { - return err - } - - processedChunks[chunkHash] = true - } - - // Create file-chunk mapping in a short transaction - err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { - fileChunk := &database.FileChunk{ - FileID: file.ID, - Idx: chunkIndex, - ChunkHash: types.ChunkHash(chunkHash), - } - - return b.repos.FileChunks.Create(ctx, tx, fileChunk) - }) - if err != nil { - return err - } - - // Create chunk-file mapping in a short transaction - err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { - chunkFile := &database.ChunkFile{ - ChunkHash: types.ChunkHash(chunkHash), - FileID: file.ID, - FileOffset: int64(chunkIndex * defaultChunkSize), - Length: int64(n), - } - - return b.repos.ChunkFiles.Create(ctx, tx, chunkFile) - }) - if err != nil { - return err - } - - chunkIndex++ - } - - return nil + return b.backupOneFile(ctx, fsys, path, info, processedChunks, counters) }) if err != nil { return "", err } // After all files are processed, create blobs for new chunks + err = b.createBlobsForChunks(ctx, snapshotID, processedChunks, counters) + if err != nil { + return "", err + } + + // Update snapshot with final counts + err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { + return b.repos.Snapshots.UpdateCounts(ctx, tx, snapshotID, + counters.fileCount, counters.chunkCount, counters.blobCount, + counters.totalSize, counters.blobSize) + }) + if err != nil { + return "", err + } + + return snapshotID, nil +} + +// backupOneFile records a single regular file and its chunks. +func (b *BackupEngine) backupOneFile( + ctx context.Context, + fsys fs.FS, + path string, + info fs.FileInfo, + processedChunks map[string]bool, + counters *backupCounters, +) error { + // Create file record in a short transaction + file := &database.File{ + Path: types.FilePath(path), + Size: info.Size(), + Mode: uint32(info.Mode()), + MTime: info.ModTime(), + UID: 1000, // Default UID for test + GID: 1000, // Default GID for test + } + + err := b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { + return b.repos.Files.Create(ctx, tx, file) + }) + if err != nil { + return err + } + + counters.fileCount++ + counters.totalSize += info.Size() + + // Read and process file in chunks + f, err := fsys.Open(path) + if err != nil { + return err + } + defer func() { + err := f.Close() + if err != nil { + // Log but don't fail since we're already in an error path potentially + fmt.Fprintf(os.Stderr, "Failed to close file: %v\n", err) + } + }() + + // Process file in chunks + chunkIndex := 0 + buffer := make([]byte, defaultChunkSize) + + for { + n, err := f.Read(buffer) + if err != nil && !errors.Is(err, io.EOF) { + return err + } + + if n == 0 { + break + } + + err = b.recordChunk(ctx, file, chunkIndex, buffer[:n], processedChunks) + if err != nil { + return err + } + + chunkIndex++ + } + + return nil +} + +// recordChunk creates the chunk record (if new) and its file associations. +func (b *BackupEngine) recordChunk( + ctx context.Context, + file *database.File, + chunkIndex int, + chunkData []byte, + processedChunks map[string]bool, +) error { + chunkHash := calculateHash(chunkData) + + // Check if chunk already exists (outside of transaction) + existingChunk, _ := b.repos.Chunks.GetByHash(ctx, chunkHash) + if existingChunk == nil { + // Create new chunk in a short transaction + err := b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { + chunk := &database.Chunk{ + ChunkHash: types.ChunkHash(chunkHash), + Size: int64(len(chunkData)), + } + + return b.repos.Chunks.Create(ctx, tx, chunk) + }) + if err != nil { + return err + } + + processedChunks[chunkHash] = true + } + + // Create file-chunk mapping in a short transaction + err := b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { + fileChunk := &database.FileChunk{ + FileID: file.ID, + Idx: chunkIndex, + ChunkHash: types.ChunkHash(chunkHash), + } + + return b.repos.FileChunks.Create(ctx, tx, fileChunk) + }) + if err != nil { + return err + } + + // Create chunk-file mapping in a short transaction + return b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { + chunkFile := &database.ChunkFile{ + ChunkHash: types.ChunkHash(chunkHash), + FileID: file.ID, + FileOffset: int64(chunkIndex * defaultChunkSize), + Length: int64(len(chunkData)), + } + + return b.repos.ChunkFiles.Create(ctx, tx, chunkFile) + }) +} + +// createBlobsForChunks uploads one blob per new chunk and records the blob +// metadata and snapshot association. +func (b *BackupEngine) createBlobsForChunks( + ctx context.Context, + snapshotID string, + processedChunks map[string]bool, + counters *backupCounters, +) error { for chunkHash := range processedChunks { // Get chunk data (outside of transaction) chunk, err := b.repos.Chunks.GetByHash(ctx, chunkHash) if err != nil { - return "", err + return err } - chunkCount++ + counters.chunkCount++ // In a real system, blobs would contain multiple chunks and be encrypted // For testing, we'll create a blob with a "blob-" prefix to differentiate @@ -479,8 +575,9 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str dummyData := []byte(chunkHash) // Upload to S3 as a blob - if err := b.s3Client.PutBlob(ctx, blobHash, dummyData); err != nil { - return "", err + err = b.s3Client.PutBlob(ctx, blobHash, dummyData) + if err != nil { + return err } // Create blob entry in a short transaction @@ -496,11 +593,11 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str return b.repos.Blobs.Create(ctx, tx, blob) }) if err != nil { - return "", err + return err } - blobCount++ - blobSize += chunk.Size + counters.blobCount++ + counters.blobSize += chunk.Size // Create blob-chunk mapping in a short transaction err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { @@ -514,27 +611,20 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str return b.repos.BlobChunks.Create(ctx, tx, blobChunk) }) if err != nil { - return "", err + return err } // Add blob to snapshot in a short transaction err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { - return b.repos.Snapshots.AddBlob(ctx, tx, snapshotID, blobID, types.BlobHash(blobHash)) + return b.repos.Snapshots.AddBlob(ctx, tx, snapshotID, blobID, + types.BlobHash(blobHash)) }) if err != nil { - return "", err + return err } } - // Update snapshot with final counts - err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { - return b.repos.Snapshots.UpdateCounts(ctx, tx, snapshotID, fileCount, chunkCount, blobCount, totalSize, blobSize) - }) - if err != nil { - return "", err - } - - return snapshotID, nil + return nil } func calculateHash(data []byte) string { diff --git a/internal/snapshot/exclude_test.go b/internal/snapshot/exclude_test.go index f908250..9870018 100644 --- a/internal/snapshot/exclude_test.go +++ b/internal/snapshot/exclude_test.go @@ -10,16 +10,15 @@ import ( "github.com/spf13/afero" "github.com/stretchr/testify/require" "sneak.berlin/go/vaultik/internal/database" - "sneak.berlin/go/vaultik/internal/log" "sneak.berlin/go/vaultik/internal/snapshot" "sneak.berlin/go/vaultik/internal/types" ) -func setupExcludeTestFS(t *testing.T) afero.Fs { +func setupExcludeTestFS(t *testing.T) *afero.MemMapFs { t.Helper() // Create in-memory filesystem - fs := afero.NewMemMapFs() + fs := &afero.MemMapFs{} // Create test directory structure: // /backup/ @@ -77,12 +76,11 @@ func setupExcludeTestFS(t *testing.T) afero.Fs { return fs } -func createTestScanner(t *testing.T, fs afero.Fs, excludePatterns []string) (*snapshot.Scanner, *database.Repositories, func()) { +func createTestScanner( + t *testing.T, fs afero.Fs, excludePatterns []string, +) (*snapshot.Scanner, *database.Repositories, func()) { t.Helper() - // Initialize logger - log.Initialize(log.Config{}) - // Create test database db, err := database.NewTestDB() require.NoError(t, err) @@ -95,8 +93,9 @@ func createTestScanner(t *testing.T, fs afero.Fs, excludePatterns []string) (*sn Repositories: repos, MaxBlobSize: 1024 * 1024, CompressionLevel: 3, - AgeRecipients: []string{"age1ql3z7hjy54pw3hyww5ayyfg7zqgvc7w3j2elw8zmrj2kg5sfn9aqmcac8p"}, - Exclude: excludePatterns, + AgeRecipients: []string{ + "age1ql3z7hjy54pw3hyww5ayyfg7zqgvc7w3j2elw8zmrj2kg5sfn9aqmcac8p"}, + Exclude: excludePatterns, }) cleanup := func() { @@ -106,14 +105,16 @@ func createTestScanner(t *testing.T, fs afero.Fs, excludePatterns []string) (*sn return scanner, repos, cleanup } -func createSnapshotRecord(t *testing.T, ctx context.Context, repos *database.Repositories, snapshotID string) { +func createSnapshotRecord( + ctx context.Context, t *testing.T, repos *database.Repositories, snapshotID string, +) { t.Helper() err := repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { snap := &database.Snapshot{ ID: types.SnapshotID(snapshotID), - Hostname: "test-host", - VaultikVersion: "test", + Hostname: testHost, + VaultikVersion: testVersion, StartedAt: time.Now(), CompletedAt: nil, FileCount: 0, @@ -130,6 +131,8 @@ func createSnapshotRecord(t *testing.T, ctx context.Context, repos *database.Rep } func TestExcludePatterns_ExcludeGitDirectory(t *testing.T) { + t.Parallel() + fs := setupExcludeTestFS(t) scanner, repos, cleanup := createTestScanner(t, fs, []string{".git"}) @@ -138,13 +141,14 @@ func TestExcludePatterns_ExcludeGitDirectory(t *testing.T) { require.NotNil(t, scanner) ctx := context.Background() - createSnapshotRecord(t, ctx, repos, "test-snapshot") + createSnapshotRecord(ctx, t, repos, "test-snapshot") result, err := scanner.Scan(ctx, "/backup", "test-snapshot") require.NoError(t, err) // Should have scanned files but NOT .git directory contents - // Expected: file1.txt, file2.log, src/main.go, src/test.go, node_modules/package/index.js, + // Expected: file1.txt, file2.log, src/main.go, src/test.go, + // node_modules/package/index.js, // cache/temp.dat, build/output.bin, docs/readme.md, .DS_Store, thumbs.db, // src/.hidden, important.log.bak // Excluded: .git/config, .git/objects/pack/data.pack @@ -152,6 +156,8 @@ func TestExcludePatterns_ExcludeGitDirectory(t *testing.T) { } func TestExcludePatterns_ExcludeByExtension(t *testing.T) { + t.Parallel() + fs := setupExcludeTestFS(t) scanner, repos, cleanup := createTestScanner(t, fs, []string{"*.log"}) @@ -160,7 +166,7 @@ func TestExcludePatterns_ExcludeByExtension(t *testing.T) { require.NotNil(t, scanner) ctx := context.Background() - createSnapshotRecord(t, ctx, repos, "test-snapshot") + createSnapshotRecord(ctx, t, repos, "test-snapshot") result, err := scanner.Scan(ctx, "/backup", "test-snapshot") require.NoError(t, err) @@ -171,6 +177,8 @@ func TestExcludePatterns_ExcludeByExtension(t *testing.T) { } func TestExcludePatterns_ExcludeNodeModules(t *testing.T) { + t.Parallel() + fs := setupExcludeTestFS(t) scanner, repos, cleanup := createTestScanner(t, fs, []string{"node_modules"}) @@ -179,7 +187,7 @@ func TestExcludePatterns_ExcludeNodeModules(t *testing.T) { require.NotNil(t, scanner) ctx := context.Background() - createSnapshotRecord(t, ctx, repos, "test-snapshot") + createSnapshotRecord(ctx, t, repos, "test-snapshot") result, err := scanner.Scan(ctx, "/backup", "test-snapshot") require.NoError(t, err) @@ -190,25 +198,32 @@ func TestExcludePatterns_ExcludeNodeModules(t *testing.T) { } func TestExcludePatterns_MultiplePatterns(t *testing.T) { + t.Parallel() + fs := setupExcludeTestFS(t) - scanner, repos, cleanup := createTestScanner(t, fs, []string{".git", "node_modules", "*.log", ".DS_Store", "thumbs.db", "cache", "build"}) + scanner, repos, cleanup := createTestScanner(t, fs, []string{ + ".git", "node_modules", "*.log", ".DS_Store", "thumbs.db", "cache", "build"}) defer cleanup() require.NotNil(t, scanner) ctx := context.Background() - createSnapshotRecord(t, ctx, repos, "test-snapshot") + createSnapshotRecord(ctx, t, repos, "test-snapshot") result, err := scanner.Scan(ctx, "/backup", "test-snapshot") require.NoError(t, err) - // Should only have: file1.txt, src/main.go, src/test.go, docs/readme.md, src/.hidden, important.log.bak - // Excluded: .git/*, node_modules/*, *.log (file2.log), .DS_Store, thumbs.db, cache/*, build/* + // Should only have: file1.txt, src/main.go, src/test.go, docs/readme.md, + // src/.hidden, important.log.bak + // Excluded: .git/*, node_modules/*, *.log (file2.log), .DS_Store, + // thumbs.db, cache/*, build/* require.Equal(t, 6, result.FilesScanned, "Should exclude multiple patterns") } func TestExcludePatterns_NoExclusions(t *testing.T) { + t.Parallel() + fs := setupExcludeTestFS(t) scanner, repos, cleanup := createTestScanner(t, fs, []string{}) @@ -217,7 +232,7 @@ func TestExcludePatterns_NoExclusions(t *testing.T) { require.NotNil(t, scanner) ctx := context.Background() - createSnapshotRecord(t, ctx, repos, "test-snapshot") + createSnapshotRecord(ctx, t, repos, "test-snapshot") result, err := scanner.Scan(ctx, "/backup", "test-snapshot") require.NoError(t, err) @@ -227,6 +242,8 @@ func TestExcludePatterns_NoExclusions(t *testing.T) { } func TestExcludePatterns_ExcludeHiddenFiles(t *testing.T) { + t.Parallel() + fs := setupExcludeTestFS(t) scanner, repos, cleanup := createTestScanner(t, fs, []string{".*"}) @@ -235,17 +252,21 @@ func TestExcludePatterns_ExcludeHiddenFiles(t *testing.T) { require.NotNil(t, scanner) ctx := context.Background() - createSnapshotRecord(t, ctx, repos, "test-snapshot") + createSnapshotRecord(ctx, t, repos, "test-snapshot") result, err := scanner.Scan(ctx, "/backup", "test-snapshot") require.NoError(t, err) // Should exclude: .git/*, .DS_Store, src/.hidden - // Total files: 14, excluded: 4 (.git/config, .git/objects/pack/data.pack, .DS_Store, src/.hidden) - require.Equal(t, 10, result.FilesScanned, "Should exclude hidden files and directories") + // Total files: 14, excluded: 4 (.git/config, + // .git/objects/pack/data.pack, .DS_Store, src/.hidden) + require.Equal(t, 10, result.FilesScanned, + "Should exclude hidden files and directories") } func TestExcludePatterns_DoubleStarGlob(t *testing.T) { + t.Parallel() + fs := setupExcludeTestFS(t) scanner, repos, cleanup := createTestScanner(t, fs, []string{"**/*.pack"}) @@ -254,7 +275,7 @@ func TestExcludePatterns_DoubleStarGlob(t *testing.T) { require.NotNil(t, scanner) ctx := context.Background() - createSnapshotRecord(t, ctx, repos, "test-snapshot") + createSnapshotRecord(ctx, t, repos, "test-snapshot") result, err := scanner.Scan(ctx, "/backup", "test-snapshot") require.NoError(t, err) @@ -265,6 +286,8 @@ func TestExcludePatterns_DoubleStarGlob(t *testing.T) { } func TestExcludePatterns_ExactFileName(t *testing.T) { + t.Parallel() + fs := setupExcludeTestFS(t) scanner, repos, cleanup := createTestScanner(t, fs, []string{"thumbs.db", ".DS_Store"}) @@ -273,7 +296,7 @@ func TestExcludePatterns_ExactFileName(t *testing.T) { require.NotNil(t, scanner) ctx := context.Background() - createSnapshotRecord(t, ctx, repos, "test-snapshot") + createSnapshotRecord(ctx, t, repos, "test-snapshot") result, err := scanner.Scan(ctx, "/backup", "test-snapshot") require.NoError(t, err) @@ -284,6 +307,8 @@ func TestExcludePatterns_ExactFileName(t *testing.T) { } func TestExcludePatterns_CaseSensitive(t *testing.T) { + t.Parallel() + // Pattern matching should be case-sensitive fs := setupExcludeTestFS(t) @@ -293,7 +318,7 @@ func TestExcludePatterns_CaseSensitive(t *testing.T) { require.NotNil(t, scanner) ctx := context.Background() - createSnapshotRecord(t, ctx, repos, "test-snapshot") + createSnapshotRecord(ctx, t, repos, "test-snapshot") result, err := scanner.Scan(ctx, "/backup", "test-snapshot") require.NoError(t, err) @@ -304,6 +329,8 @@ func TestExcludePatterns_CaseSensitive(t *testing.T) { } func TestExcludePatterns_DirectoryWithTrailingSlash(t *testing.T) { + t.Parallel() + fs := setupExcludeTestFS(t) // Some users might add trailing slashes to directory patterns scanner, repos, cleanup := createTestScanner(t, fs, []string{"cache/", "build/"}) @@ -312,17 +339,20 @@ func TestExcludePatterns_DirectoryWithTrailingSlash(t *testing.T) { require.NotNil(t, scanner) ctx := context.Background() - createSnapshotRecord(t, ctx, repos, "test-snapshot") + createSnapshotRecord(ctx, t, repos, "test-snapshot") result, err := scanner.Scan(ctx, "/backup", "test-snapshot") require.NoError(t, err) // Should exclude cache/temp.dat and build/output.bin // Total files: 14, excluded: 2 - require.Equal(t, 12, result.FilesScanned, "Should handle directory patterns with trailing slashes") + require.Equal(t, 12, result.FilesScanned, + "Should handle directory patterns with trailing slashes") } func TestExcludePatterns_PatternInSubdirectory(t *testing.T) { + t.Parallel() + fs := setupExcludeTestFS(t) // Exclude .hidden file specifically in src directory scanner, repos, cleanup := createTestScanner(t, fs, []string{"src/.hidden"}) @@ -331,7 +361,7 @@ func TestExcludePatterns_PatternInSubdirectory(t *testing.T) { require.NotNil(t, scanner) ctx := context.Background() - createSnapshotRecord(t, ctx, repos, "test-snapshot") + createSnapshotRecord(ctx, t, repos, "test-snapshot") result, err := scanner.Scan(ctx, "/backup", "test-snapshot") require.NoError(t, err) @@ -350,13 +380,14 @@ func TestExcludePatterns_PatternInSubdirectory(t *testing.T) { // file.txt (should be excluded with /projectname) // otherproject/ // projectname/ -// file.txt (should NOT be excluded with /projectname, only with projectname) +// file.txt (should NOT be excluded with /projectname, +// only with projectname) // src/ // file.go -func setupAnchoredTestFS(t *testing.T) afero.Fs { +func setupAnchoredTestFS(t *testing.T) *afero.MemMapFs { t.Helper() - fs := afero.NewMemMapFs() + fs := &afero.MemMapFs{} files := map[string]string{ "/backup/projectname/file.txt": "root project file", @@ -381,6 +412,8 @@ func setupAnchoredTestFS(t *testing.T) afero.Fs { } func TestExcludePatterns_AnchoredPattern(t *testing.T) { + t.Parallel() + // Pattern starting with / should only match from root of source dir fs := setupAnchoredTestFS(t) @@ -390,7 +423,7 @@ func TestExcludePatterns_AnchoredPattern(t *testing.T) { require.NotNil(t, scanner) ctx := context.Background() - createSnapshotRecord(t, ctx, repos, "test-snapshot") + createSnapshotRecord(ctx, t, repos, "test-snapshot") result, err := scanner.Scan(ctx, "/backup", "test-snapshot") require.NoError(t, err) @@ -398,10 +431,13 @@ func TestExcludePatterns_AnchoredPattern(t *testing.T) { // /projectname should ONLY exclude /backup/projectname/file.txt (1 file) // /backup/otherproject/projectname/file.txt should NOT be excluded // Total files: 4, excluded: 1 - require.Equal(t, 3, result.FilesScanned, "Anchored pattern /projectname should only match at root of source dir") + require.Equal(t, 3, result.FilesScanned, + "Anchored pattern /projectname should only match at root of source dir") } func TestExcludePatterns_UnanchoredPattern(t *testing.T) { + t.Parallel() + // Pattern without leading / should match anywhere in path fs := setupAnchoredTestFS(t) @@ -411,7 +447,7 @@ func TestExcludePatterns_UnanchoredPattern(t *testing.T) { require.NotNil(t, scanner) ctx := context.Background() - createSnapshotRecord(t, ctx, repos, "test-snapshot") + createSnapshotRecord(ctx, t, repos, "test-snapshot") result, err := scanner.Scan(ctx, "/backup", "test-snapshot") require.NoError(t, err) @@ -420,10 +456,13 @@ func TestExcludePatterns_UnanchoredPattern(t *testing.T) { // - /backup/projectname/file.txt // - /backup/otherproject/projectname/file.txt // Total files: 4, excluded: 2 - require.Equal(t, 2, result.FilesScanned, "Unanchored pattern should match anywhere in path") + require.Equal(t, 2, result.FilesScanned, + "Unanchored pattern should match anywhere in path") } func TestExcludePatterns_AnchoredPatternWithGlob(t *testing.T) { + t.Parallel() + // Anchored pattern with glob fs := setupAnchoredTestFS(t) @@ -433,7 +472,7 @@ func TestExcludePatterns_AnchoredPatternWithGlob(t *testing.T) { require.NotNil(t, scanner) ctx := context.Background() - createSnapshotRecord(t, ctx, repos, "test-snapshot") + createSnapshotRecord(ctx, t, repos, "test-snapshot") result, err := scanner.Scan(ctx, "/backup", "test-snapshot") require.NoError(t, err) @@ -444,6 +483,8 @@ func TestExcludePatterns_AnchoredPatternWithGlob(t *testing.T) { } func TestExcludePatterns_AnchoredPatternFile(t *testing.T) { + t.Parallel() + // Anchored pattern for exact file at root fs := setupAnchoredTestFS(t) @@ -453,7 +494,7 @@ func TestExcludePatterns_AnchoredPatternFile(t *testing.T) { require.NotNil(t, scanner) ctx := context.Background() - createSnapshotRecord(t, ctx, repos, "test-snapshot") + createSnapshotRecord(ctx, t, repos, "test-snapshot") result, err := scanner.Scan(ctx, "/backup", "test-snapshot") require.NoError(t, err) @@ -461,10 +502,13 @@ func TestExcludePatterns_AnchoredPatternFile(t *testing.T) { // /file.txt should ONLY exclude /backup/file.txt // NOT /backup/projectname/file.txt or /backup/otherproject/projectname/file.txt // Total files: 4, excluded: 1 - require.Equal(t, 3, result.FilesScanned, "Anchored pattern for file should only match at root") + require.Equal(t, 3, result.FilesScanned, + "Anchored pattern for file should only match at root") } func TestExcludePatterns_UnanchoredPatternFile(t *testing.T) { + t.Parallel() + // Unanchored pattern for file should match anywhere fs := setupAnchoredTestFS(t) @@ -474,7 +518,7 @@ func TestExcludePatterns_UnanchoredPatternFile(t *testing.T) { require.NotNil(t, scanner) ctx := context.Background() - createSnapshotRecord(t, ctx, repos, "test-snapshot") + createSnapshotRecord(ctx, t, repos, "test-snapshot") result, err := scanner.Scan(ctx, "/backup", "test-snapshot") require.NoError(t, err) @@ -484,5 +528,6 @@ func TestExcludePatterns_UnanchoredPatternFile(t *testing.T) { // - /backup/projectname/file.txt // - /backup/otherproject/projectname/file.txt // Total files: 4, excluded: 3 - require.Equal(t, 1, result.FilesScanned, "Unanchored pattern for file should match anywhere") + require.Equal(t, 1, result.FilesScanned, + "Unanchored pattern for file should match anywhere") } diff --git a/internal/snapshot/file_change_test.go b/internal/snapshot/file_change_test.go index a918010..501b69b 100644 --- a/internal/snapshot/file_change_test.go +++ b/internal/snapshot/file_change_test.go @@ -2,7 +2,6 @@ package snapshot_test import ( "context" - "database/sql" "testing" "time" @@ -15,11 +14,55 @@ import ( "sneak.berlin/go/vaultik/internal/types" ) +// verifyChunkChange checks that after a content change the file references +// the new chunk, the old chunk still exists, and the old chunk no longer +// maps to the modified file. +func verifyChunkChange( + ctx context.Context, + t *testing.T, + repos *database.Repositories, + oldChunkHash, newChunkHash types.ChunkHash, +) { + t.Helper() + + // Verify the chunk hashes are different + assert.NotEqual(t, oldChunkHash, newChunkHash, + "Chunk hash should change when content changes") + + // Get chunk files from second scan + chunkFiles2, err := repos.ChunkFiles.GetByFilePath(ctx, "/test.txt") + require.NoError(t, err) + assert.Len(t, chunkFiles2, 1) + assert.Equal(t, newChunkHash, chunkFiles2[0].ChunkHash) + + // Verify old chunk still exists (it's still valid data) + oldChunk, err := repos.Chunks.GetByHash(ctx, oldChunkHash.String()) + require.NoError(t, err) + assert.NotNil(t, oldChunk) + + // Verify new chunk exists + newChunk, err := repos.Chunks.GetByHash(ctx, newChunkHash.String()) + require.NoError(t, err) + assert.NotNil(t, newChunk) + + // Verify that chunk_files for old chunk no longer references this file + oldChunkFiles, err := repos.ChunkFiles.GetByChunkHash(ctx, oldChunkHash) + require.NoError(t, err) + + for _, cf := range oldChunkFiles { + file, err := repos.Files.GetByID(ctx, cf.FileID) + require.NoError(t, err) + assert.NotEqual(t, "/data/test.txt", file.Path, + "Old chunk should not be associated with the modified file") + } +} + // TestFileContentChange verifies that when a file's content changes, // the old chunks are properly disassociated func TestFileContentChange(t *testing.T) { // Initialize logger for tests log.Initialize(log.Config{}) + t.Parallel() // Create in-memory filesystem fs := afero.NewMemMapFs() @@ -48,23 +91,13 @@ func TestFileContentChange(t *testing.T) { Repositories: repos, MaxBlobSize: int64(1024 * 1024), // 1MB blobs CompressionLevel: 3, - AgeRecipients: []string{"age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05gl0sjq9q9wjg"}, // Test public key + AgeRecipients: []string{testAgePublicKey}, }) // Create first snapshot ctx := context.Background() snapshotID1 := "snapshot1" - err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { - snapshot := &database.Snapshot{ - ID: types.SnapshotID(snapshotID1), - Hostname: "test-host", - VaultikVersion: "test", - StartedAt: time.Now(), - } - - return repos.Snapshots.Create(ctx, tx, snapshot) - }) - require.NoError(t, err) + createSnapshotRecord(ctx, t, repos, snapshotID1) // First scan - should create chunks for initial content result1, err := scanner.Scan(ctx, "/", snapshotID1) @@ -85,22 +118,13 @@ func TestFileContentChange(t *testing.T) { // Modify the file time.Sleep(10 * time.Millisecond) // Ensure mtime changes - err = afero.WriteFile(fs, "/test.txt", []byte("Modified content with different data"), 0644) + err = afero.WriteFile(fs, "/test.txt", + []byte("Modified content with different data"), 0644) require.NoError(t, err) // Create second snapshot snapshotID2 := "snapshot2" - err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { - snapshot := &database.Snapshot{ - ID: types.SnapshotID(snapshotID2), - Hostname: "test-host", - VaultikVersion: "test", - StartedAt: time.Now(), - } - - return repos.Snapshots.Create(ctx, tx, snapshot) - }) - require.NoError(t, err) + createSnapshotRecord(ctx, t, repos, snapshotID2) // Second scan - should create new chunks and remove old associations result2, err := scanner.Scan(ctx, "/", snapshotID2) @@ -113,40 +137,14 @@ func TestFileContentChange(t *testing.T) { assert.Len(t, fileChunks2, 1) // Still 1 chunk but different hash newChunkHash := fileChunks2[0].ChunkHash - // Verify the chunk hashes are different - assert.NotEqual(t, oldChunkHash, newChunkHash, "Chunk hash should change when content changes") - - // Get chunk files from second scan - chunkFiles2, err := repos.ChunkFiles.GetByFilePath(ctx, "/test.txt") - require.NoError(t, err) - assert.Len(t, chunkFiles2, 1) - assert.Equal(t, newChunkHash, chunkFiles2[0].ChunkHash) - - // Verify old chunk still exists (it's still valid data) - oldChunk, err := repos.Chunks.GetByHash(ctx, oldChunkHash.String()) - require.NoError(t, err) - assert.NotNil(t, oldChunk) - - // Verify new chunk exists - newChunk, err := repos.Chunks.GetByHash(ctx, newChunkHash.String()) - require.NoError(t, err) - assert.NotNil(t, newChunk) - - // Verify that chunk_files for old chunk no longer references this file - oldChunkFiles, err := repos.ChunkFiles.GetByChunkHash(ctx, oldChunkHash) - require.NoError(t, err) - - for _, cf := range oldChunkFiles { - file, err := repos.Files.GetByID(ctx, cf.FileID) - require.NoError(t, err) - assert.NotEqual(t, "/data/test.txt", file.Path, "Old chunk should not be associated with the modified file") - } + verifyChunkChange(ctx, t, repos, oldChunkHash, newChunkHash) } // TestMultipleFileChanges verifies handling of multiple file changes in one scan func TestMultipleFileChanges(t *testing.T) { // Initialize logger for tests log.Initialize(log.Config{}) + t.Parallel() // Create in-memory filesystem fs := afero.NewMemMapFs() @@ -183,23 +181,13 @@ func TestMultipleFileChanges(t *testing.T) { Repositories: repos, MaxBlobSize: int64(1024 * 1024), // 1MB blobs CompressionLevel: 3, - AgeRecipients: []string{"age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05gl0sjq9q9wjg"}, // Test public key + AgeRecipients: []string{testAgePublicKey}, }) // Create first snapshot ctx := context.Background() snapshotID1 := "snapshot1" - err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { - snapshot := &database.Snapshot{ - ID: types.SnapshotID(snapshotID1), - Hostname: "test-host", - VaultikVersion: "test", - StartedAt: time.Now(), - } - - return repos.Snapshots.Create(ctx, tx, snapshot) - }) - require.NoError(t, err) + createSnapshotRecord(ctx, t, repos, snapshotID1) // First scan result1, err := scanner.Scan(ctx, "/", snapshotID1) @@ -217,17 +205,7 @@ func TestMultipleFileChanges(t *testing.T) { // Create second snapshot snapshotID2 := "snapshot2" - err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { - snapshot := &database.Snapshot{ - ID: types.SnapshotID(snapshotID2), - Hostname: "test-host", - VaultikVersion: "test", - StartedAt: time.Now(), - } - - return repos.Snapshots.Create(ctx, tx, snapshot) - }) - require.NoError(t, err) + createSnapshotRecord(ctx, t, repos, snapshotID2) // Second scan result2, err := scanner.Scan(ctx, "/", snapshotID2) @@ -240,10 +218,12 @@ func TestMultipleFileChanges(t *testing.T) { for path := range files { fileChunks, err := repos.FileChunks.GetByPath(ctx, path) require.NoError(t, err) - assert.Len(t, fileChunks, 1, "File %s should have exactly 1 chunk association", path) + assert.Len(t, fileChunks, 1, + "File %s should have exactly 1 chunk association", path) chunkFiles, err := repos.ChunkFiles.GetByFilePath(ctx, path) require.NoError(t, err) - assert.Len(t, chunkFiles, 1, "File %s should have exactly 1 chunk-file association", path) + assert.Len(t, chunkFiles, 1, + "File %s should have exactly 1 chunk-file association", path) } } diff --git a/internal/snapshot/manifest.go b/internal/snapshot/manifest.go index f6dc3ae..1f3f665 100644 --- a/internal/snapshot/manifest.go +++ b/internal/snapshot/manifest.go @@ -10,6 +10,8 @@ import ( ) // Manifest represents the structure of a snapshot's blob manifest +// +//nolint:tagliatelle // snake_case is the established on-disk manifest format type Manifest struct { SnapshotID string `json:"snapshot_id"` Timestamp string `json:"timestamp"` @@ -19,6 +21,8 @@ type Manifest struct { } // BlobInfo represents information about a single blob in the manifest +// +//nolint:tagliatelle // snake_case is the established on-disk manifest format type BlobInfo struct { Hash string `json:"hash"` CompressedSize int64 `json:"compressed_size"` @@ -35,7 +39,9 @@ func DecodeManifest(r io.Reader) (*Manifest, error) { // Decode JSON manifest var manifest Manifest - if err := json.NewDecoder(zr).Decode(&manifest); err != nil { + + err = json.NewDecoder(zr).Decode(&manifest) + if err != nil { return nil, fmt.Errorf("decoding manifest: %w", err) } @@ -53,18 +59,21 @@ func EncodeManifest(manifest *Manifest, compressionLevel int) ([]byte, error) { // Compress using zstd var compressedBuf bytes.Buffer - writer, err := zstd.NewWriter(&compressedBuf, zstd.WithEncoderLevel(zstd.EncoderLevelFromZstd(compressionLevel))) + writer, err := zstd.NewWriter(&compressedBuf, + zstd.WithEncoderLevel(zstd.EncoderLevelFromZstd(compressionLevel))) if err != nil { return nil, fmt.Errorf("creating zstd writer: %w", err) } - if _, err := writer.Write(jsonData); err != nil { + _, err = writer.Write(jsonData) + if err != nil { _ = writer.Close() return nil, fmt.Errorf("writing compressed data: %w", err) } - if err := writer.Close(); err != nil { + err = writer.Close() + if err != nil { return nil, fmt.Errorf("closing zstd writer: %w", err) } diff --git a/internal/snapshot/module.go b/internal/snapshot/module.go index eca0b1d..fc0dfec 100644 --- a/internal/snapshot/module.go +++ b/internal/snapshot/module.go @@ -21,6 +21,8 @@ type ScannerParams struct { // Module exports backup functionality as an fx module. // It provides a ScannerFactory that can create Scanner instances // with custom parameters while sharing common dependencies. +// +//nolint:gochecknoglobals // fx module definitions are conventionally globals var Module = fx.Module("backup", fx.Provide( provideScannerFactory, @@ -31,7 +33,9 @@ var Module = fx.Module("backup", // ScannerFactory creates scanners with custom parameters type ScannerFactory func(params ScannerParams) *Scanner -func provideScannerFactory(cfg *config.Config, repos *database.Repositories, storer storage.Storer) ScannerFactory { +func provideScannerFactory( + cfg *config.Config, repos *database.Repositories, storer storage.Storer, +) ScannerFactory { return func(params ScannerParams) *Scanner { // Use provided excludes, or fall back to global config excludes excludes := params.Exclude diff --git a/internal/snapshot/permission_error_test.go b/internal/snapshot/permission_error_test.go index d339d3d..8dfad86 100644 --- a/internal/snapshot/permission_error_test.go +++ b/internal/snapshot/permission_error_test.go @@ -1,3 +1,4 @@ +//nolint:testpackage // needs access to unexported wrapPermissionError package snapshot import ( @@ -9,12 +10,15 @@ import ( "testing" ) -func TestWrapPermissionError(t *testing.T) { - // Non-permission errors pass through unchanged. - plain := errors.New("disk on fire") +// errDiskOnFire is a non-permission sentinel used to verify pass-through. +var errDiskOnFire = errors.New("disk on fire") - got := wrapPermissionError("/some/path", plain) - if !errors.Is(got, plain) { +func TestWrapPermissionError(t *testing.T) { + t.Parallel() + + // Non-permission errors pass through unchanged. + got := wrapPermissionError("/some/path", errDiskOnFire) + if !errors.Is(got, errDiskOnFire) { t.Errorf("non-permission error should pass through, got %v", got) } @@ -32,15 +36,16 @@ func TestWrapPermissionError(t *testing.T) { if runtime.GOOS == "darwin" { if !strings.Contains(wrapped.Error(), "Full Disk Access") { - t.Errorf("macOS permission error should mention Full Disk Access:\n%s", wrapped.Error()) + t.Errorf("macOS permission error should mention Full Disk Access:\n%s", + wrapped.Error()) } if !strings.Contains(wrapped.Error(), "System Settings") { - t.Errorf("macOS permission error should point at System Settings:\n%s", wrapped.Error()) - } - } else { - if !strings.Contains(wrapped.Error(), "--skip-errors") { - t.Errorf("non-macOS permission error should mention --skip-errors:\n%s", wrapped.Error()) + t.Errorf("macOS permission error should point at System Settings:\n%s", + wrapped.Error()) } + } else if !strings.Contains(wrapped.Error(), "--skip-errors") { + t.Errorf("non-macOS permission error should mention --skip-errors:\n%s", + wrapped.Error()) } } diff --git a/internal/snapshot/progress.go b/internal/snapshot/progress.go index 8795e92..71fbb98 100644 --- a/internal/snapshot/progress.go +++ b/internal/snapshot/progress.go @@ -19,14 +19,41 @@ const ( // These updates show current progress, ETA, and the file being processed. SummaryInterval = 10 * time.Second - // DetailInterval defines how often multi-line detailed status reports are printed. - // These reports include comprehensive statistics about files, chunks, blobs, and uploads. + // DetailInterval defines how often multi-line detailed status reports are + // printed. These reports include comprehensive statistics about files, + // chunks, blobs, and uploads. DetailInterval = 60 * time.Second // UploadProgressInterval defines how often upload progress messages are logged. UploadProgressInterval = 15 * time.Second ) +const ( + // bitsPerByte converts byte counts to bit counts for speed display. + bitsPerByte = 8 + + // percentScale converts a ratio to a percentage. + percentScale = 100 + + // currentFileMaxLen is the display width used for current-file paths. + currentFileMaxLen = 40 + + // secondsPerMinute and minutesPerHour are used for duration formatting. + secondsPerMinute = 60 + minutesPerHour = 60 + + // Bit-rate thresholds for human-readable upload speed formatting. + bitsPerGbit = 1e9 + bitsPerMbit = 1e6 + bitsPerKbit = 1e3 + + // ellipsis prefixes truncated paths and suffixes shortened hashes. + ellipsis = "..." + + // hashPrefixLen is how many hex characters of a blob hash to show in logs. + hashPrefixLen = 8 +) + // ProgressStats holds atomic counters for progress tracking type ProgressStats struct { FilesScanned atomic.Int64 // Total files seen during scan (includes skipped) @@ -64,7 +91,7 @@ type UploadInfo struct { // ProgressReporter handles periodic progress reporting type ProgressReporter struct { stats *ProgressStats - ctx context.Context + ctx context.Context //nolint:containedctx // bound at construction cancel context.CancelFunc wg sync.WaitGroup detailTicker *time.Ticker @@ -127,6 +154,161 @@ func (pr *ProgressReporter) SetTotalSize(size int64) { pr.stats.ProcessStartTime.Store(time.Now().UTC()) } +// Helper functions + +func formatDuration(d time.Duration) string { + if d < 0 { + return "unknown" + } + + if d < time.Minute { + return fmt.Sprintf("%ds", int(d.Seconds())) + } + + if d < time.Hour { + return fmt.Sprintf("%dm%ds", int(d.Minutes()), int(d.Seconds())%secondsPerMinute) + } + + return fmt.Sprintf("%dh%dm", int(d.Hours()), int(d.Minutes())%minutesPerHour) +} + +func formatPercent(numerator, denominator int64) string { + if denominator == 0 { + return "0.0%" + } + + return fmt.Sprintf("%.1f%%", float64(numerator)/float64(denominator)*percentScale) +} + +func formatRatio(compressed, uncompressed int64) string { + if uncompressed == 0 { + return "1.00" + } + + ratio := float64(compressed) / float64(uncompressed) + + return fmt.Sprintf("%.2f", ratio) +} + +func truncatePath(path string, maxLen int) string { + if len(path) <= maxLen { + return path + } + // Keep the last maxLen-len(ellipsis) characters and prepend the ellipsis. + return ellipsis + path[len(path)-(maxLen-len(ellipsis)):] +} + +// safeUint64 converts a non-negative int64 counter to uint64 for display, +// clamping negative values to zero. +func safeUint64(n int64) uint64 { + if n < 0 { + return 0 + } + + return uint64(n) +} + +// ReportUploadStart marks the beginning of a blob upload +func (pr *ProgressReporter) ReportUploadStart(blobHash string, size int64) { + info := &UploadInfo{ + BlobHash: blobHash, + Size: size, + StartTime: time.Now().UTC(), + } + pr.stats.CurrentUpload.Store(info) + + // Log the start of upload + log.Info("Starting blob upload", + "hash", blobHash[:hashPrefixLen]+ellipsis, + "size", humanize.Bytes(safeUint64(size))) +} + +// ReportUploadComplete marks the completion of a blob upload +func (pr *ProgressReporter) ReportUploadComplete( + blobHash string, size int64, duration time.Duration, +) { + // Clear current upload + pr.stats.CurrentUpload.Store((*UploadInfo)(nil)) + + // Add to total upload duration + pr.stats.UploadDurationMs.Add(duration.Milliseconds()) + + // Calculate speed + if duration < time.Millisecond { + duration = time.Millisecond + } + + bytesPerSec := float64(size) / duration.Seconds() + bitsPerSec := bytesPerSec * bitsPerByte + + // Format speed + var speedStr string + + switch { + case bitsPerSec >= bitsPerGbit: + speedStr = fmt.Sprintf("%.1fGbit/sec", bitsPerSec/bitsPerGbit) + case bitsPerSec >= bitsPerMbit: + speedStr = fmt.Sprintf("%.0fMbit/sec", bitsPerSec/bitsPerMbit) + case bitsPerSec >= bitsPerKbit: + speedStr = fmt.Sprintf("%.0fKbit/sec", bitsPerSec/bitsPerKbit) + default: + speedStr = fmt.Sprintf("%.0fbit/sec", bitsPerSec) + } + + log.Info("Blob upload completed", + "hash", blobHash[:hashPrefixLen]+ellipsis, + "size", humanize.Bytes(safeUint64(size)), + "duration", formatDuration(duration), + "speed", speedStr) +} + +// UpdateChunkingActivity updates the last chunking time +func (pr *ProgressReporter) UpdateChunkingActivity() { + pr.stats.mu.Lock() + pr.stats.lastChunkingTime = time.Now().UTC() + pr.stats.mu.Unlock() +} + +// ReportUploadProgress reports current upload progress with instantaneous speed +func (pr *ProgressReporter) ReportUploadProgress( + blobHash string, bytesUploaded, totalSize int64, instantSpeed float64, +) { + // Update the current upload info with progress + uploadInfo, ok := pr.stats.CurrentUpload.Load().(*UploadInfo) + if ok && uploadInfo != nil { + now := time.Now() + + // Only log at the configured interval + if now.Sub(uploadInfo.LastLogTime) >= UploadProgressInterval { + // Format speed in bits/second using humanize + bitsPerSec := instantSpeed * bitsPerByte + speedStr := humanize.SI(bitsPerSec, "bit/sec") + + percent := float64(bytesUploaded) / float64(totalSize) * percentScale + + // Calculate ETA based on current speed + etaStr := "unknown" + + if instantSpeed > 0 && bytesUploaded < totalSize { + remainingBytes := totalSize - bytesUploaded + remainingSeconds := float64(remainingBytes) / instantSpeed + eta := time.Duration(remainingSeconds * float64(time.Second)) + etaStr = formatDuration(eta) + } + + log.Info("Blob upload progress", + "hash", blobHash[:hashPrefixLen]+ellipsis, + "progress", fmt.Sprintf("%.1f%%", percent), + "uploaded", humanize.Bytes(safeUint64(bytesUploaded)), + "total", humanize.Bytes(safeUint64(totalSize)), + "speed", speedStr, + "eta", etaStr) + + uploadInfo.LastLogTime = now + } + } +} + // run is the main progress reporting loop func (pr *ProgressReporter) run() { defer pr.wg.Done() @@ -150,7 +332,8 @@ func (pr *ProgressReporter) run() { // printSummaryStatus prints a one-line status update func (pr *ProgressReporter) printSummaryStatus() { // Check if we're currently uploading - if uploadInfo, ok := pr.stats.CurrentUpload.Load().(*UploadInfo); ok && uploadInfo != nil { + uploadInfo, ok := pr.stats.CurrentUpload.Load().(*UploadInfo) + if ok && uploadInfo != nil { // Show upload progress instead pr.printUploadProgress(uploadInfo) @@ -172,7 +355,7 @@ func (pr *ProgressReporter) printSummaryStatus() { bytesSkipped := pr.stats.BytesSkipped.Load() bytesProcessed := pr.stats.BytesProcessed.Load() totalSize := pr.stats.TotalSize.Load() - currentFile := pr.stats.CurrentFile.Load().(string) + currentFile, _ := pr.stats.CurrentFile.Load().(string) // Calculate ETA if we have total size and are processing etaStr := "" @@ -201,15 +384,15 @@ func (pr *ProgressReporter) printSummaryStatus() { status := fmt.Sprintf("Snapshot progress: %d/%d files, %s/%s (%.1f%%), %s/s%s", filesProcessed, totalFiles, - humanize.Bytes(uint64(bytesProcessed)), - humanize.Bytes(uint64(totalSize)), - float64(bytesProcessed)/float64(totalSize)*100, + humanize.Bytes(safeUint64(bytesProcessed)), + humanize.Bytes(safeUint64(totalSize)), + float64(bytesProcessed)/float64(totalSize)*percentScale, humanize.Bytes(uint64(rate)), etaStr, ) if currentFile != "" { - status += " | Current: " + truncatePath(currentFile, 40) + status += " | Current: " + truncatePath(currentFile, currentFileMaxLen) } log.Info(status) @@ -232,7 +415,7 @@ func (pr *ProgressReporter) printDetailedStatus() { blobsCreated := pr.stats.BlobsCreated.Load() blobsUploaded := pr.stats.BlobsUploaded.Load() bytesUploaded := pr.stats.BytesUploaded.Load() - currentFile := pr.stats.CurrentFile.Load().(string) + currentFile, _ := pr.stats.CurrentFile.Load().(string) totalBytes := bytesScanned + bytesSkipped rate := float64(totalBytes) / elapsed.Seconds() @@ -251,11 +434,11 @@ func (pr *ProgressReporter) printDetailedStatus() { remainingBytes := totalSize - bytesProcessed remainingSeconds := float64(remainingBytes) / processRate eta := time.Duration(remainingSeconds * float64(time.Second)) - percentComplete := float64(bytesProcessed) / float64(totalSize) * 100 + percentComplete := float64(bytesProcessed) / float64(totalSize) * percentScale log.Info("Overall progress", "percent", fmt.Sprintf("%.1f%%", percentComplete), - "processed", humanize.Bytes(uint64(bytesProcessed)), - "total", humanize.Bytes(uint64(totalSize)), + "processed", humanize.Bytes(safeUint64(bytesProcessed)), + "total", humanize.Bytes(safeUint64(totalSize)), "rate", humanize.Bytes(uint64(processRate))+"/s", "eta", formatDuration(eta)) } @@ -268,9 +451,9 @@ func (pr *ProgressReporter) printDetailedStatus() { "total", filesScanned, "skip_rate", formatPercent(filesSkipped, filesScanned)) log.Info("Data scanned", - "new", humanize.Bytes(uint64(bytesScanned)), - "skipped", humanize.Bytes(uint64(bytesSkipped)), - "total", humanize.Bytes(uint64(totalBytes)), + "new", humanize.Bytes(safeUint64(bytesScanned)), + "skipped", humanize.Bytes(safeUint64(bytesSkipped)), + "total", humanize.Bytes(safeUint64(totalBytes)), "scan_rate", humanize.Bytes(uint64(rate))+"/s") log.Info("Chunks created", "count", chunksCreated) log.Info("Blobs status", @@ -278,7 +461,7 @@ func (pr *ProgressReporter) printDetailedStatus() { "uploaded", blobsUploaded, "pending", blobsCreated-blobsUploaded) log.Info("Total uploaded to remote", - "uploaded", humanize.Bytes(uint64(bytesUploaded)), + "uploaded", humanize.Bytes(safeUint64(bytesUploaded)), "compression_ratio", formatRatio(bytesUploaded, bytesScanned)) if currentFile != "" { @@ -288,146 +471,8 @@ func (pr *ProgressReporter) printDetailedStatus() { log.Notice("=============================") } -// Helper functions - -func formatDuration(d time.Duration) string { - if d < 0 { - return "unknown" - } - - if d < time.Minute { - return fmt.Sprintf("%ds", int(d.Seconds())) - } - - if d < time.Hour { - return fmt.Sprintf("%dm%ds", int(d.Minutes()), int(d.Seconds())%60) - } - - return fmt.Sprintf("%dh%dm", int(d.Hours()), int(d.Minutes())%60) -} - -func formatPercent(numerator, denominator int64) string { - if denominator == 0 { - return "0.0%" - } - - return fmt.Sprintf("%.1f%%", float64(numerator)/float64(denominator)*100) -} - -func formatRatio(compressed, uncompressed int64) string { - if uncompressed == 0 { - return "1.00" - } - - ratio := float64(compressed) / float64(uncompressed) - - return fmt.Sprintf("%.2f", ratio) -} - -func truncatePath(path string, maxLen int) string { - if len(path) <= maxLen { - return path - } - // Keep the last maxLen-3 characters and prepend "..." - return "..." + path[len(path)-(maxLen-3):] -} - // printUploadProgress prints upload progress -func (pr *ProgressReporter) printUploadProgress(info *UploadInfo) { +func (pr *ProgressReporter) printUploadProgress(_ *UploadInfo) { // This function is called repeatedly during upload, not just at start // Don't print anything here - the actual progress is shown by ReportUploadProgress } - -// ReportUploadStart marks the beginning of a blob upload -func (pr *ProgressReporter) ReportUploadStart(blobHash string, size int64) { - info := &UploadInfo{ - BlobHash: blobHash, - Size: size, - StartTime: time.Now().UTC(), - } - pr.stats.CurrentUpload.Store(info) - - // Log the start of upload - log.Info("Starting blob upload", - "hash", blobHash[:8]+"...", - "size", humanize.Bytes(uint64(size))) -} - -// ReportUploadComplete marks the completion of a blob upload -func (pr *ProgressReporter) ReportUploadComplete(blobHash string, size int64, duration time.Duration) { - // Clear current upload - pr.stats.CurrentUpload.Store((*UploadInfo)(nil)) - - // Add to total upload duration - pr.stats.UploadDurationMs.Add(duration.Milliseconds()) - - // Calculate speed - if duration < time.Millisecond { - duration = time.Millisecond - } - - bytesPerSec := float64(size) / duration.Seconds() - bitsPerSec := bytesPerSec * 8 - - // Format speed - var speedStr string - if bitsPerSec >= 1e9 { - speedStr = fmt.Sprintf("%.1fGbit/sec", bitsPerSec/1e9) - } else if bitsPerSec >= 1e6 { - speedStr = fmt.Sprintf("%.0fMbit/sec", bitsPerSec/1e6) - } else if bitsPerSec >= 1e3 { - speedStr = fmt.Sprintf("%.0fKbit/sec", bitsPerSec/1e3) - } else { - speedStr = fmt.Sprintf("%.0fbit/sec", bitsPerSec) - } - - log.Info("Blob upload completed", - "hash", blobHash[:8]+"...", - "size", humanize.Bytes(uint64(size)), - "duration", formatDuration(duration), - "speed", speedStr) -} - -// UpdateChunkingActivity updates the last chunking time -func (pr *ProgressReporter) UpdateChunkingActivity() { - pr.stats.mu.Lock() - pr.stats.lastChunkingTime = time.Now().UTC() - pr.stats.mu.Unlock() -} - -// ReportUploadProgress reports current upload progress with instantaneous speed -func (pr *ProgressReporter) ReportUploadProgress(blobHash string, bytesUploaded, totalSize int64, instantSpeed float64) { - // Update the current upload info with progress - if uploadInfo, ok := pr.stats.CurrentUpload.Load().(*UploadInfo); ok && uploadInfo != nil { - now := time.Now() - - // Only log at the configured interval - if now.Sub(uploadInfo.LastLogTime) >= UploadProgressInterval { - // Format speed in bits/second using humanize - bitsPerSec := instantSpeed * 8 - speedStr := humanize.SI(bitsPerSec, "bit/sec") - - percent := float64(bytesUploaded) / float64(totalSize) * 100 - - // Calculate ETA based on current speed - etaStr := "unknown" - - if instantSpeed > 0 && bytesUploaded < totalSize { - remainingBytes := totalSize - bytesUploaded - remainingSeconds := float64(remainingBytes) / instantSpeed - eta := time.Duration(remainingSeconds * float64(time.Second)) - etaStr = formatDuration(eta) - } - - log.Info("Blob upload progress", - "hash", blobHash[:8]+"...", - "progress", fmt.Sprintf("%.1f%%", percent), - "uploaded", humanize.Bytes(uint64(bytesUploaded)), - "total", humanize.Bytes(uint64(totalSize)), - "speed", speedStr, - "eta", etaStr) - - uploadInfo.LastLogTime = now - } - } -} diff --git a/internal/snapshot/scanner.go b/internal/snapshot/scanner.go index 06452e6..9a8eb27 100644 --- a/internal/snapshot/scanner.go +++ b/internal/snapshot/scanner.go @@ -48,28 +48,32 @@ type compiledPattern struct { // Scanner scans directories and populates the database with file and chunk information type Scanner struct { - fs afero.Fs - chunker *chunker.Chunker - packer *blob.Packer - repos *database.Repositories - storage storage.Storer - maxBlobSize int64 - compressionLevel int - ageRecipient string - snapshotID string // Current snapshot being processed - currentSourcePath string // Current source directory being scanned (for restore path stripping) + fs afero.Fs + chunker *chunker.Chunker + packer *blob.Packer + repos *database.Repositories + storage storage.Storer + maxBlobSize int64 + compressionLevel int + ageRecipient string + snapshotID string // Current snapshot being processed + // currentSourcePath is the source directory being scanned (used for + // restore path stripping). + currentSourcePath string exclude []string // Glob patterns for files/directories to exclude compiledExclude []compiledPattern // Compiled glob patterns progress *ProgressReporter - skipErrors bool // Skip file read errors (log loudly but continue) - ui *ui.Writer // User-facing output; never nil (defaults to a discarding writer) + skipErrors bool // Skip file read errors (log loudly but continue) + // ui is the user-facing output; never nil (defaults to a discarding writer). + ui *ui.Writer // In-memory cache of known chunk hashes for fast existence checks knownChunks map[string]struct{} knownChunksMu sync.RWMutex - // Pending chunk hashes - chunks that have been added to packer but not yet committed to DB - // When a blob finalizes, the committed chunks are removed from this set + // Pending chunk hashes - chunks that have been added to packer but not + // yet committed to DB. When a blob finalizes, the committed chunks are + // removed from this set. pendingChunkHashes map[string]struct{} pendingChunkHashesMu sync.Mutex @@ -82,9 +86,29 @@ type Scanner struct { packerMu sync.Mutex // Blocks chunk production during blob creation // Context for cancellation - scanCtx context.Context + scanCtx context.Context //nolint:containedctx // set per-Scan for packer callbacks } +// Periodic status output intervals and thresholds for the scan and +// process phases. +const ( + // scanStatusInterval is how often periodic progress lines are printed + // during the scan and process phases. + scanStatusInterval = 15 * time.Second + + // assocStatusInterval is how often progress is printed while + // associating unchanged files with the snapshot. + assocStatusInterval = 5 * time.Second + + // uploadProgressMinSeconds is the minimum elapsed time between live + // upload-progress reporter updates. + uploadProgressMinSeconds = 0.5 + + // snapshotAssocBatchSize is how many file IDs are associated with the + // snapshot per transaction. + snapshotAssocBatchSize = 1000 +) + // ScannerConfig contains configuration for the scanner type ScannerConfig struct { FS afero.Fs @@ -171,9 +195,12 @@ func NewScanner(cfg ScannerConfig) *Scanner { } // Scan scans a directory and populates the database -func (s *Scanner) Scan(ctx context.Context, path string, snapshotID string) (*ScanResult, error) { +func (s *Scanner) Scan( + ctx context.Context, path string, snapshotID string, +) (*ScanResult, error) { s.snapshotID = snapshotID - s.currentSourcePath = path // Store source path for file records (used during restore) + // Store source path for file records (used during restore) + s.currentSourcePath = path s.scanCtx = ctx result := &ScanResult{ StartTime: time.Now().UTC(), @@ -213,13 +240,15 @@ func (s *Scanner) Scan(ctx context.Context, path string, snapshotID string) (*Sc filesToProcess := scanResult.FilesToProcess // Phase 1b: Detect deleted files by comparing DB against scanned files - if err := s.detectDeletedFilesFromMap(ctx, knownFiles, existingFiles, result); err != nil { + err = s.detectDeletedFilesFromMap(ctx, knownFiles, existingFiles, result) + if err != nil { return nil, fmt.Errorf("detecting deleted files: %w", err) } // Phase 1c: Associate unchanged files with this snapshot (no new records needed) if len(scanResult.UnchangedFileIDs) > 0 { - s.ui.Beginf("Associating %s unchanged files with the snapshot.", s.ui.Count(len(scanResult.UnchangedFileIDs))) + s.ui.Beginf("Associating %s unchanged files with the snapshot.", + s.ui.Count(len(scanResult.UnchangedFileIDs))) err := s.batchAddFilesToSnapshot(ctx, scanResult.UnchangedFileIDs) if err != nil { @@ -232,15 +261,19 @@ func (s *Scanner) Scan(ctx context.Context, path string, snapshotID string) (*Sc // Phase 2: Process files and create chunks if len(filesToProcess) > 0 { - s.ui.Beginf("Backing up %s snapshot source files (chunking, compressing, encrypting, uploading).", s.ui.Count(len(filesToProcess))) - log.Info("Phase 2/3: Creating snapshot (chunking, compressing, encrypting, and uploading blobs)") + s.ui.Beginf("Backing up %s snapshot source files "+ + "(chunking, compressing, encrypting, uploading).", + s.ui.Count(len(filesToProcess))) + log.Info("Phase 2/3: Creating snapshot " + + "(chunking, compressing, encrypting, and uploading blobs)") err := s.processPhase(ctx, filesToProcess, result) if err != nil { return nil, fmt.Errorf("process phase failed: %w", err) } } else { - s.ui.Infof("Snapshot file backup skipped: no changed files (creating metadata-only snapshot).") + s.ui.Infof("Snapshot file backup skipped: no changed files " + + "(creating metadata-only snapshot).") log.Info("Phase 2/3: Skipping (no files need processing, metadata-only snapshot)") } @@ -250,9 +283,17 @@ func (s *Scanner) Scan(ctx context.Context, path string, snapshotID string) (*Sc return result, nil } -// loadDatabaseState loads known files and chunks from the database into memory for fast lookup -// This avoids per-file and per-chunk database queries during the scan and process phases -func (s *Scanner) loadDatabaseState(ctx context.Context, path string) (map[string]*database.File, error) { +// GetProgress returns the progress reporter for this scanner +func (s *Scanner) GetProgress() *ProgressReporter { + return s.progress +} + +// loadDatabaseState loads known files and chunks from the database into +// memory for fast lookup. This avoids per-file and per-chunk database +// queries during the scan and process phases. +func (s *Scanner) loadDatabaseState( + ctx context.Context, path string, +) (map[string]*database.File, error) { s.ui.Beginf("Loading known files from local index database.") knownFiles, err := s.loadKnownFiles(ctx, path) @@ -260,22 +301,27 @@ func (s *Scanner) loadDatabaseState(ctx context.Context, path string) (map[strin return nil, fmt.Errorf("loading known files: %w", err) } - s.ui.Completef("Loaded %s known files from local index database.", s.ui.Count(len(knownFiles))) + s.ui.Completef("Loaded %s known files from local index database.", + s.ui.Count(len(knownFiles))) s.ui.Beginf("Loading known chunks from local index database.") - if err := s.loadKnownChunks(ctx); err != nil { + err = s.loadKnownChunks(ctx) + if err != nil { return nil, fmt.Errorf("loading known chunks: %w", err) } - s.ui.Completef("Loaded %s known chunks from local index database.", s.ui.Count(len(s.knownChunks))) + s.ui.Completef("Loaded %s known chunks from local index database.", + s.ui.Count(len(s.knownChunks))) return knownFiles, nil } // summarizeScanPhase calculates total size to process, updates progress tracking, // and prints the scan phase summary with file counts and sizes -func (s *Scanner) summarizeScanPhase(result *ScanResult, filesToProcess []*FileToProcess) { +func (s *Scanner) summarizeScanPhase( + result *ScanResult, filesToProcess []*FileToProcess, +) { var totalSizeToProcess int64 for _, file := range filesToProcess { totalSizeToProcess += file.FileInfo.Size() @@ -288,11 +334,12 @@ func (s *Scanner) summarizeScanPhase(result *ScanResult, filesToProcess []*FileT log.Info("Phase 1 complete", "total_files", len(filesToProcess), - "total_size", humanize.Bytes(uint64(totalSizeToProcess)), + "total_size", humanize.Bytes(safeUint64(totalSizeToProcess)), "files_skipped", result.FilesSkipped, - "bytes_skipped", humanize.Bytes(uint64(result.BytesSkipped))) + "bytes_skipped", humanize.Bytes(safeUint64(result.BytesSkipped))) - msg := fmt.Sprintf("Enumerated %s snapshot source files (%s total), %s to back up (%s)", + msg := fmt.Sprintf( + "Enumerated %s snapshot source files (%s total), %s to back up (%s)", s.ui.Count(result.FilesScanned), s.ui.Size(totalSizeToProcess+result.BytesSkipped), s.ui.Count(len(filesToProcess)), @@ -329,7 +376,9 @@ func (s *Scanner) finalizeScanResult(ctx context.Context, result *ScanResult) { // loadKnownFiles loads all known files from the database into a map for fast lookup // This avoids per-file database queries during the scan phase -func (s *Scanner) loadKnownFiles(ctx context.Context, path string) (map[string]*database.File, error) { +func (s *Scanner) loadKnownFiles( + ctx context.Context, path string, +) (map[string]*database.File, error) { files, err := s.repos.Files.ListByPrefix(ctx, path) if err != nil { return nil, fmt.Errorf("listing files by prefix: %w", err) @@ -343,8 +392,9 @@ func (s *Scanner) loadKnownFiles(ctx context.Context, path string) (map[string]* return result, nil } -// loadKnownChunks loads all known chunk hashes from the database into a map for fast lookup -// This avoids per-chunk database queries during file processing +// loadKnownChunks loads all known chunk hashes from the database into a +// map for fast lookup. This avoids per-chunk database queries during file +// processing. func (s *Scanner) loadKnownChunks(ctx context.Context) error { chunks, err := s.repos.Chunks.List(ctx) if err != nil { @@ -396,7 +446,8 @@ func (s *Scanner) removePendingChunkHashes(hashes []string) { delete(s.pendingChunkHashes, hash) } s.pendingChunkHashesMu.Unlock() - log.Debug("removePendingChunkHashes: done", "count", len(hashes), "duration", time.Since(start)) + log.Debug("removePendingChunkHashes: done", + "count", len(hashes), "duration", time.Since(start)) } // isChunkPending returns true if the chunk is still pending (not yet committed to DB) @@ -508,19 +559,24 @@ func (s *Scanner) flushCompletedPendingFiles(ctx context.Context) error { txStart := time.Now() err := s.executeBatchFileFlush(ctx, allFiles, allFileIDs, allFileChunks, allChunkFiles) - log.Debug("flushCompletedPendingFiles: transaction done", "duration", time.Since(txStart)) - log.Debug("flushCompletedPendingFiles: total duration", "duration", time.Since(flushStart)) + log.Debug("flushCompletedPendingFiles: transaction done", + "duration", time.Since(txStart)) + log.Debug("flushCompletedPendingFiles: total duration", + "duration", time.Since(flushStart)) return err } // partitionPendingByChunkStatus separates pending files into those whose chunks -// are all committed to DB (ready to flush) and those still waiting on pending chunks. -// Updates s.pendingFiles to contain only the still-pending files. -func (s *Scanner) partitionPendingByChunkStatus() (canFlush []pendingFileData, stillPendingCount int) { +// are all committed to DB (ready to flush) and those still waiting on pending +// chunks. Updates s.pendingFiles to contain only the still-pending files. +func (s *Scanner) partitionPendingByChunkStatus() ([]pendingFileData, int) { + var canFlush []pendingFileData + log.Debug("flushCompletedPendingFiles: acquiring pendingFilesMu lock") s.pendingFilesMu.Lock() - log.Debug("flushCompletedPendingFiles: acquired lock", "pending_files", len(s.pendingFiles)) + log.Debug("flushCompletedPendingFiles: acquired lock", + "pending_files", len(s.pendingFiles)) var stillPending []pendingFileData @@ -546,7 +602,10 @@ func (s *Scanner) partitionPendingByChunkStatus() (canFlush []pendingFileData, s } } - log.Debug("flushCompletedPendingFiles: check done", "duration", time.Since(checkStart), "can_flush", len(canFlush), "still_pending", len(stillPending)) + log.Debug("flushCompletedPendingFiles: check done", + "duration", time.Since(checkStart), + "can_flush", len(canFlush), + "still_pending", len(stillPending)) s.pendingFiles = stillPending s.pendingFilesMu.Unlock() @@ -555,9 +614,12 @@ func (s *Scanner) partitionPendingByChunkStatus() (canFlush []pendingFileData, s return canFlush, len(stillPending) } -// collectBatchFlushData aggregates file records, IDs, file-chunk mappings, and chunk-file -// mappings from the given pending file data for efficient batch database operations -func (s *Scanner) collectBatchFlushData(canFlush []pendingFileData) ([]*database.File, []types.FileID, []database.FileChunk, []database.ChunkFile) { +// collectBatchFlushData aggregates file records, IDs, file-chunk mappings, +// and chunk-file mappings from the given pending file data for efficient +// batch database operations +func (s *Scanner) collectBatchFlushData( + canFlush []pendingFileData, +) ([]*database.File, []types.FileID, []database.FileChunk, []database.ChunkFile) { log.Debug("flushCompletedPendingFiles: collecting data for batch ops") collectStart := time.Now() @@ -565,10 +627,11 @@ func (s *Scanner) collectBatchFlushData(canFlush []pendingFileData) ([]*database var ( allFileChunks []database.FileChunk allChunkFiles []database.ChunkFile - allFileIDs []types.FileID - allFiles []*database.File ) + allFileIDs := make([]types.FileID, 0, len(canFlush)) + allFiles := make([]*database.File, 0, len(canFlush)) + for _, data := range canFlush { allFileChunks = append(allFileChunks, data.fileChunks...) allChunkFiles = append(allChunkFiles, data.chunkFiles...) @@ -585,82 +648,42 @@ func (s *Scanner) collectBatchFlushData(canFlush []pendingFileData) ([]*database return allFiles, allFileIDs, allFileChunks, allChunkFiles } -// executeBatchFileFlush writes all collected file data to the database in a single transaction, -// including deleting old mappings, creating file records, and adding snapshot associations -func (s *Scanner) executeBatchFileFlush(ctx context.Context, allFiles []*database.File, allFileIDs []types.FileID, allFileChunks []database.FileChunk, allChunkFiles []database.ChunkFile) error { +// executeBatchFileFlush writes all collected file data to the database in a +// single transaction, including deleting old mappings, creating file +// records, and adding snapshot associations +func (s *Scanner) executeBatchFileFlush( + ctx context.Context, + allFiles []*database.File, + allFileIDs []types.FileID, + allFileChunks []database.FileChunk, + allChunkFiles []database.ChunkFile, +) error { return s.repos.WithTx(ctx, func(txCtx context.Context, tx *sql.Tx) error { log.Debug("flushCompletedPendingFiles: inside transaction") - // Batch delete old file_chunks and chunk_files - log.Debug("flushCompletedPendingFiles: deleting old file_chunks") - - opStart := time.Now() - - err := s.repos.FileChunks.DeleteByFileIDs(txCtx, tx, allFileIDs) + err := s.batchDeleteOldMappings(txCtx, tx, allFileIDs) if err != nil { - return fmt.Errorf("batch deleting old file chunks: %w", err) + return err } - log.Debug("flushCompletedPendingFiles: deleted file_chunks", "duration", time.Since(opStart)) - - log.Debug("flushCompletedPendingFiles: deleting old chunk_files") - - opStart = time.Now() - - err = s.repos.ChunkFiles.DeleteByFileIDs(txCtx, tx, allFileIDs) + err = s.batchInsertFileData( + txCtx, tx, allFiles, allFileChunks, allChunkFiles) if err != nil { - return fmt.Errorf("batch deleting old chunk files: %w", err) + return err } - log.Debug("flushCompletedPendingFiles: deleted chunk_files", "duration", time.Since(opStart)) - - // Batch create/update file records - log.Debug("flushCompletedPendingFiles: creating files") - - opStart = time.Now() - - err = s.repos.Files.CreateBatch(txCtx, tx, allFiles) - if err != nil { - return fmt.Errorf("batch creating file records: %w", err) - } - - log.Debug("flushCompletedPendingFiles: created files", "duration", time.Since(opStart)) - - // Batch insert file_chunks - log.Debug("flushCompletedPendingFiles: inserting file_chunks") - - opStart = time.Now() - - err = s.repos.FileChunks.CreateBatch(txCtx, tx, allFileChunks) - if err != nil { - return fmt.Errorf("batch creating file chunks: %w", err) - } - - log.Debug("flushCompletedPendingFiles: inserted file_chunks", "duration", time.Since(opStart)) - - // Batch insert chunk_files - log.Debug("flushCompletedPendingFiles: inserting chunk_files") - - opStart = time.Now() - - err = s.repos.ChunkFiles.CreateBatch(txCtx, tx, allChunkFiles) - if err != nil { - return fmt.Errorf("batch creating chunk files: %w", err) - } - - log.Debug("flushCompletedPendingFiles: inserted chunk_files", "duration", time.Since(opStart)) - // Batch add files to snapshot log.Debug("flushCompletedPendingFiles: adding files to snapshot") - opStart = time.Now() + opStart := time.Now() err = s.repos.Snapshots.AddFilesByIDBatch(txCtx, tx, s.snapshotID, allFileIDs) if err != nil { return fmt.Errorf("batch adding files to snapshot: %w", err) } - log.Debug("flushCompletedPendingFiles: added files to snapshot", "duration", time.Since(opStart)) + log.Debug("flushCompletedPendingFiles: added files to snapshot", + "duration", time.Since(opStart)) log.Debug("flushCompletedPendingFiles: transaction complete") @@ -668,6 +691,89 @@ func (s *Scanner) executeBatchFileFlush(ctx context.Context, allFiles []*databas }) } +// batchDeleteOldMappings removes stale file_chunks and chunk_files rows +// for the given files inside the flush transaction. +func (s *Scanner) batchDeleteOldMappings( + txCtx context.Context, tx *sql.Tx, allFileIDs []types.FileID, +) error { + // Batch delete old file_chunks and chunk_files + log.Debug("flushCompletedPendingFiles: deleting old file_chunks") + + opStart := time.Now() + + err := s.repos.FileChunks.DeleteByFileIDs(txCtx, tx, allFileIDs) + if err != nil { + return fmt.Errorf("batch deleting old file chunks: %w", err) + } + + log.Debug("flushCompletedPendingFiles: deleted file_chunks", + "duration", time.Since(opStart)) + + log.Debug("flushCompletedPendingFiles: deleting old chunk_files") + + opStart = time.Now() + + err = s.repos.ChunkFiles.DeleteByFileIDs(txCtx, tx, allFileIDs) + if err != nil { + return fmt.Errorf("batch deleting old chunk files: %w", err) + } + + log.Debug("flushCompletedPendingFiles: deleted chunk_files", + "duration", time.Since(opStart)) + + return nil +} + +// batchInsertFileData creates file records and their chunk association +// rows inside the flush transaction. +func (s *Scanner) batchInsertFileData( + txCtx context.Context, tx *sql.Tx, + allFiles []*database.File, + allFileChunks []database.FileChunk, + allChunkFiles []database.ChunkFile, +) error { + // Batch create/update file records + log.Debug("flushCompletedPendingFiles: creating files") + + opStart := time.Now() + + err := s.repos.Files.CreateBatch(txCtx, tx, allFiles) + if err != nil { + return fmt.Errorf("batch creating file records: %w", err) + } + + log.Debug("flushCompletedPendingFiles: created files", + "duration", time.Since(opStart)) + + // Batch insert file_chunks + log.Debug("flushCompletedPendingFiles: inserting file_chunks") + + opStart = time.Now() + + err = s.repos.FileChunks.CreateBatch(txCtx, tx, allFileChunks) + if err != nil { + return fmt.Errorf("batch creating file chunks: %w", err) + } + + log.Debug("flushCompletedPendingFiles: inserted file_chunks", + "duration", time.Since(opStart)) + + // Batch insert chunk_files + log.Debug("flushCompletedPendingFiles: inserting chunk_files") + + opStart = time.Now() + + err = s.repos.ChunkFiles.CreateBatch(txCtx, tx, allChunkFiles) + if err != nil { + return fmt.Errorf("batch creating chunk files: %w", err) + } + + log.Debug("flushCompletedPendingFiles: inserted chunk_files", + "duration", time.Since(opStart)) + + return nil +} + // ScanPhaseResult contains the results of the scan phase type ScanPhaseResult struct { FilesToProcess []*FileToProcess @@ -675,40 +781,31 @@ type ScanPhaseResult struct { } // scanPhase performs the initial directory scan to identify files to process -// It uses the pre-loaded knownFiles map for fast change detection without DB queries -// It also populates existingFiles map for deletion detection -// Returns files needing processing and IDs of unchanged files for snapshot association -func (s *Scanner) scanPhase(ctx context.Context, path string, result *ScanResult, existingFiles map[string]struct{}, knownFiles map[string]*database.File) (*ScanPhaseResult, error) { +// It uses the pre-loaded knownFiles map for fast change detection without DB +// queries. It also populates existingFiles map for deletion detection. +// Returns files needing processing and IDs of unchanged files for snapshot +// association. +func (s *Scanner) scanPhase( + ctx context.Context, + path string, + result *ScanResult, + existingFiles map[string]struct{}, + knownFiles map[string]*database.File, +) (*ScanPhaseResult, error) { // Use known file count as estimate for progress (accurate for subsequent backups) estimatedTotal := int64(len(knownFiles)) - var filesToProcess []*FileToProcess - - var unchangedFileIDs []types.FileID // Just IDs - no new records needed - - var mu sync.Mutex + collector := &scanCollector{} // Set up periodic status output startTime := time.Now() lastStatusTime := time.Now() - statusInterval := 15 * time.Second - - var filesScanned int64 log.Debug("Starting directory walk", "path", path) - err := afero.Walk(s.fs, path, func(filePath string, info os.FileInfo, err error) error { + walkFn := func(filePath string, info os.FileInfo, err error) error { if err != nil { - if s.skipErrors { - log.Error("Failed to access file (skipping due to --skip-errors)", "path", filePath, "error", err) - s.ui.Errorf("Failed to access %s: %v. Skipping (--skip-errors).", s.ui.Path(filePath), err) - - return nil // Continue scanning - } - - log.Debug("Error accessing filesystem entry", "path", filePath, "error", err) - - return wrapPermissionError(filePath, err) + return s.handleWalkError(filePath, err) } // Check context cancellation @@ -727,42 +824,9 @@ func (s *Scanner) scanPhase(ctx context.Context, path string, result *ScanResult return nil } - // Handle symlinks - if info.Mode()&os.ModeSymlink != 0 { - file := s.buildSymlinkEntry(filePath, info) - if file != nil { - existingFiles[filePath] = struct{}{} - - mu.Lock() - - filesToProcess = append(filesToProcess, &FileToProcess{ - Path: filePath, - FileInfo: info, - File: file, - }) - filesScanned++ - mu.Unlock() - s.updateScanEntryStats(result, true, info) - } - - return nil - } - - // Handle directories (record for permission/ownership preservation and empty-dir support) - if info.IsDir() { - file := s.buildDirectoryEntry(filePath, info) - existingFiles[filePath] = struct{}{} - - mu.Lock() - - filesToProcess = append(filesToProcess, &FileToProcess{ - Path: filePath, - FileInfo: info, - File: file, - }) - filesScanned++ - mu.Unlock() - + // Handle symlinks and directories + if handled := s.recordSpecialEntry( + filePath, info, existingFiles, collector, result); handled { return nil } @@ -777,48 +841,136 @@ func (s *Scanner) scanPhase(ctx context.Context, path string, result *ScanResult // Check file against in-memory map (no DB query!) file, needsProcessing := s.checkFileInMemory(filePath, info, knownFiles) - mu.Lock() - if needsProcessing { - // New or changed file - will create record after processing - filesToProcess = append(filesToProcess, &FileToProcess{ - Path: filePath, - FileInfo: info, - File: file, - }) - } else if !file.ID.IsZero() { - // Unchanged file with existing ID - just need snapshot association - unchangedFileIDs = append(unchangedFileIDs, file.ID) - } - - filesScanned++ - changedCount := len(filesToProcess) - mu.Unlock() + filesScanned, changedCount := collector.recordRegularFile( + filePath, info, file, needsProcessing) // Update result stats s.updateScanEntryStats(result, needsProcessing, info) // Output periodic status - if time.Since(lastStatusTime) >= statusInterval { + if time.Since(lastStatusTime) >= scanStatusInterval { s.printScanProgressLine(filesScanned, changedCount, estimatedTotal, startTime) lastStatusTime = time.Now() } return nil - }) + } + + err := afero.Walk(s.fs, path, walkFn) if err != nil { return nil, err } return &ScanPhaseResult{ - FilesToProcess: filesToProcess, - UnchangedFileIDs: unchangedFileIDs, + FilesToProcess: collector.filesToProcess, + UnchangedFileIDs: collector.unchangedFileIDs, }, nil } +// recordSpecialEntry records symlinks and directories (which have no +// data to chunk) and reports whether it handled the entry. +func (s *Scanner) recordSpecialEntry( + filePath string, info os.FileInfo, + existingFiles map[string]struct{}, + collector *scanCollector, result *ScanResult, +) bool { + // Handle symlinks + if info.Mode()&os.ModeSymlink != 0 { + file := s.buildSymlinkEntry(filePath, info) + if file != nil { + existingFiles[filePath] = struct{}{} + collector.addToProcess(filePath, info, file) + s.updateScanEntryStats(result, true, info) + } + + return true + } + + // Handle directories (record for permission/ownership preservation + // and empty-dir support) + if info.IsDir() { + file := s.buildDirectoryEntry(filePath, info) + existingFiles[filePath] = struct{}{} + collector.addToProcess(filePath, info, file) + + return true + } + + return false +} + +// handleWalkError deals with a filesystem error surfaced by the walk: +// skipped (with a UI notice) under --skip-errors, fatal otherwise. +func (s *Scanner) handleWalkError(filePath string, err error) error { + if s.skipErrors { + log.Error("Failed to access file (skipping due to --skip-errors)", + "path", filePath, "error", err) + s.ui.Errorf("Failed to access %s: %v. Skipping (--skip-errors).", + s.ui.Path(filePath), err) + + return nil // Continue scanning + } + + log.Debug("Error accessing filesystem entry", "path", filePath, "error", err) + + return wrapPermissionError(filePath, err) +} + +// scanCollector accumulates scan-phase walk results under a mutex. +type scanCollector struct { + mu sync.Mutex + filesToProcess []*FileToProcess + unchangedFileIDs []types.FileID // Just IDs - no new records needed + filesScanned int64 +} + +// addToProcess queues a file entry for processing. +func (c *scanCollector) addToProcess( + path string, info os.FileInfo, file *database.File, +) { + c.mu.Lock() + defer c.mu.Unlock() + + c.filesToProcess = append(c.filesToProcess, &FileToProcess{ + Path: path, + FileInfo: info, + File: file, + }) + c.filesScanned++ +} + +// recordRegularFile records a regular file as changed (to process) or +// unchanged (snapshot association only) and returns the running scanned +// and changed counts. +func (c *scanCollector) recordRegularFile( + path string, info os.FileInfo, file *database.File, needsProcessing bool, +) (int64, int) { + c.mu.Lock() + defer c.mu.Unlock() + + if needsProcessing { + // New or changed file - will create record after processing + c.filesToProcess = append(c.filesToProcess, &FileToProcess{ + Path: path, + FileInfo: info, + File: file, + }) + } else if !file.ID.IsZero() { + // Unchanged file with existing ID - just need snapshot association + c.unchangedFileIDs = append(c.unchangedFileIDs, file.ID) + } + + c.filesScanned++ + + return c.filesScanned, len(c.filesToProcess) +} + // updateScanEntryStats updates the scan result and progress reporter statistics // for a single scanned file entry based on whether it needs processing -func (s *Scanner) updateScanEntryStats(result *ScanResult, needsProcessing bool, info os.FileInfo) { +func (s *Scanner) updateScanEntryStats( + result *ScanResult, needsProcessing bool, info os.FileInfo, +) { if needsProcessing { result.BytesScanned += info.Size() if s.progress != nil { @@ -842,47 +994,55 @@ func (s *Scanner) updateScanEntryStats(result *ScanResult, needsProcessing bool, // printScanProgressLine prints a periodic progress line during the scan phase, // showing files scanned, percentage complete (if estimate available), and ETA -func (s *Scanner) printScanProgressLine(filesScanned int64, changedCount int, estimatedTotal int64, startTime time.Time) { +func (s *Scanner) printScanProgressLine( + filesScanned int64, changedCount int, estimatedTotal int64, startTime time.Time, +) { elapsed := time.Since(startTime) rate := float64(filesScanned) / elapsed.Seconds() - if estimatedTotal > 0 { - // Show actual scanned vs estimate (may exceed estimate if files were added) - pct := float64(filesScanned) / float64(estimatedTotal) * 100 - if pct > 100 { - pct = 100 // Cap at 100% for display - } - - remaining := max(estimatedTotal-filesScanned, 0) - - var eta time.Duration - if rate > 0 && remaining > 0 { - eta = time.Duration(float64(remaining)/rate) * time.Second - } - - if eta > 0 { - s.ui.Progressf("Snapshot source files enumeration: %s files (~%s), %s changed or new, %.0f files/sec, enumeration elapsed: %s, enumeration ETA: %s (est remain %s).", - s.ui.Count(int(filesScanned)), - s.ui.Percent(pct), - s.ui.Count(changedCount), - rate, - s.ui.Duration(elapsed), - s.ui.Time(time.Now().Add(eta)), - s.ui.Duration(eta)) - } else { - s.ui.Progressf("Snapshot source files enumeration: %s files (~%s), %s changed or new, %.0f files/sec, enumeration elapsed: %s.", - s.ui.Count(int(filesScanned)), - s.ui.Percent(pct), - s.ui.Count(changedCount), - rate, - s.ui.Duration(elapsed)) - } - } else { - s.ui.Progressf("Snapshot source files enumeration: %s files seen, %s changed or new, %.0f files/sec, enumeration elapsed: %s.", + if estimatedTotal <= 0 { + s.ui.Progressf("Snapshot source files enumeration: %s files seen, "+ + "%s changed or new, %.0f files/sec, enumeration elapsed: %s.", s.ui.Count(int(filesScanned)), s.ui.Count(changedCount), rate, s.ui.Duration(elapsed)) + + return + } + + // Show actual scanned vs estimate (may exceed estimate if files were added) + pct := float64(filesScanned) / float64(estimatedTotal) * percentScale + if pct > percentScale { + pct = percentScale // Cap at 100% for display + } + + remaining := max(estimatedTotal-filesScanned, 0) + + var eta time.Duration + if rate > 0 && remaining > 0 { + eta = time.Duration(float64(remaining)/rate) * time.Second + } + + if eta > 0 { + s.ui.Progressf("Snapshot source files enumeration: %s files (~%s), "+ + "%s changed or new, %.0f files/sec, enumeration elapsed: %s, "+ + "enumeration ETA: %s (est remain %s).", + s.ui.Count(int(filesScanned)), + s.ui.Percent(pct), + s.ui.Count(changedCount), + rate, + s.ui.Duration(elapsed), + s.ui.Time(time.Now().Add(eta)), + s.ui.Duration(eta)) + } else { + s.ui.Progressf("Snapshot source files enumeration: %s files (~%s), "+ + "%s changed or new, %.0f files/sec, enumeration elapsed: %s.", + s.ui.Count(int(filesScanned)), + s.ui.Percent(pct), + s.ui.Count(changedCount), + rate, + s.ui.Duration(elapsed)) } } @@ -956,7 +1116,9 @@ func (s *Scanner) recordNonRegularFile(ctx context.Context, ftp *FileToProcess) // checkFileInMemory checks if a file needs processing using the in-memory map // No database access is performed - this is purely CPU/memory work -func (s *Scanner) checkFileInMemory(path string, info os.FileInfo, knownFiles map[string]*database.File) (*database.File, bool) { +func (s *Scanner) checkFileInMemory( + path string, info os.FileInfo, knownFiles map[string]*database.File, +) (*database.File, bool) { // Get file stats stat, ok := info.Sys().(interface { Uid() uint32 @@ -983,9 +1145,10 @@ func (s *Scanner) checkFileInMemory(path string, info os.FileInfo, knownFiles ma } file := &database.File{ - ID: fileID, - Path: types.FilePath(path), - SourcePath: types.SourcePath(s.currentSourcePath), // Store source directory for restore path stripping + ID: fileID, + Path: types.FilePath(path), + // Store source directory for restore path stripping + SourcePath: types.SourcePath(s.currentSourcePath), MTime: info.ModTime(), Size: info.Size(), Mode: uint32(info.Mode()), @@ -1013,14 +1176,13 @@ func (s *Scanner) checkFileInMemory(path string, info os.FileInfo, knownFiles ma // batchAddFilesToSnapshot adds existing file IDs to the snapshot association table // This is used for unchanged files that already have records in the database -func (s *Scanner) batchAddFilesToSnapshot(ctx context.Context, fileIDs []types.FileID) error { - const batchSize = 1000 - +func (s *Scanner) batchAddFilesToSnapshot( + ctx context.Context, fileIDs []types.FileID, +) error { startTime := time.Now() lastStatusTime := time.Now() - statusInterval := 5 * time.Second - for i := 0; i < len(fileIDs); i += batchSize { + for i := 0; i < len(fileIDs); i += snapshotAssocBatchSize { // Check context cancellation select { case <-ctx.Done(): @@ -1028,7 +1190,7 @@ func (s *Scanner) batchAddFilesToSnapshot(ctx context.Context, fileIDs []types.F default: } - end := min(i+batchSize, len(fileIDs)) + end := min(i+snapshotAssocBatchSize, len(fileIDs)) batch := fileIDs[i:end] @@ -1047,10 +1209,10 @@ func (s *Scanner) batchAddFilesToSnapshot(ctx context.Context, fileIDs []types.F } // Periodic status - if time.Since(lastStatusTime) >= statusInterval { + if time.Since(lastStatusTime) >= assocStatusInterval { elapsed := time.Since(startTime) rate := float64(end) / elapsed.Seconds() - pct := float64(end) / float64(len(fileIDs)) * 100 + pct := float64(end) / float64(len(fileIDs)) * percentScale s.ui.Progressf("Snapshot unchanged-file association: %s/%s (%s), %.0f files/sec.", s.ui.Count(end), s.ui.Count(len(fileIDs)), s.ui.Percent(pct), rate) @@ -1060,14 +1222,17 @@ func (s *Scanner) batchAddFilesToSnapshot(ctx context.Context, fileIDs []types.F elapsed := time.Since(startTime) rate := float64(len(fileIDs)) / elapsed.Seconds() - s.ui.Completef("Associated %s unchanged files with the snapshot in %s (%.0f files/sec).", + s.ui.Completef("Associated %s unchanged files with the snapshot in %s "+ + "(%.0f files/sec).", s.ui.Count(len(fileIDs)), s.ui.Duration(elapsed), rate) return nil } // processPhase processes the files that need backing up -func (s *Scanner) processPhase(ctx context.Context, filesToProcess []*FileToProcess, result *ScanResult) error { +func (s *Scanner) processPhase( + ctx context.Context, filesToProcess []*FileToProcess, result *ScanResult, +) error { // Calculate total bytes to process var totalBytes int64 for _, f := range filesToProcess { @@ -1076,7 +1241,6 @@ func (s *Scanner) processPhase(ctx context.Context, filesToProcess []*FileToProc // Set up periodic status output lastStatusTime := time.Now() - statusInterval := 15 * time.Second startTime := time.Now() filesProcessed := 0 @@ -1110,8 +1274,9 @@ func (s *Scanner) processPhase(ctx context.Context, filesToProcess []*FileToProc bytesProcessed += fileToProcess.FileInfo.Size() // Output periodic status - if time.Since(lastStatusTime) >= statusInterval { - s.printProcessingProgress(filesProcessed, totalFiles, bytesProcessed, totalBytes, startTime) + if time.Since(lastStatusTime) >= scanStatusInterval { + s.printProcessingProgress( + filesProcessed, totalFiles, bytesProcessed, totalBytes, startTime) lastStatusTime = time.Now() } @@ -1123,12 +1288,15 @@ func (s *Scanner) processPhase(ctx context.Context, filesToProcess []*FileToProc // processFileWithErrorHandling wraps processFileStreaming with error recovery for // deleted files and skip-errors mode. Returns (skipped, error). -func (s *Scanner) processFileWithErrorHandling(ctx context.Context, fileToProcess *FileToProcess, result *ScanResult) (bool, error) { +func (s *Scanner) processFileWithErrorHandling( + ctx context.Context, fileToProcess *FileToProcess, result *ScanResult, +) (bool, error) { err := s.processFileStreaming(ctx, fileToProcess, result) if err != nil { // Handle files that were deleted between scan and process phases if errors.Is(err, os.ErrNotExist) { - log.Warn("File was deleted during backup, skipping", "path", fileToProcess.Path) + log.Warn("File was deleted during backup, skipping", + "path", fileToProcess.Path) result.FilesSkipped++ @@ -1136,8 +1304,10 @@ func (s *Scanner) processFileWithErrorHandling(ctx context.Context, fileToProces } // Skip file read errors if --skip-errors is enabled if s.skipErrors { - log.Error("Failed to process file (skipping due to --skip-errors)", "path", fileToProcess.Path, "error", err) - s.ui.Errorf("Failed to process %s: %v. Skipping (--skip-errors).", s.ui.Path(fileToProcess.Path), err) + log.Error("Failed to process file (skipping due to --skip-errors)", + "path", fileToProcess.Path, "error", err) + s.ui.Errorf("Failed to process %s: %v. Skipping (--skip-errors).", + s.ui.Path(fileToProcess.Path), err) result.FilesSkipped++ @@ -1152,9 +1322,12 @@ func (s *Scanner) processFileWithErrorHandling(ctx context.Context, fileToProces // printProcessingProgress prints a periodic progress line during the process phase, // showing files processed, bytes transferred, throughput, and ETA -func (s *Scanner) printProcessingProgress(filesProcessed, totalFiles int, bytesProcessed, totalBytes int64, startTime time.Time) { +func (s *Scanner) printProcessingProgress( + filesProcessed, totalFiles int, bytesProcessed, totalBytes int64, + startTime time.Time, +) { elapsed := time.Since(startTime) - pct := float64(bytesProcessed) / float64(totalBytes) * 100 + pct := float64(bytesProcessed) / float64(totalBytes) * percentScale byteRate := float64(bytesProcessed) / elapsed.Seconds() fileRate := float64(filesProcessed) / elapsed.Seconds() @@ -1167,7 +1340,8 @@ func (s *Scanner) printProcessingProgress(filesProcessed, totalFiles int, bytesP } if eta > 0 { - s.ui.Progressf("Snapshot backup: %s/%s files (%s), %s/%s, %s, %.0f files/sec, backup elapsed: %s, backup ETA: %s (est remain %s).", + s.ui.Progressf("Snapshot backup: %s/%s files (%s), %s/%s, %s, "+ + "%.0f files/sec, backup elapsed: %s, backup ETA: %s (est remain %s).", s.ui.Count(filesProcessed), s.ui.Count(totalFiles), s.ui.Percent(pct), @@ -1179,7 +1353,8 @@ func (s *Scanner) printProcessingProgress(filesProcessed, totalFiles int, bytesP s.ui.Time(time.Now().Add(eta)), s.ui.Duration(eta)) } else { - s.ui.Progressf("Snapshot backup: %s/%s files (%s), %s/%s, %s, %.0f files/sec, backup elapsed: %s.", + s.ui.Progressf("Snapshot backup: %s/%s files (%s), %s/%s, %s, "+ + "%.0f files/sec, backup elapsed: %s.", s.ui.Count(filesProcessed), s.ui.Count(totalFiles), s.ui.Percent(pct), @@ -1191,14 +1366,15 @@ func (s *Scanner) printProcessingProgress(filesProcessed, totalFiles int, bytesP } } -// finalizeProcessPhase flushes the packer, writes remaining pending files to the database, -// and handles local blob storage when no remote storage is configured +// finalizeProcessPhase flushes the packer, writes remaining pending files to +// the database, and handles local blob storage when no remote storage is +// configured func (s *Scanner) finalizeProcessPhase(ctx context.Context, result *ScanResult) error { // Final packer flush first - this commits remaining chunks to DB // and handleBlobReady will flush files whose chunks are now committed s.packerMu.Lock() - err := s.packer.Flush() + err := s.packer.Flush(ctx) if err != nil { s.packerMu.Unlock() @@ -1206,8 +1382,8 @@ func (s *Scanner) finalizeProcessPhase(ctx context.Context, result *ScanResult) } s.packerMu.Unlock() - // Flush any remaining pending files (e.g., files with only pre-existing chunks - // that didn't trigger a blob finalize) + // Flush any remaining pending files (e.g., files with only pre-existing + // chunks that didn't trigger a blob finalize) err = s.flushAllPending(ctx) if err != nil { return fmt.Errorf("flushing remaining pending files: %w", err) @@ -1225,7 +1401,8 @@ func (s *Scanner) finalizeProcessPhase(ctx context.Context, result *ScanResult) } err = s.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { - return s.repos.Snapshots.AddBlob(ctx, tx, s.snapshotID, blobID, types.BlobHash(b.Hash)) + return s.repos.Snapshots.AddBlob(ctx, tx, s.snapshotID, blobID, + types.BlobHash(b.Hash)) }) if err != nil { return fmt.Errorf("storing blob metadata: %w", err) @@ -1239,7 +1416,7 @@ func (s *Scanner) finalizeProcessPhase(ctx context.Context, result *ScanResult) } // handleBlobReady is called by the packer when a blob is finalized -func (s *Scanner) handleBlobReady(blobWithReader *blob.BlobWithReader) error { +func (s *Scanner) handleBlobReady(blobWithReader *blob.WithReader) error { startTime := time.Now().UTC() finishedBlob := blobWithReader.FinishedBlob @@ -1253,7 +1430,8 @@ func (s *Scanner) handleBlobReady(blobWithReader *blob.BlobWithReader) error { ctx = context.Background() } - blobPath := fmt.Sprintf("blobs/%s/%s/%s", finishedBlob.Hash[:2], finishedBlob.Hash[2:4], finishedBlob.Hash) + blobPath := fmt.Sprintf("blobs/%s/%s/%s", + finishedBlob.Hash[:2], finishedBlob.Hash[2:4], finishedBlob.Hash) blobExists, err := s.uploadBlobIfNeeded(ctx, blobPath, blobWithReader, startTime) if err != nil { @@ -1262,7 +1440,8 @@ func (s *Scanner) handleBlobReady(blobWithReader *blob.BlobWithReader) error { return fmt.Errorf("uploading blob %s: %w", finishedBlob.Hash, err) } - if err := s.recordBlobMetadata(ctx, finishedBlob, blobExists, startTime); err != nil { + err = s.recordBlobMetadata(ctx, finishedBlob, blobExists, startTime) + if err != nil { s.cleanupBlobTempFile(blobWithReader) return err @@ -1274,34 +1453,47 @@ func (s *Scanner) handleBlobReady(blobWithReader *blob.BlobWithReader) error { s.removePendingChunkHashes(blobWithReader.InsertedChunkHashes) // Flush files whose chunks are now all committed - if err := s.flushCompletedPendingFiles(ctx); err != nil { + err = s.flushCompletedPendingFiles(ctx) + if err != nil { return fmt.Errorf("flushing completed files: %w", err) } return nil } -// uploadBlobIfNeeded uploads the blob to storage if it doesn't already exist, returns whether it existed -func (s *Scanner) uploadBlobIfNeeded(ctx context.Context, blobPath string, blobWithReader *blob.BlobWithReader, startTime time.Time) (bool, error) { +// uploadBlobIfNeeded uploads the blob to storage if it doesn't already +// exist, returns whether it existed +func (s *Scanner) uploadBlobIfNeeded( + ctx context.Context, + blobPath string, + blobWithReader *blob.WithReader, + startTime time.Time, +) (bool, error) { finishedBlob := blobWithReader.FinishedBlob // Check if blob already exists (deduplication after restart) destination := s.storage.Info().Location - if _, err := s.storage.Stat(ctx, blobPath); err == nil { + + _, err := s.storage.Stat(ctx, blobPath) + if err == nil { log.Info("Blob already exists in storage, skipping upload", - "hash", finishedBlob.Hash, "size", humanize.Bytes(uint64(finishedBlob.Compressed))) + "hash", finishedBlob.Hash, + "size", humanize.Bytes(safeUint64(finishedBlob.Compressed))) s.ui.Infof("Blob %s (%s) already exists at %s. Skipping upload.", - s.ui.Hex(finishedBlob.Hash), s.ui.Size(finishedBlob.Compressed), s.ui.Path(destination)) + s.ui.Hex(finishedBlob.Hash), s.ui.Size(finishedBlob.Compressed), + s.ui.Path(destination)) return true, nil } s.ui.Beginf("Uploading blob %s (%s) to %s.", - s.ui.Hex(finishedBlob.Hash), s.ui.Size(finishedBlob.Compressed), s.ui.Path(destination)) + s.ui.Hex(finishedBlob.Hash), s.ui.Size(finishedBlob.Compressed), + s.ui.Path(destination)) progressCallback := s.makeUploadProgressCallback(ctx, finishedBlob, startTime) - err := s.storage.PutWithProgress(ctx, blobPath, blobWithReader.Reader, finishedBlob.Compressed, progressCallback) + err = s.storage.PutWithProgress(ctx, blobPath, blobWithReader.Reader, + finishedBlob.Compressed, progressCallback) if err != nil { log.Error("Failed to upload blob", "hash", finishedBlob.Hash, "error", err) @@ -1319,12 +1511,14 @@ func (s *Scanner) uploadBlobIfNeeded(ctx context.Context, blobPath string, blobW log.Info("Successfully uploaded blob to storage", "path", blobPath, - "size", humanize.Bytes(uint64(finishedBlob.Compressed)), + "size", humanize.Bytes(safeUint64(finishedBlob.Compressed)), "duration", uploadDuration, - "speed", humanize.SI(uploadSpeedBps*8, "bps")) + "speed", humanize.SI(uploadSpeedBps*bitsPerByte, "bps")) if s.progress != nil { - s.progress.ReportUploadComplete(finishedBlob.Hash, finishedBlob.Compressed, uploadDuration) + s.progress.ReportUploadComplete(finishedBlob.Hash, finishedBlob.Compressed, + uploadDuration) + stats := s.progress.GetStats() stats.BlobsUploaded.Add(1) stats.BytesUploaded.Add(finishedBlob.Compressed) @@ -1336,7 +1530,9 @@ func (s *Scanner) uploadBlobIfNeeded(ctx context.Context, blobPath string, blobW // makeUploadProgressCallback creates a progress callback for blob uploads. // It updates the live progress reporter ~twice/sec for ETAs and prints a // human-readable status line to s.output at most every 15 seconds. -func (s *Scanner) makeUploadProgressCallback(ctx context.Context, finishedBlob *blob.FinishedBlob, uploadStart time.Time) func(int64) error { +func (s *Scanner) makeUploadProgressCallback( + ctx context.Context, finishedBlob *blob.FinishedBlob, uploadStart time.Time, +) func(int64) error { lastProgressTime := time.Now() lastProgressBytes := int64(0) lastStdoutTime := time.Now() @@ -1347,12 +1543,13 @@ func (s *Scanner) makeUploadProgressCallback(ctx context.Context, finishedBlob * now := time.Now() elapsed := now.Sub(lastProgressTime).Seconds() - if elapsed > 0.5 { + if elapsed > uploadProgressMinSeconds { bytesSinceLastUpdate := uploaded - lastProgressBytes speed := float64(bytesSinceLastUpdate) / elapsed if s.progress != nil { - s.progress.ReportUploadProgress(finishedBlob.Hash, uploaded, finishedBlob.Compressed, speed) + s.progress.ReportUploadProgress(finishedBlob.Hash, uploaded, + finishedBlob.Compressed, speed) } lastProgressTime = now @@ -1362,15 +1559,17 @@ func (s *Scanner) makeUploadProgressCallback(ctx context.Context, finishedBlob * // Periodic stdout status line so the user knows the upload is alive. if now.Sub(lastStdoutTime) >= stdoutInterval { totalElapsed := now.Sub(uploadStart) - pct := float64(uploaded) / float64(finishedBlob.Compressed) * 100 + pct := float64(uploaded) / float64(finishedBlob.Compressed) * percentScale avgSpeed := float64(uploaded) / totalElapsed.Seconds() var eta time.Duration if avgSpeed > 0 { - eta = time.Duration(float64(finishedBlob.Compressed-uploaded)/avgSpeed) * time.Second + eta = time.Duration( + float64(finishedBlob.Compressed-uploaded)/avgSpeed) * time.Second } - s.ui.Progressf("Blob upload %s: %s / %s (%s) at %s, blob upload elapsed: %s, blob upload ETA: %s (est remain %s).", + s.ui.Progressf("Blob upload %s: %s / %s (%s) at %s, "+ + "blob upload elapsed: %s, blob upload ETA: %s (est remain %s).", s.ui.Hex(finishedBlob.Hash), s.ui.Size(uploaded), s.ui.Size(finishedBlob.Compressed), @@ -1392,7 +1591,12 @@ func (s *Scanner) makeUploadProgressCallback(ctx context.Context, finishedBlob * } // recordBlobMetadata stores blob upload metadata in the database -func (s *Scanner) recordBlobMetadata(ctx context.Context, finishedBlob *blob.FinishedBlob, blobExists bool, startTime time.Time) error { +func (s *Scanner) recordBlobMetadata( + ctx context.Context, + finishedBlob *blob.FinishedBlob, + blobExists bool, + startTime time.Time, +) error { finishedBlobID, err := types.ParseBlobID(finishedBlob.ID) if err != nil { return fmt.Errorf("parsing finished blob ID: %w", err) @@ -1406,7 +1610,8 @@ func (s *Scanner) recordBlobMetadata(ctx context.Context, finishedBlob *blob.Fin return fmt.Errorf("updating blob upload timestamp: %w", err) } - err = s.repos.Snapshots.AddBlob(txCtx, tx, s.snapshotID, finishedBlobID, types.BlobHash(finishedBlob.Hash)) + err = s.repos.Snapshots.AddBlob(txCtx, tx, s.snapshotID, finishedBlobID, + types.BlobHash(finishedBlob.Hash)) if err != nil { return fmt.Errorf("adding blob to snapshot: %w", err) } @@ -1431,7 +1636,7 @@ func (s *Scanner) recordBlobMetadata(ctx context.Context, finishedBlob *blob.Fin } // cleanupBlobTempFile closes and removes the blob's temporary file -func (s *Scanner) cleanupBlobTempFile(blobWithReader *blob.BlobWithReader) { +func (s *Scanner) cleanupBlobTempFile(blobWithReader *blob.WithReader) { if blobWithReader.TempFile != nil { tempName := blobWithReader.TempFile.Name() @@ -1455,7 +1660,9 @@ type streamingChunkInfo struct { } // processFileStreaming processes a file by streaming chunks directly to the packer -func (s *Scanner) processFileStreaming(ctx context.Context, fileToProcess *FileToProcess, result *ScanResult) error { +func (s *Scanner) processFileStreaming( + ctx context.Context, fileToProcess *FileToProcess, result *ScanResult, +) error { // Symlinks and directories have no data to chunk — just record them in the DB. mode := os.FileMode(fileToProcess.File.Mode) if mode&os.ModeSymlink != 0 || mode.IsDir() { @@ -1499,7 +1706,7 @@ func (s *Scanner) processFileStreaming(ctx context.Context, fileToProcess *FileT s.updateChunkStats(chunkExists, chunk.Size, result) if !chunkExists { - err := s.addChunkToPacker(chunk) + err := s.addChunkToPacker(ctx, chunk) if err != nil { return err } @@ -1523,7 +1730,9 @@ func (s *Scanner) processFileStreaming(ctx context.Context, fileToProcess *FileT } // updateChunkStats updates scan result and progress stats for a processed chunk -func (s *Scanner) updateChunkStats(chunkExists bool, chunkSize int64, result *ScanResult) { +func (s *Scanner) updateChunkStats( + chunkExists bool, chunkSize int64, result *ScanResult, +) { if chunkExists { result.FilesSkipped++ @@ -1543,20 +1752,21 @@ func (s *Scanner) updateChunkStats(chunkExists bool, chunkSize int64, result *Sc } } -// addChunkToPacker adds a chunk to the blob packer, finalizing the current blob if needed -func (s *Scanner) addChunkToPacker(chunk chunker.Chunk) error { +// addChunkToPacker adds a chunk to the blob packer, finalizing the current +// blob if needed +func (s *Scanner) addChunkToPacker(ctx context.Context, chunk chunker.Chunk) error { s.packerMu.Lock() - err := s.packer.AddChunk(&blob.ChunkRef{Hash: chunk.Hash, Data: chunk.Data}) + err := s.packer.AddChunk(ctx, &blob.ChunkRef{Hash: chunk.Hash, Data: chunk.Data}) if errors.Is(err, blob.ErrBlobSizeLimitExceeded) { - err := s.packer.FinalizeBlob() + err = s.packer.FinalizeBlob(ctx) if err != nil { s.packerMu.Unlock() return fmt.Errorf("finalizing blob: %w", err) } - err = s.packer.AddChunk(&blob.ChunkRef{Hash: chunk.Hash, Data: chunk.Data}) + err = s.packer.AddChunk(ctx, &blob.ChunkRef{Hash: chunk.Hash, Data: chunk.Data}) if err != nil { s.packerMu.Unlock() @@ -1572,8 +1782,11 @@ func (s *Scanner) addChunkToPacker(chunk chunker.Chunk) error { return nil } -// queueFileForBatchInsert builds file/chunk associations and queues the file for batch DB insert -func (s *Scanner) queueFileForBatchInsert(ctx context.Context, fileToProcess *FileToProcess, chunks []streamingChunkInfo) { +// queueFileForBatchInsert builds file/chunk associations and queues the +// file for batch DB insert +func (s *Scanner) queueFileForBatchInsert( + ctx context.Context, fileToProcess *FileToProcess, chunks []streamingChunkInfo, +) { fileChunks := make([]database.FileChunk, len(chunks)) chunkFiles := make([]database.ChunkFile, len(chunks)) @@ -1598,14 +1811,15 @@ func (s *Scanner) queueFileForBatchInsert(ctx context.Context, fileToProcess *Fi }) } -// GetProgress returns the progress reporter for this scanner -func (s *Scanner) GetProgress() *ProgressReporter { - return s.progress -} - -// detectDeletedFilesFromMap finds files that existed in previous snapshots but no longer exist -// Uses pre-loaded maps to avoid any filesystem or database access -func (s *Scanner) detectDeletedFilesFromMap(ctx context.Context, knownFiles map[string]*database.File, existingFiles map[string]struct{}, result *ScanResult) error { +// detectDeletedFilesFromMap finds files that existed in previous snapshots +// but no longer exist. Uses pre-loaded maps to avoid any filesystem or +// database access. +func (s *Scanner) detectDeletedFilesFromMap( + ctx context.Context, + knownFiles map[string]*database.File, + existingFiles map[string]struct{}, + result *ScanResult, +) error { if len(knownFiles) == 0 { return nil } @@ -1629,7 +1843,8 @@ func (s *Scanner) detectDeletedFilesFromMap(ctx context.Context, knownFiles map[ } if result.FilesDeleted > 0 { - s.ui.Infof("Snapshot source files enumeration detected %s deleted files.", s.ui.Count(result.FilesDeleted)) + s.ui.Infof("Snapshot source files enumeration detected %s deleted files.", + s.ui.Count(result.FilesDeleted)) } return nil @@ -1652,7 +1867,8 @@ func wrapPermissionError(path string, err error) error { "then quit and reopen the terminal and re-run the backup", path, err) } - return fmt.Errorf("cannot read %s: %w (check file permissions, or run with --skip-errors to continue past unreadable files)", path, err) + return fmt.Errorf("cannot read %s: %w (check file permissions, or run "+ + "with --skip-errors to continue past unreadable files)", path, err) } // compileExcludePatterns compiles the exclude patterns into glob matchers @@ -1719,39 +1935,66 @@ func (s *Scanner) shouldExclude(filePath, rootPath string) bool { // Check each pattern for _, cp := range s.compiledExclude { - if cp.anchored { - // Anchored pattern: must match from the root - // Match the relative path directly - if cp.pattern.Match(relPath) { - return true - } - // Also check if any prefix of the path matches (for directory patterns) - parts := strings.Split(relPath, "/") - for i := 1; i <= len(parts); i++ { - prefix := strings.Join(parts[:i], "/") - if cp.pattern.Match(prefix) { - return true - } - } - } else { - // Unanchored pattern: can match anywhere in path - // Check the full relative path - if cp.pattern.Match(relPath) { - return true - } - // Check each path component and subpath - parts := strings.Split(relPath, "/") - for i := range parts { - // Match individual component (e.g., ".git" matches ".git" directory) - if cp.pattern.Match(parts[i]) { - return true - } - // Match subpath from this component onwards - subpath := strings.Join(parts[i:], "/") - if cp.pattern.Match(subpath) { - return true - } - } + if cp.matches(relPath) { + return true + } + } + + return false +} + +// matches reports whether the compiled pattern matches the given +// slash-normalized relative path, honoring anchoring. +func (cp compiledPattern) matches(relPath string) bool { + if cp.anchored { + return cp.matchesAnchored(relPath) + } + + return cp.matchesUnanchored(relPath) +} + +// matchesAnchored matches the pattern from the root of the source +// directory: the relative path itself or any of its leading directory +// prefixes must match. +func (cp compiledPattern) matchesAnchored(relPath string) bool { + // Match the relative path directly + if cp.pattern.Match(relPath) { + return true + } + + // Also check if any prefix of the path matches (for directory patterns) + parts := strings.Split(relPath, "/") + for i := 1; i <= len(parts); i++ { + prefix := strings.Join(parts[:i], "/") + if cp.pattern.Match(prefix) { + return true + } + } + + return false +} + +// matchesUnanchored matches the pattern anywhere in the path: against the +// full relative path, each individual component, and every trailing +// subpath. +func (cp compiledPattern) matchesUnanchored(relPath string) bool { + // Check the full relative path + if cp.pattern.Match(relPath) { + return true + } + + // Check each path component and subpath + parts := strings.Split(relPath, "/") + for i := range parts { + // Match individual component (e.g., ".git" matches ".git" directory) + if cp.pattern.Match(parts[i]) { + return true + } + + // Match subpath from this component onwards + subpath := strings.Join(parts[i:], "/") + if cp.pattern.Match(subpath) { + return true } } diff --git a/internal/snapshot/scanner_test.go b/internal/snapshot/scanner_test.go index 4ed2d3b..96c3e83 100644 --- a/internal/snapshot/scanner_test.go +++ b/internal/snapshot/scanner_test.go @@ -3,6 +3,7 @@ package snapshot_test import ( "context" "database/sql" + "os" "path/filepath" "testing" "time" @@ -14,9 +15,114 @@ import ( "sneak.berlin/go/vaultik/internal/types" ) +// Shared test fixture values for the snapshot_test package. +const ( + // testHost is the hostname recorded on test snapshot rows. + testHost = "test-host" + + // testVersion is the vaultik version recorded on test snapshot rows. + testVersion = "test" + + // testAgePublicKey is the fixed age public key used for test encryption. + testAgePublicKey = "age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05gl0sjq9q9wjg" +) + +// TestMain initializes the shared logger once, before any tests run, so +// parallel tests never race on the logger's global state. +func TestMain(m *testing.M) { + log.Initialize(log.Config{}) + os.Exit(m.Run()) +} + +// createTestSnapshotRecord inserts an empty snapshot row used as the +// association target for scan tests. +func createTestSnapshotRecord( + ctx context.Context, t *testing.T, repos *database.Repositories, snapshotID string, +) { + t.Helper() + + err := repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { + snapshot := &database.Snapshot{ + ID: types.SnapshotID(snapshotID), + Hostname: testHost, + VaultikVersion: testVersion, + StartedAt: time.Now(), + CompletedAt: nil, + FileCount: 0, + ChunkCount: 0, + BlobCount: 0, + TotalSize: 0, + BlobSize: 0, + CompressionRatio: 1.0, + } + + return repos.Snapshots.Create(ctx, tx, snapshot) + }) + if err != nil { + t.Fatalf("failed to create snapshot: %v", err) + } +} + +// verifySimpleScanDatabase checks the database contents produced by +// TestScannerSimpleDirectory's scan. +func verifySimpleScanDatabase( + ctx context.Context, t *testing.T, repos *database.Repositories, +) { + t.Helper() + + // Verify files in database - includes regular files and directories + files, err := repos.Files.ListByPrefix(ctx, "/source") + if err != nil { + t.Fatalf("failed to list files: %v", err) + } + + // 6 regular files + 3 directories (/source, /source/subdir, /source/subdir2) + if len(files) != 9 { + t.Errorf("expected 9 entries in database (6 files + 3 dirs), got %d", len(files)) + } + + // Verify specific file + file1, err := repos.Files.GetByPath(ctx, "/source/file1.txt") + if err != nil { + t.Fatalf("failed to get file1.txt: %v", err) + } + + if file1.Size != 13 { + t.Errorf("expected file1.txt size 13, got %d", file1.Size) + } + + if file1.Mode != 0644 { + t.Errorf("expected file1.txt mode 0644, got %o", file1.Mode) + } + + // Verify chunks were created + chunks, err := repos.FileChunks.GetByFile(ctx, "/source/file1.txt") + if err != nil { + t.Fatalf("failed to get chunks for file1.txt: %v", err) + } + + if len(chunks) != 1 { // Small file should be one chunk + t.Errorf("expected 1 chunk for file1.txt, got %d", len(chunks)) + } + + // Verify deduplication - file3.txt and file4.txt have different content + // but we should still have the correct number of unique chunks + allChunks, err := repos.Chunks.List(ctx) + if err != nil { + t.Fatalf("failed to list all chunks: %v", err) + } + + // We should have at most 6 chunks (one per unique file content) + // Empty file might not create a chunk + if len(allChunks) > 6 { + t.Errorf("expected at most 6 chunks, got %d", len(allChunks)) + } +} + func TestScannerSimpleDirectory(t *testing.T) { // Initialize logger for tests log.Initialize(log.Config{}) + t.Parallel() // Create in-memory filesystem fs := afero.NewMemMapFs() @@ -74,38 +180,17 @@ func TestScannerSimpleDirectory(t *testing.T) { Repositories: repos, MaxBlobSize: int64(1024 * 1024), // 1MB blobs CompressionLevel: 3, - AgeRecipients: []string{"age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05gl0sjq9q9wjg"}, // Test public key + AgeRecipients: []string{testAgePublicKey}, }) // Create a snapshot record for testing ctx := context.Background() snapshotID := "test-snapshot-001" - err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { - snapshot := &database.Snapshot{ - ID: types.SnapshotID(snapshotID), - Hostname: "test-host", - VaultikVersion: "test", - StartedAt: time.Now(), - CompletedAt: nil, - FileCount: 0, - ChunkCount: 0, - BlobCount: 0, - TotalSize: 0, - BlobSize: 0, - CompressionRatio: 1.0, - } - - return repos.Snapshots.Create(ctx, tx, snapshot) - }) - if err != nil { - t.Fatalf("failed to create snapshot: %v", err) - } + createTestSnapshotRecord(ctx, t, repos, snapshotID) // Scan the directory - var result *snapshot.ScanResult - - result, err = scanner.Scan(ctx, "/source", snapshotID) + result, err := scanner.Scan(ctx, "/source", snapshotID) if err != nil { t.Fatalf("scan failed: %v", err) } @@ -120,58 +205,13 @@ func TestScannerSimpleDirectory(t *testing.T) { t.Errorf("expected at least 97 bytes scanned, got %d", result.BytesScanned) } - // Verify files in database - includes regular files and directories - files, err := repos.Files.ListByPrefix(ctx, "/source") - if err != nil { - t.Fatalf("failed to list files: %v", err) - } - - // 6 regular files + 3 directories (/source, /source/subdir, /source/subdir2) - if len(files) != 9 { - t.Errorf("expected 9 entries in database (6 files + 3 dirs), got %d", len(files)) - } - - // Verify specific file - file1, err := repos.Files.GetByPath(ctx, "/source/file1.txt") - if err != nil { - t.Fatalf("failed to get file1.txt: %v", err) - } - - if file1.Size != 13 { - t.Errorf("expected file1.txt size 13, got %d", file1.Size) - } - - if file1.Mode != 0644 { - t.Errorf("expected file1.txt mode 0644, got %o", file1.Mode) - } - - // Verify chunks were created - chunks, err := repos.FileChunks.GetByFile(ctx, "/source/file1.txt") - if err != nil { - t.Fatalf("failed to get chunks for file1.txt: %v", err) - } - - if len(chunks) != 1 { // Small file should be one chunk - t.Errorf("expected 1 chunk for file1.txt, got %d", len(chunks)) - } - - // Verify deduplication - file3.txt and file4.txt have different content - // but we should still have the correct number of unique chunks - allChunks, err := repos.Chunks.List(ctx) - if err != nil { - t.Fatalf("failed to list all chunks: %v", err) - } - - // We should have at most 6 chunks (one per unique file content) - // Empty file might not create a chunk - if len(allChunks) > 6 { - t.Errorf("expected at most 6 chunks, got %d", len(allChunks)) - } + verifySimpleScanDatabase(ctx, t, repos) } func TestScannerLargeFile(t *testing.T) { // Initialize logger for tests log.Initialize(log.Config{}) + t.Parallel() // Create in-memory filesystem fs := afero.NewMemMapFs() @@ -182,14 +222,17 @@ func TestScannerLargeFile(t *testing.T) { // Fill with pseudo-random data to ensure chunk boundaries for i := range largeContent { // Simple pseudo-random generator for deterministic tests + //nolint:gosec // G115: intentional byte truncation of test data largeContent[i] = byte((i * 7919) ^ (i >> 3)) } - if err := fs.MkdirAll("/source", 0755); err != nil { + err := fs.MkdirAll("/source", 0755) + if err != nil { t.Fatal(err) } - if err := afero.WriteFile(fs, "/source/large.bin", largeContent, 0644); err != nil { + err = afero.WriteFile(fs, "/source/large.bin", largeContent, 0644) + if err != nil { t.Fatal(err) } @@ -214,38 +257,17 @@ func TestScannerLargeFile(t *testing.T) { Repositories: repos, MaxBlobSize: int64(1024 * 1024), CompressionLevel: 3, - AgeRecipients: []string{"age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05gl0sjq9q9wjg"}, // Test public key + AgeRecipients: []string{testAgePublicKey}, }) // Create a snapshot record for testing ctx := context.Background() snapshotID := "test-snapshot-001" - err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { - snapshot := &database.Snapshot{ - ID: types.SnapshotID(snapshotID), - Hostname: "test-host", - VaultikVersion: "test", - StartedAt: time.Now(), - CompletedAt: nil, - FileCount: 0, - ChunkCount: 0, - BlobCount: 0, - TotalSize: 0, - BlobSize: 0, - CompressionRatio: 1.0, - } - - return repos.Snapshots.Create(ctx, tx, snapshot) - }) - if err != nil { - t.Fatalf("failed to create snapshot: %v", err) - } + createTestSnapshotRecord(ctx, t, repos, snapshotID) // Scan the directory - var result *snapshot.ScanResult - - result, err = scanner.Scan(ctx, "/source", snapshotID) + result, err := scanner.Scan(ctx, "/source", snapshotID) if err != nil { t.Fatalf("scan failed: %v", err) } @@ -257,7 +279,8 @@ func TestScannerLargeFile(t *testing.T) { // The file size should be at least 1MB if result.BytesScanned < 1024*1024 { - t.Errorf("expected at least %d bytes scanned, got %d", 1024*1024, result.BytesScanned) + t.Errorf("expected at least %d bytes scanned, got %d", + 1024*1024, result.BytesScanned) } // Verify chunks diff --git a/internal/snapshot/snapshot.go b/internal/snapshot/snapshot.go index 545d6e7..2efe8d8 100644 --- a/internal/snapshot/snapshot.go +++ b/internal/snapshot/snapshot.go @@ -1,3 +1,6 @@ +// Package snapshot implements snapshot creation: scanning source +// directories, chunking and deduplicating file data, packing chunks into +// encrypted blobs, and exporting per-snapshot metadata to remote storage. package snapshot // Snapshot Metadata Export Process @@ -58,6 +61,8 @@ import ( ) // SnapshotManager handles snapshot creation and metadata export +// +//nolint:revive // renaming snapshot.SnapshotManager is a cross-package API change type SnapshotManager struct { repos *database.Repositories storage storage.Storer @@ -66,6 +71,8 @@ type SnapshotManager struct { } // SnapshotManagerParams holds dependencies for NewSnapshotManager +// +//nolint:revive // renaming this alongside SnapshotManager is a cross-package API change type SnapshotManagerParams struct { fx.In @@ -88,15 +95,22 @@ func (sm *SnapshotManager) SetFilesystem(fs afero.Fs) { sm.fs = fs } -// CreateSnapshot creates a new snapshot record in the database at the start of a backup. +// CreateSnapshot creates a new snapshot record in the database at the +// start of a backup. +// // Deprecated: Use CreateSnapshotWithName instead for multi-snapshot support. -func (sm *SnapshotManager) CreateSnapshot(ctx context.Context, hostname, version, gitRevision string) (string, error) { +func (sm *SnapshotManager) CreateSnapshot( + ctx context.Context, hostname, version, gitRevision string, +) (string, error) { return sm.CreateSnapshotWithName(ctx, hostname, "", version, gitRevision) } -// CreateSnapshotWithName creates a new snapshot record with an optional snapshot name. -// The snapshot ID format is: hostname_name_timestamp or hostname_timestamp if name is empty. -func (sm *SnapshotManager) CreateSnapshotWithName(ctx context.Context, hostname, name, version, gitRevision string) (string, error) { +// CreateSnapshotWithName creates a new snapshot record with an optional +// snapshot name. The snapshot ID format is: hostname_name_timestamp or +// hostname_timestamp if name is empty. +func (sm *SnapshotManager) CreateSnapshotWithName( + ctx context.Context, hostname, name, version, gitRevision string, +) (string, error) { // Use short hostname (strip domain if present) shortHostname := hostname if before, _, ok := strings.Cut(hostname, "."); ok { @@ -141,7 +155,9 @@ func (sm *SnapshotManager) CreateSnapshotWithName(ctx context.Context, hostname, } // UpdateSnapshotStats updates the statistics for a snapshot during backup -func (sm *SnapshotManager) UpdateSnapshotStats(ctx context.Context, snapshotID string, stats BackupStats) error { +func (sm *SnapshotManager) UpdateSnapshotStats( + ctx context.Context, snapshotID string, stats BackupStats, +) error { err := sm.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { return sm.repos.Snapshots.UpdateCounts(ctx, tx, snapshotID, int64(stats.FilesScanned), @@ -160,7 +176,9 @@ func (sm *SnapshotManager) UpdateSnapshotStats(ctx context.Context, snapshotID s // UpdateSnapshotStatsExtended updates snapshot statistics with extended metrics. // This includes compression level, uncompressed blob size, and upload duration. -func (sm *SnapshotManager) UpdateSnapshotStatsExtended(ctx context.Context, snapshotID string, stats ExtendedBackupStats) error { +func (sm *SnapshotManager) UpdateSnapshotStatsExtended( + ctx context.Context, snapshotID string, stats ExtendedBackupStats, +) error { return sm.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { // First update basic stats err := sm.repos.Snapshots.UpdateCounts(ctx, tx, snapshotID, @@ -187,7 +205,9 @@ func (sm *SnapshotManager) UpdateSnapshotStatsExtended(ctx context.Context, snap // is populated with every blob holding any chunk referenced by the // snapshot's files (including deduplicated blobs uploaded by prior // snapshots). Without this, fully-deduplicated snapshots are unrestorable. -func (sm *SnapshotManager) CompleteSnapshot(ctx context.Context, snapshotID string) error { +func (sm *SnapshotManager) CompleteSnapshot( + ctx context.Context, snapshotID string, +) error { err := sm.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { added, err := sm.repos.Snapshots.PopulateReferencedBlobs(ctx, tx, snapshotID) if err != nil { @@ -226,8 +246,11 @@ func (sm *SnapshotManager) CompleteSnapshot(ctx context.Context, snapshotID stri // - Reopening the main database after this method returns // // This ensures database consistency during the copy operation. -func (sm *SnapshotManager) ExportSnapshotMetadata(ctx context.Context, dbPath string, snapshotID string) error { - log.Info("Phase 3/3: Exporting snapshot metadata", "snapshot_id", snapshotID, "source_db", dbPath) +func (sm *SnapshotManager) ExportSnapshotMetadata( + ctx context.Context, dbPath string, snapshotID string, +) error { + log.Info("Phase 3/3: Exporting snapshot metadata", + "snapshot_id", snapshotID, "source_db", dbPath) // Create temp directory for all temporary files tempDir, err := afero.TempDir(sm.fs, "", "vaultik-snapshot-*") @@ -258,7 +281,8 @@ func (sm *SnapshotManager) ExportSnapshotMetadata(ctx context.Context, dbPath st } // Step 7: Upload to S3 in snapshot subdirectory - if err := sm.uploadSnapshotArtifacts(ctx, snapshotID, finalData, blobManifest); err != nil { + err = sm.uploadSnapshotArtifacts(ctx, snapshotID, finalData, blobManifest) + if err != nil { return err } @@ -270,15 +294,130 @@ func (sm *SnapshotManager) ExportSnapshotMetadata(ctx context.Context, dbPath st return nil } -// prepareExportDB copies, cleans, vacuums, and compresses the snapshot database for export. -// Returns the compressed data and the path to the temporary database (needed for manifest generation). -func (sm *SnapshotManager) prepareExportDB(ctx context.Context, dbPath, snapshotID, tempDir string) ([]byte, string, error) { +// CleanupIncompleteSnapshots removes incomplete snapshots that don't have +// metadata in S3. This is critical for data safety: incomplete snapshots +// can cause deduplication to skip files that were never successfully +// backed up, resulting in data loss. +func (sm *SnapshotManager) CleanupIncompleteSnapshots( + ctx context.Context, hostname string, +) error { + log.Info("Checking for incomplete snapshots", "hostname", hostname) + + // Get all incomplete snapshots for this hostname + incompleteSnapshots, err := sm.repos.Snapshots.GetIncompleteByHostname(ctx, hostname) + if err != nil { + return fmt.Errorf("getting incomplete snapshots: %w", err) + } + + if len(incompleteSnapshots) == 0 { + log.Debug("No incomplete snapshots found") + + return nil + } + + log.Info("Found incomplete snapshots", "count", len(incompleteSnapshots)) + + // Check each incomplete snapshot for metadata in storage + for _, snapshot := range incompleteSnapshots { + // Check if metadata exists in storage (paths use the hashed + // remote key so we don't leak host info to the listing). + metadataKey := fmt.Sprintf("metadata/%s/db.zst", + RemoteSnapshotKey(snapshot.ID.String())) + + _, err := sm.storage.Stat(ctx, metadataKey) + if err != nil { + // Metadata doesn't exist in S3 - this is an incomplete snapshot + log.Info("Cleaning up incomplete snapshot record", + "snapshot_id", snapshot.ID, "started_at", snapshot.StartedAt) + + // Delete the snapshot and all its associations + err := sm.deleteSnapshot(ctx, snapshot.ID.String()) + if err != nil { + return fmt.Errorf("deleting incomplete snapshot %s: %w", + snapshot.ID, err) + } + + log.Info("Deleted incomplete snapshot record and associated data", + "snapshot_id", snapshot.ID) + } else { + // Metadata exists - this snapshot was completed but database wasn't updated + // This shouldn't happen in normal operation, but mark it complete + log.Warn("Found snapshot with remote metadata but incomplete in database", + "snapshot_id", snapshot.ID) + + err := sm.repos.Snapshots.MarkComplete(ctx, nil, snapshot.ID.String()) + if err != nil { + log.Error("Failed to mark snapshot as complete in database", + "snapshot_id", snapshot.ID, "error", err) + } + } + } + + return nil +} + +// CleanupOrphanedData removes files, chunks, and blobs that are no longer +// referenced by any snapshot. This should be called periodically to clean +// up data from deleted or incomplete snapshots. +func (sm *SnapshotManager) CleanupOrphanedData(ctx context.Context) error { + // Order is important to respect foreign key constraints: + // 1. Delete orphaned files (will cascade delete file_chunks) + // 2. Delete orphaned blobs (will cascade delete blob_chunks for deleted blobs) + // 3. Delete orphaned blob_chunks (where blob exists but chunk doesn't) + // 4. Delete orphaned chunks (now safe after all blob_chunks are gone) + + // Delete orphaned files (files not in any snapshot) + log.Debug("Deleting orphaned file records from database") + + err := sm.repos.Files.DeleteOrphaned(ctx) + if err != nil { + return fmt.Errorf("deleting orphaned files: %w", err) + } + + // Delete orphaned blobs (blobs not in any snapshot) + // This will cascade delete blob_chunks for deleted blobs + log.Debug("Deleting orphaned blob records from database") + + err = sm.repos.Blobs.DeleteOrphaned(ctx) + if err != nil { + return fmt.Errorf("deleting orphaned blobs: %w", err) + } + + // Delete orphaned blob_chunks entries + // This handles cases where the blob still exists but chunks were deleted + log.Debug("Deleting orphaned blob_chunks associations from database") + + err = sm.repos.BlobChunks.DeleteOrphaned(ctx) + if err != nil { + return fmt.Errorf("deleting orphaned blob_chunks: %w", err) + } + + // Delete orphaned chunks (chunks not referenced by any file) + // This must come after cleaning up blob_chunks to avoid foreign key violations + log.Debug("Deleting orphaned chunk records from database") + + err = sm.repos.Chunks.DeleteOrphaned(ctx) + if err != nil { + return fmt.Errorf("deleting orphaned chunks: %w", err) + } + + return nil +} + +// prepareExportDB copies, cleans, vacuums, and compresses the snapshot +// database for export. Returns the compressed data and the path to the +// temporary database (needed for manifest generation). +func (sm *SnapshotManager) prepareExportDB( + ctx context.Context, dbPath, snapshotID, tempDir string, +) ([]byte, string, error) { // Step 1: Copy database to temp file // The main database should be closed at this point tempDBPath := filepath.Join(tempDir, "snapshot.db") - log.Debug("Copying database to temporary location", "source", dbPath, "destination", tempDBPath) + log.Debug("Copying database to temporary location", + "source", dbPath, "destination", tempDBPath) - if err := sm.copyFile(dbPath, tempDBPath); err != nil { + err := sm.copyFile(dbPath, tempDBPath) + if err != nil { return nil, "", fmt.Errorf("copying database: %w", err) } @@ -294,31 +433,36 @@ func (sm *SnapshotManager) prepareExportDB(ctx context.Context, dbPath, snapshot log.Info("Temporary database cleanup complete", "db_path", tempDBPath, - "size_after_clean", humanize.Bytes(uint64(sm.getFileSize(tempDBPath))), + "size_after_clean", humanize.Bytes(safeUint64(sm.getFileSize(tempDBPath))), "files", stats.FileCount, "chunks", stats.ChunkCount, "blobs", stats.BlobCount, - "total_compressed_size", humanize.Bytes(uint64(stats.CompressedSize)), - "total_uncompressed_size", humanize.Bytes(uint64(stats.UncompressedSize)), - "compression_ratio", fmt.Sprintf("%.2fx", float64(stats.UncompressedSize)/float64(stats.CompressedSize))) + "total_compressed_size", humanize.Bytes(safeUint64(stats.CompressedSize)), + "total_uncompressed_size", humanize.Bytes(safeUint64(stats.UncompressedSize)), + "compression_ratio", fmt.Sprintf("%.2fx", + float64(stats.UncompressedSize)/float64(stats.CompressedSize))) // Step 3: VACUUM the database to remove deleted data and compact // This is critical for security - ensures no stale/deleted data is uploaded - if err := sm.vacuumDatabase(tempDBPath); err != nil { + err = sm.vacuumDatabase(ctx, tempDBPath) + if err != nil { return nil, "", fmt.Errorf("vacuuming database: %w", err) } - log.Debug("Database vacuumed", "size", humanize.Bytes(uint64(sm.getFileSize(tempDBPath)))) + log.Debug("Database vacuumed", + "size", humanize.Bytes(safeUint64(sm.getFileSize(tempDBPath)))) // Step 4: Compress and encrypt the binary database file compressedPath := filepath.Join(tempDir, "db.zst.age") - if err := sm.compressFile(tempDBPath, compressedPath); err != nil { + + err = sm.compressFile(tempDBPath, compressedPath) + if err != nil { return nil, "", fmt.Errorf("compressing database: %w", err) } log.Debug("Compression complete", - "original_size", humanize.Bytes(uint64(sm.getFileSize(tempDBPath))), - "compressed_size", humanize.Bytes(uint64(sm.getFileSize(compressedPath)))) + "original_size", humanize.Bytes(safeUint64(sm.getFileSize(tempDBPath))), + "compressed_size", humanize.Bytes(safeUint64(sm.getFileSize(compressedPath)))) // Step 5: Read compressed and encrypted data for upload finalData, err := afero.ReadFile(sm.fs, compressedPath) @@ -335,7 +479,9 @@ func (sm *SnapshotManager) prepareExportDB(ctx context.Context, dbPath, snapshot // We never write the human-readable snapshot ID into any unencrypted // part of remote storage so a listing of the destination bucket leaks // no host, configuration, or scheduling information. -func (sm *SnapshotManager) uploadSnapshotArtifacts(ctx context.Context, snapshotID string, dbData, manifestData []byte) error { +func (sm *SnapshotManager) uploadSnapshotArtifacts( + ctx context.Context, snapshotID string, dbData, manifestData []byte, +) error { remoteKey := RemoteSnapshotKey(snapshotID) // Upload database backup (compressed and encrypted) @@ -349,7 +495,8 @@ func (sm *SnapshotManager) uploadSnapshotArtifacts(ctx context.Context, snapshot } dbUploadDuration := time.Since(dbUploadStart) - dbUploadSpeed := float64(len(dbData)) * 8 / dbUploadDuration.Seconds() // bits per second + // bits per second + dbUploadSpeed := float64(len(dbData)) * bitsPerByte / dbUploadDuration.Seconds() log.Info("Uploaded snapshot database", "path", dbKey, "size", humanize.Bytes(uint64(len(dbData))), @@ -366,7 +513,9 @@ func (sm *SnapshotManager) uploadSnapshotArtifacts(ctx context.Context, snapshot } manifestUploadDuration := time.Since(manifestUploadStart) - manifestUploadSpeed := float64(len(manifestData)) * 8 / manifestUploadDuration.Seconds() // bits per second + // bits per second + manifestUploadSpeed := float64(len(manifestData)) * bitsPerByte / + manifestUploadDuration.Seconds() log.Info("Uploaded blob manifest", "path", manifestKey, "size", humanize.Bytes(uint64(len(manifestData))), @@ -388,16 +537,19 @@ type CleanupStats struct { // cleanSnapshotDB removes all data except for the specified snapshot // // The cleanup is performed in a specific order to maintain referential integrity: -// 1. Delete other snapshots -// 2. Delete orphaned snapshot associations (snapshot_files, snapshot_blobs) for deleted snapshots -// 3. Delete orphaned files (not in the current snapshot) -// 4. Delete orphaned chunk-to-file mappings (references to deleted files) -// 5. Delete orphaned blobs (not in the current snapshot) -// 6. Delete orphaned blob-to-chunk mappings (references to deleted chunks) -// 7. Delete orphaned chunks (not referenced by any file) +// 1. Delete other snapshots +// 2. Delete orphaned snapshot associations (snapshot_files, snapshot_blobs) +// for deleted snapshots +// 3. Delete orphaned files (not in the current snapshot) +// 4. Delete orphaned chunk-to-file mappings (references to deleted files) +// 5. Delete orphaned blobs (not in the current snapshot) +// 6. Delete orphaned blob-to-chunk mappings (references to deleted chunks) +// 7. Delete orphaned chunks (not referenced by any file) // // Each step is implemented as a separate method for clarity and maintainability. -func (sm *SnapshotManager) cleanSnapshotDB(ctx context.Context, dbPath string, snapshotID string) (*CleanupStats, error) { +func (sm *SnapshotManager) cleanSnapshotDB( + ctx context.Context, dbPath string, snapshotID string, +) (*CleanupStats, error) { // Open the temp database db, err := database.New(ctx, dbPath) if err != nil { @@ -423,48 +575,54 @@ func (sm *SnapshotManager) cleanSnapshotDB(ctx context.Context, dbPath string, s }() // Execute cleanup steps in order - if err := sm.deleteOtherSnapshots(ctx, tx, snapshotID); err != nil { - return nil, fmt.Errorf("step 1 - delete other snapshots: %w", err) + steps := []struct { + name string + fn func() error + }{ + {"delete other snapshots", + func() error { return sm.deleteOtherSnapshots(ctx, tx, snapshotID) }}, + {"delete orphaned snapshot associations", + func() error { return sm.deleteOrphanedSnapshotAssociations(ctx, tx, snapshotID) }}, + {"delete orphaned files", + func() error { return sm.deleteOrphanedFiles(ctx, tx, snapshotID) }}, + {"delete orphaned chunk-to-file mappings", + func() error { return sm.deleteOrphanedChunkToFileMappings(ctx, tx) }}, + {"delete orphaned blobs", + func() error { return sm.deleteOrphanedBlobs(ctx, tx, snapshotID) }}, + {"delete orphaned blob-to-chunk mappings", + func() error { return sm.deleteOrphanedBlobToChunkMappings(ctx, tx) }}, + {"delete orphaned chunks", + func() error { return sm.deleteOrphanedChunks(ctx, tx) }}, } - if err := sm.deleteOrphanedSnapshotAssociations(ctx, tx, snapshotID); err != nil { - return nil, fmt.Errorf("step 2 - delete orphaned snapshot associations: %w", err) - } - - if err := sm.deleteOrphanedFiles(ctx, tx, snapshotID); err != nil { - return nil, fmt.Errorf("step 3 - delete orphaned files: %w", err) - } - - if err := sm.deleteOrphanedChunkToFileMappings(ctx, tx); err != nil { - return nil, fmt.Errorf("step 4 - delete orphaned chunk-to-file mappings: %w", err) - } - - if err := sm.deleteOrphanedBlobs(ctx, tx, snapshotID); err != nil { - return nil, fmt.Errorf("step 5 - delete orphaned blobs: %w", err) - } - - if err := sm.deleteOrphanedBlobToChunkMappings(ctx, tx); err != nil { - return nil, fmt.Errorf("step 6 - delete orphaned blob-to-chunk mappings: %w", err) - } - - if err := sm.deleteOrphanedChunks(ctx, tx); err != nil { - return nil, fmt.Errorf("step 7 - delete orphaned chunks: %w", err) + for i, step := range steps { + err = step.fn() + if err != nil { + return nil, fmt.Errorf("step %d - %s: %w", i+1, step.name, err) + } } // Commit transaction log.Debug("[Temp DB Cleanup] Committing cleanup transaction") - if err := tx.Commit(); err != nil { + err = tx.Commit() + if err != nil { return nil, fmt.Errorf("committing transaction: %w", err) } - // Collect statistics about the cleaned database + return sm.collectCleanupStats(ctx, db, snapshotID) +} + +// collectCleanupStats gathers statistics about the cleaned database. +func (sm *SnapshotManager) collectCleanupStats( + ctx context.Context, db *database.DB, snapshotID string, +) (*CleanupStats, error) { stats := &CleanupStats{} // Count files var fileCount int - err = db.QueryRowWithLog(ctx, "SELECT COUNT(*) FROM files").Scan(&fileCount) + err := db.QueryRowWithLog(ctx, "SELECT COUNT(*) FROM files").Scan(&fileCount) if err != nil { return nil, fmt.Errorf("counting files: %w", err) } @@ -488,9 +646,12 @@ func (sm *SnapshotManager) cleanSnapshotDB(ctx context.Context, dbPath string, s ) err = db.QueryRowWithLog(ctx, ` - SELECT COUNT(*), COALESCE(SUM(compressed_size), 0), COALESCE(SUM(uncompressed_size), 0) - FROM blobs - WHERE blob_hash IN (SELECT blob_hash FROM snapshot_blobs WHERE snapshot_id = ?) + SELECT COUNT(*), + COALESCE(SUM(compressed_size), 0), + COALESCE(SUM(uncompressed_size), 0) + FROM blobs + WHERE blob_hash IN + (SELECT blob_hash FROM snapshot_blobs WHERE snapshot_id = ?) `, snapshotID).Scan(&blobCount, &compressedSize, &uncompressedSize) if err != nil { return nil, fmt.Errorf("counting blobs and sizes: %w", err) @@ -505,11 +666,13 @@ func (sm *SnapshotManager) cleanSnapshotDB(ctx context.Context, dbPath string, s // vacuumDatabase runs VACUUM on the database to remove deleted data and compact // This is critical for security - ensures no stale/deleted data pages are uploaded -func (sm *SnapshotManager) vacuumDatabase(dbPath string) error { +func (sm *SnapshotManager) vacuumDatabase(ctx context.Context, dbPath string) error { log.Debug("Running VACUUM on database", "path", dbPath) - cmd := exec.Command("sqlite3", dbPath, "VACUUM;") + //nolint:gosec // G204: fixed argv; dbPath is our own temp file path + cmd := exec.CommandContext(ctx, "sqlite3", dbPath, "VACUUM;") - if output, err := cmd.CombinedOutput(); err != nil { + output, err := cmd.CombinedOutput() + if err != nil { return fmt.Errorf("running VACUUM: %w (output: %s)", err, string(output)) } @@ -543,7 +706,8 @@ func (sm *SnapshotManager) compressFile(inputPath, outputPath string) error { // Use blobgen for compression and encryption log.Debug("Compressing and encrypting data") - writer, err := blobgen.NewWriter(output, sm.config.CompressionLevel, sm.config.AgeRecipients) + writer, err := blobgen.NewWriter(output, sm.config.CompressionLevel, + sm.config.AgeRecipients) if err != nil { return fmt.Errorf("creating blobgen writer: %w", err) } @@ -559,12 +723,14 @@ func (sm *SnapshotManager) compressFile(inputPath, outputPath string) error { } }() - if _, err := io.Copy(writer, input); err != nil { + _, err = io.Copy(writer, input) + if err != nil { return fmt.Errorf("compressing data: %w", err) } // Close writer to flush all data - if err := writer.Close(); err != nil { + err = writer.Close() + if err != nil { return fmt.Errorf("closing writer: %w", err) } @@ -620,7 +786,9 @@ func (sm *SnapshotManager) copyFile(src, dst string) error { } // generateBlobManifest creates a compressed JSON list of all blobs in the snapshot -func (sm *SnapshotManager) generateBlobManifest(ctx context.Context, dbPath string, snapshotID string) ([]byte, error) { +func (sm *SnapshotManager) generateBlobManifest( + ctx context.Context, dbPath string, snapshotID string, +) ([]byte, error) { // Open the cleaned database using the database package db, err := database.New(ctx, dbPath) if err != nil { @@ -718,61 +886,10 @@ type ExtendedBackupStats struct { UploadDurationMs int64 // Total milliseconds spent uploading to S3 } -// CleanupIncompleteSnapshots removes incomplete snapshots that don't have metadata in S3. -// This is critical for data safety: incomplete snapshots can cause deduplication to skip -// files that were never successfully backed up, resulting in data loss. -func (sm *SnapshotManager) CleanupIncompleteSnapshots(ctx context.Context, hostname string) error { - log.Info("Checking for incomplete snapshots", "hostname", hostname) - - // Get all incomplete snapshots for this hostname - incompleteSnapshots, err := sm.repos.Snapshots.GetIncompleteByHostname(ctx, hostname) - if err != nil { - return fmt.Errorf("getting incomplete snapshots: %w", err) - } - - if len(incompleteSnapshots) == 0 { - log.Debug("No incomplete snapshots found") - - return nil - } - - log.Info("Found incomplete snapshots", "count", len(incompleteSnapshots)) - - // Check each incomplete snapshot for metadata in storage - for _, snapshot := range incompleteSnapshots { - // Check if metadata exists in storage (paths use the hashed - // remote key so we don't leak host info to the listing). - metadataKey := fmt.Sprintf("metadata/%s/db.zst", RemoteSnapshotKey(snapshot.ID.String())) - - _, err := sm.storage.Stat(ctx, metadataKey) - if err != nil { - // Metadata doesn't exist in S3 - this is an incomplete snapshot - log.Info("Cleaning up incomplete snapshot record", "snapshot_id", snapshot.ID, "started_at", snapshot.StartedAt) - - // Delete the snapshot and all its associations - err := sm.deleteSnapshot(ctx, snapshot.ID.String()) - if err != nil { - return fmt.Errorf("deleting incomplete snapshot %s: %w", snapshot.ID, err) - } - - log.Info("Deleted incomplete snapshot record and associated data", "snapshot_id", snapshot.ID) - } else { - // Metadata exists - this snapshot was completed but database wasn't updated - // This shouldn't happen in normal operation, but mark it complete - log.Warn("Found snapshot with remote metadata but incomplete in database", "snapshot_id", snapshot.ID) - - err := sm.repos.Snapshots.MarkComplete(ctx, nil, snapshot.ID.String()) - if err != nil { - log.Error("Failed to mark snapshot as complete in database", "snapshot_id", snapshot.ID, "error", err) - } - } - } - - return nil -} - // deleteSnapshot removes a snapshot and all its associations from the database -func (sm *SnapshotManager) deleteSnapshot(ctx context.Context, snapshotID string) error { +func (sm *SnapshotManager) deleteSnapshot( + ctx context.Context, snapshotID string, +) error { // Delete snapshot_files entries err := sm.repos.Snapshots.DeleteSnapshotFiles(ctx, snapshotID) if err != nil { @@ -808,61 +925,20 @@ func (sm *SnapshotManager) deleteSnapshot(ctx context.Context, snapshotID string return nil } -// CleanupOrphanedData removes files, chunks, and blobs that are no longer referenced by any snapshot. -// This should be called periodically to clean up data from deleted or incomplete snapshots. -func (sm *SnapshotManager) CleanupOrphanedData(ctx context.Context) error { - // Order is important to respect foreign key constraints: - // 1. Delete orphaned files (will cascade delete file_chunks) - // 2. Delete orphaned blobs (will cascade delete blob_chunks for deleted blobs) - // 3. Delete orphaned blob_chunks (where blob exists but chunk doesn't) - // 4. Delete orphaned chunks (now safe after all blob_chunks are gone) - - // Delete orphaned files (files not in any snapshot) - log.Debug("Deleting orphaned file records from database") - - err := sm.repos.Files.DeleteOrphaned(ctx) - if err != nil { - return fmt.Errorf("deleting orphaned files: %w", err) - } - - // Delete orphaned blobs (blobs not in any snapshot) - // This will cascade delete blob_chunks for deleted blobs - log.Debug("Deleting orphaned blob records from database") - - err = sm.repos.Blobs.DeleteOrphaned(ctx) - if err != nil { - return fmt.Errorf("deleting orphaned blobs: %w", err) - } - - // Delete orphaned blob_chunks entries - // This handles cases where the blob still exists but chunks were deleted - log.Debug("Deleting orphaned blob_chunks associations from database") - - err = sm.repos.BlobChunks.DeleteOrphaned(ctx) - if err != nil { - return fmt.Errorf("deleting orphaned blob_chunks: %w", err) - } - - // Delete orphaned chunks (chunks not referenced by any file) - // This must come after cleaning up blob_chunks to avoid foreign key violations - log.Debug("Deleting orphaned chunk records from database") - - err = sm.repos.Chunks.DeleteOrphaned(ctx) - if err != nil { - return fmt.Errorf("deleting orphaned chunks: %w", err) - } - - return nil -} - // deleteOtherSnapshots deletes all snapshots except the current one -func (sm *SnapshotManager) deleteOtherSnapshots(ctx context.Context, tx *sql.Tx, currentSnapshotID string) error { - log.Debug("[Temp DB Cleanup] Deleting all snapshot records except current", "keeping", currentSnapshotID) +func (sm *SnapshotManager) deleteOtherSnapshots( + ctx context.Context, tx *sql.Tx, currentSnapshotID string, +) error { + log.Debug("[Temp DB Cleanup] Deleting all snapshot records except current", + "keeping", currentSnapshotID) - // First delete uploads that reference other snapshots (no CASCADE DELETE on this FK) - database.LogSQL("Execute", "DELETE FROM uploads WHERE snapshot_id != ?", currentSnapshotID) + // First delete uploads that reference other snapshots (no CASCADE DELETE + // on this FK) + database.LogSQL("Execute", "DELETE FROM uploads WHERE snapshot_id != ?", + currentSnapshotID) - uploadResult, err := tx.ExecContext(ctx, "DELETE FROM uploads WHERE snapshot_id != ?", currentSnapshotID) + uploadResult, err := tx.ExecContext(ctx, + "DELETE FROM uploads WHERE snapshot_id != ?", currentSnapshotID) if err != nil { return fmt.Errorf("deleting uploads for other snapshots: %w", err) } @@ -871,66 +947,84 @@ func (sm *SnapshotManager) deleteOtherSnapshots(ctx context.Context, tx *sql.Tx, log.Debug("[Temp DB Cleanup] Deleted upload records", "count", uploadsDeleted) // Now we can safely delete the snapshots - database.LogSQL("Execute", "DELETE FROM snapshots WHERE id != ?", currentSnapshotID) + database.LogSQL("Execute", "DELETE FROM snapshots WHERE id != ?", + currentSnapshotID) - result, err := tx.ExecContext(ctx, "DELETE FROM snapshots WHERE id != ?", currentSnapshotID) + result, err := tx.ExecContext(ctx, "DELETE FROM snapshots WHERE id != ?", + currentSnapshotID) if err != nil { return fmt.Errorf("deleting other snapshots: %w", err) } rowsAffected, _ := result.RowsAffected() - log.Debug("[Temp DB Cleanup] Deleted snapshot records from database", "count", rowsAffected) + log.Debug("[Temp DB Cleanup] Deleted snapshot records from database", + "count", rowsAffected) return nil } -// deleteOrphanedSnapshotAssociations deletes snapshot_files and snapshot_blobs for deleted snapshots -func (sm *SnapshotManager) deleteOrphanedSnapshotAssociations(ctx context.Context, tx *sql.Tx, currentSnapshotID string) error { +// deleteOrphanedSnapshotAssociations deletes snapshot_files and +// snapshot_blobs for deleted snapshots +func (sm *SnapshotManager) deleteOrphanedSnapshotAssociations( + ctx context.Context, tx *sql.Tx, currentSnapshotID string, +) error { // Delete orphaned snapshot_files log.Debug("[Temp DB Cleanup] Deleting orphaned snapshot_files associations") - database.LogSQL("Execute", "DELETE FROM snapshot_files WHERE snapshot_id != ?", currentSnapshotID) + database.LogSQL("Execute", "DELETE FROM snapshot_files WHERE snapshot_id != ?", + currentSnapshotID) - result, err := tx.ExecContext(ctx, "DELETE FROM snapshot_files WHERE snapshot_id != ?", currentSnapshotID) + result, err := tx.ExecContext(ctx, + "DELETE FROM snapshot_files WHERE snapshot_id != ?", currentSnapshotID) if err != nil { return fmt.Errorf("deleting orphaned snapshot_files: %w", err) } rowsAffected, _ := result.RowsAffected() - log.Debug("[Temp DB Cleanup] Deleted snapshot_files associations", "count", rowsAffected) + log.Debug("[Temp DB Cleanup] Deleted snapshot_files associations", + "count", rowsAffected) // Delete orphaned snapshot_blobs log.Debug("[Temp DB Cleanup] Deleting orphaned snapshot_blobs associations") - database.LogSQL("Execute", "DELETE FROM snapshot_blobs WHERE snapshot_id != ?", currentSnapshotID) + database.LogSQL("Execute", "DELETE FROM snapshot_blobs WHERE snapshot_id != ?", + currentSnapshotID) - result, err = tx.ExecContext(ctx, "DELETE FROM snapshot_blobs WHERE snapshot_id != ?", currentSnapshotID) + result, err = tx.ExecContext(ctx, + "DELETE FROM snapshot_blobs WHERE snapshot_id != ?", currentSnapshotID) if err != nil { return fmt.Errorf("deleting orphaned snapshot_blobs: %w", err) } rowsAffected, _ = result.RowsAffected() - log.Debug("[Temp DB Cleanup] Deleted snapshot_blobs associations", "count", rowsAffected) + log.Debug("[Temp DB Cleanup] Deleted snapshot_blobs associations", + "count", rowsAffected) return nil } // deleteOrphanedFiles deletes files not in the current snapshot -func (sm *SnapshotManager) deleteOrphanedFiles(ctx context.Context, tx *sql.Tx, currentSnapshotID string) error { - log.Debug("[Temp DB Cleanup] Deleting file records not referenced by current snapshot") - database.LogSQL("Execute", `DELETE FROM files WHERE NOT EXISTS (SELECT 1 FROM snapshot_files WHERE snapshot_files.file_id = files.id AND snapshot_files.snapshot_id = ?)`, currentSnapshotID) +func (sm *SnapshotManager) deleteOrphanedFiles( + ctx context.Context, tx *sql.Tx, currentSnapshotID string, +) error { + log.Debug( + "[Temp DB Cleanup] Deleting file records not referenced by current snapshot") - result, err := tx.ExecContext(ctx, ` - DELETE FROM files + query := ` + DELETE FROM files WHERE NOT EXISTS ( - SELECT 1 FROM snapshot_files - WHERE snapshot_files.file_id = files.id + SELECT 1 FROM snapshot_files + WHERE snapshot_files.file_id = files.id AND snapshot_files.snapshot_id = ? - )`, currentSnapshotID) + )` + database.LogSQL("Execute", query, currentSnapshotID) + + result, err := tx.ExecContext(ctx, query, currentSnapshotID) if err != nil { return fmt.Errorf("deleting orphaned files: %w", err) } rowsAffected, _ := result.RowsAffected() - log.Debug("[Temp DB Cleanup] Deleted file records from database", "count", rowsAffected) + log.Debug("[Temp DB Cleanup] Deleted file records from database", + "count", rowsAffected) // Note: file_chunks will be deleted via CASCADE log.Debug("[Temp DB Cleanup] file_chunks associations deleted via CASCADE") @@ -939,65 +1033,81 @@ func (sm *SnapshotManager) deleteOrphanedFiles(ctx context.Context, tx *sql.Tx, } // deleteOrphanedChunkToFileMappings deletes chunk_files entries for deleted files -func (sm *SnapshotManager) deleteOrphanedChunkToFileMappings(ctx context.Context, tx *sql.Tx) error { +func (sm *SnapshotManager) deleteOrphanedChunkToFileMappings( + ctx context.Context, tx *sql.Tx, +) error { log.Debug("[Temp DB Cleanup] Deleting orphaned chunk_files associations") - database.LogSQL("Execute", `DELETE FROM chunk_files WHERE NOT EXISTS (SELECT 1 FROM files WHERE files.id = chunk_files.file_id)`) - result, err := tx.ExecContext(ctx, ` - DELETE FROM chunk_files + query := ` + DELETE FROM chunk_files WHERE NOT EXISTS ( - SELECT 1 FROM files + SELECT 1 FROM files WHERE files.id = chunk_files.file_id - )`) + )` + database.LogSQL("Execute", query) + + result, err := tx.ExecContext(ctx, query) if err != nil { return fmt.Errorf("deleting orphaned chunk_files: %w", err) } rowsAffected, _ := result.RowsAffected() - log.Debug("[Temp DB Cleanup] Deleted chunk_files associations", "count", rowsAffected) + log.Debug("[Temp DB Cleanup] Deleted chunk_files associations", + "count", rowsAffected) return nil } // deleteOrphanedBlobs deletes blobs not in the current snapshot -func (sm *SnapshotManager) deleteOrphanedBlobs(ctx context.Context, tx *sql.Tx, currentSnapshotID string) error { - log.Debug("[Temp DB Cleanup] Deleting blob records not referenced by current snapshot") - database.LogSQL("Execute", `DELETE FROM blobs WHERE NOT EXISTS (SELECT 1 FROM snapshot_blobs WHERE snapshot_blobs.blob_hash = blobs.blob_hash AND snapshot_blobs.snapshot_id = ?)`, currentSnapshotID) +func (sm *SnapshotManager) deleteOrphanedBlobs( + ctx context.Context, tx *sql.Tx, currentSnapshotID string, +) error { + log.Debug( + "[Temp DB Cleanup] Deleting blob records not referenced by current snapshot") - result, err := tx.ExecContext(ctx, ` - DELETE FROM blobs + query := ` + DELETE FROM blobs WHERE NOT EXISTS ( - SELECT 1 FROM snapshot_blobs - WHERE snapshot_blobs.blob_hash = blobs.blob_hash + SELECT 1 FROM snapshot_blobs + WHERE snapshot_blobs.blob_hash = blobs.blob_hash AND snapshot_blobs.snapshot_id = ? - )`, currentSnapshotID) + )` + database.LogSQL("Execute", query, currentSnapshotID) + + result, err := tx.ExecContext(ctx, query, currentSnapshotID) if err != nil { return fmt.Errorf("deleting orphaned blobs: %w", err) } rowsAffected, _ := result.RowsAffected() - log.Debug("[Temp DB Cleanup] Deleted blob records from database", "count", rowsAffected) + log.Debug("[Temp DB Cleanup] Deleted blob records from database", + "count", rowsAffected) return nil } // deleteOrphanedBlobToChunkMappings deletes blob_chunks entries for deleted blobs -func (sm *SnapshotManager) deleteOrphanedBlobToChunkMappings(ctx context.Context, tx *sql.Tx) error { +func (sm *SnapshotManager) deleteOrphanedBlobToChunkMappings( + ctx context.Context, tx *sql.Tx, +) error { log.Debug("[Temp DB Cleanup] Deleting orphaned blob_chunks associations") - database.LogSQL("Execute", `DELETE FROM blob_chunks WHERE NOT EXISTS (SELECT 1 FROM blobs WHERE blobs.id = blob_chunks.blob_id)`) - result, err := tx.ExecContext(ctx, ` - DELETE FROM blob_chunks + query := ` + DELETE FROM blob_chunks WHERE NOT EXISTS ( - SELECT 1 FROM blobs + SELECT 1 FROM blobs WHERE blobs.id = blob_chunks.blob_id - )`) + )` + database.LogSQL("Execute", query) + + result, err := tx.ExecContext(ctx, query) if err != nil { return fmt.Errorf("deleting orphaned blob_chunks: %w", err) } rowsAffected, _ := result.RowsAffected() - log.Debug("[Temp DB Cleanup] Deleted blob_chunks associations", "count", rowsAffected) + log.Debug("[Temp DB Cleanup] Deleted blob_chunks associations", + "count", rowsAffected) return nil } @@ -1024,7 +1134,8 @@ func (sm *SnapshotManager) deleteOrphanedChunks(ctx context.Context, tx *sql.Tx) } rowsAffected, _ := result.RowsAffected() - log.Debug("[Temp DB Cleanup] Deleted chunk records from database", "count", rowsAffected) + log.Debug("[Temp DB Cleanup] Deleted chunk records from database", + "count", rowsAffected) return nil } diff --git a/internal/snapshot/snapshot_test.go b/internal/snapshot/snapshot_test.go index 969aeed..fd83d72 100644 --- a/internal/snapshot/snapshot_test.go +++ b/internal/snapshot/snapshot_test.go @@ -1,3 +1,4 @@ +//nolint:testpackage // exercises unexported SnapshotManager internals package snapshot import ( @@ -37,9 +38,65 @@ func copyFile(fs afero.Fs, src, dst string) error { return err } +// verifyCleanedDB opens the cleaned database and checks that the kept +// snapshot survived while the orphan file and chunk were removed. +func verifyCleanedDB( + ctx context.Context, + t *testing.T, + tempDBPath, snapshotID string, + file *database.File, + chunk *database.Chunk, +) { + t.Helper() + + cleanedDB, err := database.New(ctx, tempDBPath) + if err != nil { + t.Fatalf("failed to open cleaned database: %v", err) + } + defer func() { + err := cleanedDB.Close() + if err != nil { + t.Errorf("failed to close database: %v", err) + } + }() + + cleanedRepos := database.NewRepositories(cleanedDB) + + // Verify snapshot exists + verifySnapshot, err := cleanedRepos.Snapshots.GetByID(ctx, snapshotID) + if err != nil { + t.Fatalf("failed to get snapshot: %v", err) + } + + if verifySnapshot == nil { + t.Error("snapshot should exist") + } + + // Verify orphan file is gone + f, err := cleanedRepos.Files.GetByPath(ctx, file.Path.String()) + if err != nil { + t.Fatalf("failed to check file: %v", err) + } + + if f != nil { + t.Error("orphan file should not exist") + } + + // Verify orphan chunk is gone + c, err := cleanedRepos.Chunks.GetByHash(ctx, chunk.ChunkHash.String()) + if err != nil { + t.Fatalf("failed to check chunk: %v", err) + } + + if c != nil { + t.Error("orphan chunk should not exist") + } +} + func TestCleanSnapshotDBEmptySnapshot(t *testing.T) { // Initialize logger log.Initialize(log.Config{}) + t.Parallel() ctx := context.Background() fs := afero.NewOsFs() @@ -85,13 +142,16 @@ func TestCleanSnapshotDBEmptySnapshot(t *testing.T) { } // Close the database - if err := db.Close(); err != nil { + err = db.Close() + if err != nil { t.Fatalf("failed to close database: %v", err) } // Copy database tempDBPath := filepath.Join(tempDir, "temp.db") - if err := copyFile(fs, dbPath, tempDBPath); err != nil { + + err = copyFile(fs, dbPath, tempDBPath) + if err != nil { t.Fatalf("failed to copy database: %v", err) } @@ -105,58 +165,20 @@ func TestCleanSnapshotDBEmptySnapshot(t *testing.T) { config: cfg, fs: fs, } - if _, err := sm.cleanSnapshotDB(ctx, tempDBPath, snapshot.ID.String()); err != nil { + + _, err = sm.cleanSnapshotDB(ctx, tempDBPath, snapshot.ID.String()) + if err != nil { t.Fatalf("failed to clean snapshot database: %v", err) } // Verify the cleaned database - cleanedDB, err := database.New(ctx, tempDBPath) - if err != nil { - t.Fatalf("failed to open cleaned database: %v", err) - } - defer func() { - err := cleanedDB.Close() - if err != nil { - t.Errorf("failed to close database: %v", err) - } - }() - - cleanedRepos := database.NewRepositories(cleanedDB) - - // Verify snapshot exists - verifySnapshot, err := cleanedRepos.Snapshots.GetByID(ctx, snapshot.ID.String()) - if err != nil { - t.Fatalf("failed to get snapshot: %v", err) - } - - if verifySnapshot == nil { - t.Error("snapshot should exist") - } - - // Verify orphan file is gone - f, err := cleanedRepos.Files.GetByPath(ctx, file.Path.String()) - if err != nil { - t.Fatalf("failed to check file: %v", err) - } - - if f != nil { - t.Error("orphan file should not exist") - } - - // Verify orphan chunk is gone - c, err := cleanedRepos.Chunks.GetByHash(ctx, chunk.ChunkHash.String()) - if err != nil { - t.Fatalf("failed to check chunk: %v", err) - } - - if c != nil { - t.Error("orphan chunk should not exist") - } + verifyCleanedDB(ctx, t, tempDBPath, snapshot.ID.String(), file, chunk) } func TestCleanSnapshotDBNonExistentSnapshot(t *testing.T) { // Initialize logger log.Initialize(log.Config{}) + t.Parallel() ctx := context.Background() fs := afero.NewOsFs() @@ -171,13 +193,16 @@ func TestCleanSnapshotDBNonExistentSnapshot(t *testing.T) { } // Close immediately - if err := db.Close(); err != nil { + err = db.Close() + if err != nil { t.Fatalf("failed to close database: %v", err) } // Copy database tempDBPath := filepath.Join(tempDir, "temp.db") - if err := copyFile(fs, dbPath, tempDBPath); err != nil { + + err = copyFile(fs, dbPath, tempDBPath) + if err != nil { t.Fatalf("failed to copy database: %v", err) } diff --git a/internal/storage/file.go b/internal/storage/file.go index 2d2ecf0..1dab239 100644 --- a/internal/storage/file.go +++ b/internal/storage/file.go @@ -42,18 +42,19 @@ func (f *FileStorer) SetFilesystem(fs afero.Fs) { f.fs = fs } -// fullPath returns the full filesystem path for a key. -func (f *FileStorer) fullPath(key string) string { - return filepath.Join(f.basePath, key) -} +// storageDirPerm is the mode used for directories created under the +// storage base path. +const storageDirPerm = 0o755 // Put stores data at the specified key. -func (f *FileStorer) Put(ctx context.Context, key string, data io.Reader) error { +func (f *FileStorer) Put(_ context.Context, key string, data io.Reader) error { path := f.fullPath(key) // Create parent directories dir := filepath.Dir(path) - if err := f.fs.MkdirAll(dir, 0755); err != nil { + + err := f.fs.MkdirAll(dir, storageDirPerm) + if err != nil { return fmt.Errorf("creating directories: %w", err) } @@ -63,7 +64,8 @@ func (f *FileStorer) Put(ctx context.Context, key string, data io.Reader) error } defer func() { _ = file.Close() }() - if _, err := io.Copy(file, data); err != nil { + _, err = io.Copy(file, data) + if err != nil { return fmt.Errorf("writing file: %w", err) } @@ -71,12 +73,17 @@ func (f *FileStorer) Put(ctx context.Context, key string, data io.Reader) error } // PutWithProgress stores data with progress reporting. -func (f *FileStorer) PutWithProgress(ctx context.Context, key string, data io.Reader, size int64, progress ProgressCallback) error { +func (f *FileStorer) PutWithProgress( + _ context.Context, key string, data io.Reader, + _ int64, progress ProgressCallback, +) error { path := f.fullPath(key) // Create parent directories dir := filepath.Dir(path) - if err := f.fs.MkdirAll(dir, 0755); err != nil { + + err := f.fs.MkdirAll(dir, storageDirPerm) + if err != nil { return fmt.Errorf("creating directories: %w", err) } @@ -92,7 +99,8 @@ func (f *FileStorer) PutWithProgress(ctx context.Context, key string, data io.Re callback: progress, } - if _, err := io.Copy(pw, data); err != nil { + _, err = io.Copy(pw, data) + if err != nil { return fmt.Errorf("writing file: %w", err) } @@ -100,7 +108,7 @@ func (f *FileStorer) PutWithProgress(ctx context.Context, key string, data io.Re } // Get retrieves data from the specified key. -func (f *FileStorer) Get(ctx context.Context, key string) (io.ReadCloser, error) { +func (f *FileStorer) Get(_ context.Context, key string) (io.ReadCloser, error) { path := f.fullPath(key) file, err := f.fs.Open(path) @@ -116,7 +124,7 @@ func (f *FileStorer) Get(ctx context.Context, key string) (io.ReadCloser, error) } // Stat returns metadata about an object without retrieving its contents. -func (f *FileStorer) Stat(ctx context.Context, key string) (*ObjectInfo, error) { +func (f *FileStorer) Stat(_ context.Context, key string) (*ObjectInfo, error) { path := f.fullPath(key) info, err := f.fs.Stat(path) @@ -135,7 +143,7 @@ func (f *FileStorer) Stat(ctx context.Context, key string) (*ObjectInfo, error) } // Delete removes an object. -func (f *FileStorer) Delete(ctx context.Context, key string) error { +func (f *FileStorer) Delete(_ context.Context, key string) error { path := f.fullPath(key) err := f.fs.Remove(path) @@ -231,7 +239,7 @@ func (f *FileStorer) ListStream(ctx context.Context, prefix string) <-chan Objec if err != nil { ch <- ObjectInfo{Err: err} - return nil // Continue walking despite errors + return nil //nolint:nilerr // continue walking despite errors } if !info.IsDir() { @@ -257,13 +265,18 @@ func (f *FileStorer) ListStream(ctx context.Context, prefix string) <-chan Objec } // Info returns human-readable storage location information. -func (f *FileStorer) Info() StorageInfo { - return StorageInfo{ - Type: "file", +func (f *FileStorer) Info() Info { + return Info{ + Type: schemeFile, Location: f.basePath, } } +// fullPath returns the full filesystem path for a key. +func (f *FileStorer) fullPath(key string) string { + return filepath.Join(f.basePath, key) +} + // progressWriter wraps an io.Writer to track write progress. type progressWriter struct { writer io.Writer diff --git a/internal/storage/module.go b/internal/storage/module.go index 42cc28a..8699a7c 100644 --- a/internal/storage/module.go +++ b/internal/storage/module.go @@ -10,9 +10,17 @@ import ( "sneak.berlin/go/vaultik/internal/s3" ) +// defaultS3Region is used when neither the URL nor the config specify one. +const defaultS3Region = "us-east-1" + +// defaultS3Endpoint is the AWS endpoint used when none is configured. +const defaultS3Endpoint = "s3.amazonaws.com" + // Module exports storage functionality as an fx module. // It provides a Storer implementation based on the configured storage URL // or falls back to legacy S3 configuration. +// +//nolint:gochecknoglobals // fx module definitions are package globals var Module = fx.Module("storage", fx.Provide(NewStorer), ) @@ -20,6 +28,8 @@ var Module = fx.Module("storage", // NewStorer creates a Storer based on configuration. // If StorageURL is set, it uses URL-based configuration. // Otherwise, it falls back to legacy S3 configuration. +// +//nolint:ireturn // fx provider intentionally returns the Storer interface func NewStorer(cfg *config.Config) (Storer, error) { if cfg.StorageURL != "" { return storerFromURL(cfg.StorageURL, cfg) @@ -28,6 +38,7 @@ func NewStorer(cfg *config.Config) (Storer, error) { return storerFromLegacyS3Config(cfg) } +//nolint:ireturn // factory intentionally returns the Storer interface func storerFromURL(rawURL string, cfg *config.Config) (Storer, error) { parsed, err := ParseStorageURL(rawURL) if err != nil { @@ -35,59 +46,74 @@ func storerFromURL(rawURL string, cfg *config.Config) (Storer, error) { } switch parsed.Scheme { - case "file": + case schemeFile: return NewFileStorer(parsed.Prefix) - case "s3": - // Build endpoint URL - endpoint := parsed.Endpoint - if endpoint == "" { - endpoint = "s3.amazonaws.com" - } + case schemeS3: + return storerFromParsedS3URL(parsed, cfg) - // Add protocol if not present - if parsed.UseSSL && !strings.HasPrefix(endpoint, "https://") && !strings.HasPrefix(endpoint, "http://") { - endpoint = "https://" + endpoint - } else if !parsed.UseSSL && !strings.HasPrefix(endpoint, "http://") && !strings.HasPrefix(endpoint, "https://") { - endpoint = "http://" + endpoint - } - - region := parsed.Region - if region == "" { - region = cfg.S3.Region - if region == "" { - region = "us-east-1" - } - } - - // Credentials come from config (not URL for security) - client, err := s3.NewClient(context.Background(), s3.Config{ - Endpoint: endpoint, - Bucket: parsed.Bucket, - Prefix: parsed.Prefix, - AccessKeyID: cfg.S3.AccessKeyID, - SecretAccessKey: cfg.S3.SecretAccessKey, - Region: region, - }) - if err != nil { - return nil, fmt.Errorf("creating S3 client: %w", err) - } - - return NewS3Storer(client), nil - - case "rclone": - return NewRcloneStorer(context.Background(), parsed.RcloneRemote, parsed.Prefix) + case schemeRclone: + return NewRcloneStorer( + context.Background(), parsed.RcloneRemote, parsed.Prefix) default: - return nil, fmt.Errorf("unsupported storage scheme: %s", parsed.Scheme) + return nil, fmt.Errorf("%w: %s", ErrUnsupportedStorage, parsed.Scheme) } } +// storerFromParsedS3URL builds an S3 storer from a parsed s3:// URL, +// filling endpoint protocol and region defaults from the config. +// +//nolint:ireturn // factory intentionally returns the Storer interface +func storerFromParsedS3URL(parsed *URL, cfg *config.Config) (Storer, error) { + // Build endpoint URL + endpoint := parsed.Endpoint + if endpoint == "" { + endpoint = defaultS3Endpoint + } + + // Add protocol if not present + hasProtocol := strings.HasPrefix(endpoint, "https://") || + strings.HasPrefix(endpoint, "http://") + if !hasProtocol { + if parsed.UseSSL { + endpoint = "https://" + endpoint + } else { + endpoint = "http://" + endpoint + } + } + + region := parsed.Region + if region == "" { + region = cfg.S3.Region + if region == "" { + region = defaultS3Region + } + } + + // Credentials come from config (not URL for security) + client, err := s3.NewClient(context.Background(), s3.Config{ + Endpoint: endpoint, + Bucket: parsed.Bucket, + Prefix: parsed.Prefix, + AccessKeyID: cfg.S3.AccessKeyID, + SecretAccessKey: cfg.S3.SecretAccessKey, + Region: region, + }) + if err != nil { + return nil, fmt.Errorf("creating S3 client: %w", err) + } + + return NewS3Storer(client), nil +} + +//nolint:ireturn // factory intentionally returns the Storer interface func storerFromLegacyS3Config(cfg *config.Config) (Storer, error) { endpoint := cfg.S3.Endpoint // Ensure protocol is present - if !strings.HasPrefix(endpoint, "http://") && !strings.HasPrefix(endpoint, "https://") { + if !strings.HasPrefix(endpoint, "http://") && + !strings.HasPrefix(endpoint, "https://") { if cfg.S3.UseSSL { endpoint = "https://" + endpoint } else { @@ -97,7 +123,7 @@ func storerFromLegacyS3Config(cfg *config.Config) (Storer, error) { region := cfg.S3.Region if region == "" { - region = "us-east-1" + region = defaultS3Region } client, err := s3.NewClient(context.Background(), s3.Config{ diff --git a/internal/storage/rclone.go b/internal/storage/rclone.go index 19d39d4..c3a3433 100644 --- a/internal/storage/rclone.go +++ b/internal/storage/rclone.go @@ -69,7 +69,8 @@ func (r *RcloneStorer) Put(ctx context.Context, key string, data io.Reader) erro } // Upload the object - _, err = operations.Rcat(ctx, r.fsys, key, io.NopCloser(bytes.NewReader(buf)), time.Now(), nil) + _, err = operations.Rcat(ctx, r.fsys, key, + io.NopCloser(bytes.NewReader(buf)), time.Now(), nil) if err != nil { return fmt.Errorf("uploading object: %w", err) } @@ -78,7 +79,10 @@ func (r *RcloneStorer) Put(ctx context.Context, key string, data io.Reader) erro } // PutWithProgress stores data with progress reporting. -func (r *RcloneStorer) PutWithProgress(ctx context.Context, key string, data io.Reader, size int64, progress ProgressCallback) error { +func (r *RcloneStorer) PutWithProgress( + ctx context.Context, key string, data io.Reader, + _ int64, progress ProgressCallback, +) error { // Wrap reader with progress tracking pr := &progressReader{ reader: data, @@ -155,7 +159,8 @@ func (r *RcloneStorer) Delete(ctx context.Context, key string) error { return fmt.Errorf("getting object: %w", err) } - if err := obj.Remove(ctx); err != nil { + err = obj.Remove(ctx) + if err != nil { return fmt.Errorf("removing object: %w", err) } @@ -180,7 +185,9 @@ func (r *RcloneStorer) List(ctx context.Context, prefix string) ([]string, error } // ListStream returns a channel of ObjectInfo for large result sets. -func (r *RcloneStorer) ListStream(ctx context.Context, prefix string) <-chan ObjectInfo { +func (r *RcloneStorer) ListStream( + ctx context.Context, prefix string, +) <-chan ObjectInfo { ch := make(chan ObjectInfo) go func() { @@ -211,14 +218,14 @@ func (r *RcloneStorer) ListStream(ctx context.Context, prefix string) <-chan Obj } // Info returns human-readable storage location information. -func (r *RcloneStorer) Info() StorageInfo { +func (r *RcloneStorer) Info() Info { location := r.remote if r.path != "" { location += ":" + r.path } - return StorageInfo{ - Type: "rclone", + return Info{ + Type: schemeRclone, Location: location, } } diff --git a/internal/storage/s3.go b/internal/storage/s3.go index 64b3ede..580ac80 100644 --- a/internal/storage/s3.go +++ b/internal/storage/s3.go @@ -24,7 +24,10 @@ func (s *S3Storer) Put(ctx context.Context, key string, data io.Reader) error { } // PutWithProgress stores data with progress reporting. -func (s *S3Storer) PutWithProgress(ctx context.Context, key string, data io.Reader, size int64, progress ProgressCallback) error { +func (s *S3Storer) PutWithProgress( + ctx context.Context, key string, data io.Reader, + size int64, progress ProgressCallback, +) error { // Convert storage.ProgressCallback to s3.ProgressCallback var s3Progress s3.ProgressCallback if progress != nil { @@ -81,8 +84,8 @@ func (s *S3Storer) ListStream(ctx context.Context, prefix string) <-chan ObjectI } // Info returns human-readable storage location information. -func (s *S3Storer) Info() StorageInfo { - return StorageInfo{ +func (s *S3Storer) Info() Info { + return Info{ Type: "s3", Location: fmt.Sprintf("%s/%s", s.client.Endpoint(), s.client.BucketName()), } diff --git a/internal/storage/storer.go b/internal/storage/storer.go index 9cd4e25..2334a7d 100644 --- a/internal/storage/storer.go +++ b/internal/storage/storer.go @@ -30,14 +30,15 @@ type ObjectInfo struct { Err error // Error for streaming results (nil on success) } -// StorageInfo provides human-readable storage configuration. -type StorageInfo struct { +// Info provides human-readable storage configuration. +type Info struct { Type string // "s3" or "file" Location string // endpoint/bucket for S3, base path for filesystem } // Storer defines the interface for storage backends. -// All paths are relative to the storage root (bucket/prefix for S3, base directory for filesystem). +// All paths are relative to the storage root (bucket/prefix for S3, base +// directory for filesystem). type Storer interface { // Put stores data at the specified key. // Parent directories are created automatically for filesystem backends. @@ -46,7 +47,8 @@ type Storer interface { // PutWithProgress stores data with progress reporting. // Size must be the exact size of the data to store. // The progress callback is called periodically with bytes transferred. - PutWithProgress(ctx context.Context, key string, data io.Reader, size int64, progress ProgressCallback) error + PutWithProgress(ctx context.Context, key string, data io.Reader, + size int64, progress ProgressCallback) error // Get retrieves data from the specified key. // The caller must close the returned ReadCloser. @@ -70,5 +72,5 @@ type Storer interface { ListStream(ctx context.Context, prefix string) <-chan ObjectInfo // Info returns human-readable storage location information. - Info() StorageInfo + Info() Info } diff --git a/internal/storage/url.go b/internal/storage/url.go index 4ab169d..1e3ddf2 100644 --- a/internal/storage/url.go +++ b/internal/storage/url.go @@ -7,8 +7,26 @@ import ( "strings" ) -// StorageURL represents a parsed storage URL. -type StorageURL struct { +// Storage URL scheme names. +const ( + schemeFile = "file" + schemeS3 = "s3" + schemeRclone = "rclone" +) + +// Sentinel errors for storage URL parsing. +var ( + ErrEmptyStorageURL = errors.New("storage URL is empty") + ErrEmptyFilePath = errors.New("file URL path is empty") + ErrMissingBucket = errors.New("s3 URL missing bucket name") + ErrMissingRemote = errors.New("rclone URL missing remote name") + ErrUnsupportedScheme = errors.New( + "unsupported URL scheme: must start with s3://, file://, or rclone://") + ErrUnsupportedStorage = errors.New("unsupported storage scheme") +) + +// URL represents a parsed storage URL. +type URL struct { Scheme string // "s3", "file", or "rclone" Bucket string // S3 bucket name (empty for file/rclone) Prefix string // Path within bucket or filesystem base path @@ -23,20 +41,20 @@ type StorageURL struct { // - s3://bucket/prefix?endpoint=host®ion=us-east-1&ssl=true // - file:///absolute/path/to/backup // - rclone://remote/path/to/backups -func ParseStorageURL(rawURL string) (*StorageURL, error) { +func ParseStorageURL(rawURL string) (*URL, error) { if rawURL == "" { - return nil, errors.New("storage URL is empty") + return nil, ErrEmptyStorageURL } // Handle file:// URLs if after, ok := strings.CutPrefix(rawURL, "file://"); ok { path := after if path == "" { - return nil, errors.New("file URL path is empty") + return nil, ErrEmptyFilePath } - return &StorageURL{ - Scheme: "file", + return &URL{ + Scheme: schemeFile, Prefix: path, }, nil } @@ -50,7 +68,7 @@ func ParseStorageURL(rawURL string) (*StorageURL, error) { bucket := u.Host if bucket == "" { - return nil, errors.New("s3 URL missing bucket name") + return nil, ErrMissingBucket } prefix := strings.TrimPrefix(u.Path, "/") @@ -62,8 +80,8 @@ func ParseStorageURL(rawURL string) (*StorageURL, error) { useSSL = false } - return &StorageURL{ - Scheme: "s3", + return &URL{ + Scheme: schemeS3, Bucket: bucket, Prefix: prefix, Endpoint: query.Get("endpoint"), @@ -81,27 +99,27 @@ func ParseStorageURL(rawURL string) (*StorageURL, error) { remote := u.Host if remote == "" { - return nil, errors.New("rclone URL missing remote name") + return nil, ErrMissingRemote } path := strings.TrimPrefix(u.Path, "/") - return &StorageURL{ - Scheme: "rclone", + return &URL{ + Scheme: schemeRclone, Prefix: path, RcloneRemote: remote, }, nil } - return nil, errors.New("unsupported URL scheme: must start with s3://, file://, or rclone://") + return nil, ErrUnsupportedScheme } // String returns a human-readable representation of the storage URL. -func (u *StorageURL) String() string { +func (u *URL) String() string { switch u.Scheme { - case "file": + case schemeFile: return "file://" + u.Prefix - case "s3": + case schemeS3: endpoint := u.Endpoint if endpoint == "" { endpoint = "s3.amazonaws.com" @@ -112,7 +130,7 @@ func (u *StorageURL) String() string { } return fmt.Sprintf("s3://%s (endpoint: %s)", u.Bucket, endpoint) - case "rclone": + case schemeRclone: if u.Prefix != "" { return fmt.Sprintf("rclone://%s/%s", u.RcloneRemote, u.Prefix) } diff --git a/internal/types/types.go b/internal/types/types.go index 9abc02c..c076f41 100644 --- a/internal/types/types.go +++ b/internal/types/types.go @@ -1,16 +1,26 @@ -// Package types provides custom types for better type safety across the vaultik codebase. -// Using distinct types for IDs, hashes, paths, and credentials prevents accidental -// mixing of semantically different values that happen to share the same underlying type. +// Package types provides custom types for better type safety across the +// vaultik codebase. Using distinct types for IDs, hashes, paths, and +// credentials prevents accidental mixing of semantically different values +// that happen to share the same underlying type. package types import ( "database/sql/driver" + "errors" "fmt" "github.com/google/uuid" ) +// errCannotScan is returned when a database value cannot be scanned into +// an ID type. +var errCannotScan = errors.New("cannot scan value") + // FileID is a UUID identifying a file record in the database. +// +// used on values. +// +//nolint:recvcheck // Scan requires a pointer receiver; String/Value are type FileID uuid.UUID // NewFileID generates a new random FileID. @@ -54,7 +64,7 @@ func (id *FileID) Scan(src any) error { case []byte: s = string(v) default: - return fmt.Errorf("cannot scan %T into FileID", src) + return fmt.Errorf("%w: %T into FileID", errCannotScan, src) } parsed, err := uuid.Parse(s) @@ -69,6 +79,10 @@ func (id *FileID) Scan(src any) error { // BlobID is a UUID identifying a blob record in the database. // This is distinct from BlobHash which is the content-addressed hash of the blob. +// +// used on values. +// +//nolint:recvcheck // Scan requires a pointer receiver; String/Value are type BlobID uuid.UUID // NewBlobID generates a new random BlobID. @@ -112,7 +126,7 @@ func (id *BlobID) Scan(src any) error { case []byte: s = string(v) default: - return fmt.Errorf("cannot scan %T into BlobID", src) + return fmt.Errorf("%w: %T into BlobID", errCannotScan, src) } parsed, err := uuid.Parse(s) @@ -208,6 +222,8 @@ func (p GlobPattern) String() string { return string(p) } func (k AgeSecretKey) String() string { return "[REDACTED]" } func (k AWSSecretAccessKey) String() string { return "[REDACTED]" } -// Raw returns the actual value for sensitive types when explicitly needed -func (k AgeSecretKey) Raw() string { return string(k) } +// Raw returns the actual value for sensitive types when explicitly needed. +func (k AgeSecretKey) Raw() string { return string(k) } + +// Raw returns the actual value for sensitive types when explicitly needed. func (k AWSSecretAccessKey) Raw() string { return string(k) } diff --git a/internal/ui/ui.go b/internal/ui/ui.go index 83abf4d..8216c83 100644 --- a/internal/ui/ui.go +++ b/internal/ui/ui.go @@ -4,14 +4,14 @@ // // Message classes (see Writer methods): // -// - Begin — operation start, left-aligned, marker "》" (white) -// - Complete— operation completion, left-aligned, marker "》" (green) -// - Info — left-aligned neutral status, marker "》" (white) -// - Notice — left-aligned important note, marker "》" (cyan) -// - Warning — left-aligned warning, full word "Warning: " (orange/yellow) -// - Error — left-aligned error, full word "ERROR: " (red) -// - Progress— indented heartbeat / per-item update, marker " 》" (white) -// - Banner — application banner line, left-aligned, no marker +// - Beginf — operation start, left-aligned, marker "》" (white) +// - Completef— operation completion, left-aligned, marker "》" (green) +// - Infof — left-aligned neutral status, marker "》" (white) +// - Noticef — left-aligned important note, marker "》" (cyan) +// - Warningf — left-aligned warning, full word "Warning: " (orange/yellow) +// - Errorf — left-aligned error, full word "ERROR: " (red) +// - Progressf— indented heartbeat / per-item update, marker " 》" (white) +// - Bannerf — application banner line, left-aligned, no marker // // Value formatters (Hex, Size, Duration, Time, Path, Snapshot, Speed, // Count, Percent) return ANSI-colored strings the caller composes into @@ -45,6 +45,19 @@ const ( // Marker is the chevron prefix used for all non-error/warning lines. const Marker = "》" +const ( + // hexAbbrevLen is the number of hash characters Hex keeps before "...". + hexAbbrevLen = 12 + + // bitsPerByte converts bytes/sec into bits/sec for Speed. + bitsPerByte = 8 + + // SI thresholds for Speed's unit selection, in bits/sec. + gigabit = 1e9 + megabit = 1e6 + kilobit = 1e3 +) + // Writer formats and emits user-facing messages with optional ANSI color. // It also counts warnings and errors emitted so the caller can summarize at // the end of an operation ("Finished successfully." vs "Finished with @@ -103,15 +116,6 @@ func shouldColor(w io.Writer) bool { return term.IsTerminal(int(f.Fd())) } -// paint wraps s in the given ANSI color when color is enabled. -func (w *Writer) paint(color, s string) string { - if !w.color { - return s - } - - return color + s + ansiReset -} - // ───────────────────────── message methods ───────────────────────── // Beginf prints an operation-start line, left-aligned with a white marker. @@ -208,17 +212,6 @@ func (w *Writer) Bannerf(format string, args ...any) { _, _ = fmt.Fprintln(w.out, body) } -// emit writes " \n" with the prefix painted in prefixColor -// and the body optionally painted in bodyColor (empty = no body color). -func (w *Writer) emit(prefixColor, prefix, bodyColor, format string, args []any) { - body := fmt.Sprintf(format, args...) - if bodyColor != "" { - body = w.paint(bodyColor, body) - } - - _, _ = fmt.Fprintln(w.out, w.paint(prefixColor, prefix)+" "+body) -} - // ───────────────────────── value formatters ───────────────────────── // // These return ANSI-colored strings the caller composes into a message @@ -228,8 +221,8 @@ func (w *Writer) emit(prefixColor, prefix, bodyColor, format string, args []any) // Long hashes are abbreviated to first 12 chars with "...". func (w *Writer) Hex(s string) string { short := s - if len(s) > 12 { - short = s[:12] + "..." + if len(s) > hexAbbrevLen { + short = s[:hexAbbrevLen] + "..." } return w.paint(ansiCyan, short) @@ -247,7 +240,7 @@ func (w *Writer) Path(p string) string { // Size colorizes a byte count using humanize.Bytes. func (w *Writer) Size(bytes int64) string { - return w.paint(ansiMagenta, humanize.Bytes(uint64(bytes))) + return w.paint(ansiMagenta, humanize.Bytes(uint64(bytes))) //nolint:gosec // G115: >=0 } // Speed colorizes a network transfer rate. Input is bytes/sec; output is @@ -258,17 +251,17 @@ func (w *Writer) Speed(bytesPerSec float64) string { return w.paint(ansiMagenta, "N/A") } - bitsPerSec := bytesPerSec * 8 + bitsPerSec := bytesPerSec * bitsPerByte var s string switch { - case bitsPerSec >= 1e9: - s = fmt.Sprintf("%.1f Gbit/sec", bitsPerSec/1e9) - case bitsPerSec >= 1e6: - s = fmt.Sprintf("%.0f Mbit/sec", bitsPerSec/1e6) - case bitsPerSec >= 1e3: - s = fmt.Sprintf("%.0f Kbit/sec", bitsPerSec/1e3) + case bitsPerSec >= gigabit: + s = fmt.Sprintf("%.1f Gbit/sec", bitsPerSec/gigabit) + case bitsPerSec >= megabit: + s = fmt.Sprintf("%.0f Mbit/sec", bitsPerSec/megabit) + case bitsPerSec >= kilobit: + s = fmt.Sprintf("%.0f Kbit/sec", bitsPerSec/kilobit) default: s = fmt.Sprintf("%.0f bit/sec", bitsPerSec) } @@ -286,7 +279,7 @@ func (w *Writer) Duration(d time.Duration) string { // "YYYY-MM-DD HH:MM:SS". No timezone is included — values are // displayed in the process's local zone. func (w *Writer) Time(t time.Time) string { - t = t.Local() + t = t.Local() //nolint:gosmopolitan // local-time display is intentional now := time.Now() if t.Year() == now.Year() && t.YearDay() == now.YearDay() { @@ -305,3 +298,25 @@ func (w *Writer) Count(n int) string { func (w *Writer) Percent(p float64) string { return w.paint(ansiMagenta, fmt.Sprintf("%.1f%%", p)) } + +// ───────────────────────── internal helpers ───────────────────────── + +// paint wraps s in the given ANSI color when color is enabled. +func (w *Writer) paint(color, s string) string { + if !w.color { + return s + } + + return color + s + ansiReset +} + +// emit writes " \n" with the prefix painted in prefixColor +// and the body optionally painted in bodyColor (empty = no body color). +func (w *Writer) emit(prefixColor, prefix, bodyColor, format string, args []any) { + body := fmt.Sprintf(format, args...) + if bodyColor != "" { + body = w.paint(bodyColor, body) + } + + _, _ = fmt.Fprintln(w.out, w.paint(prefixColor, prefix)+" "+body) +} diff --git a/internal/ui/ui_test.go b/internal/ui/ui_test.go index 0f06b86..7da92c3 100644 --- a/internal/ui/ui_test.go +++ b/internal/ui/ui_test.go @@ -1,37 +1,45 @@ -package ui +package ui_test import ( "bytes" "strings" "testing" "time" + + "sneak.berlin/go/vaultik/internal/ui" ) -func newTestWriter(color bool) (*Writer, *bytes.Buffer) { +func newTestWriter(color bool) (*ui.Writer, *bytes.Buffer) { buf := &bytes.Buffer{} - return NewWithColor(buf, color), buf + return ui.NewWithColor(buf, color), buf } func TestMessageMethodsPlain(t *testing.T) { + t.Parallel() + tests := []struct { method string - fn func(*Writer) + fn func(*ui.Writer) want string }{ - {"Begin", func(w *Writer) { w.Beginf("starting %s", "thing") }, "》 starting thing\n"}, - {"Complete", func(w *Writer) { w.Completef("done %s", "thing") }, "》 done thing\n"}, - {"Info", func(w *Writer) { w.Infof("status") }, "》 status\n"}, - {"Notice", func(w *Writer) { w.Noticef("note") }, "》 note\n"}, - {"Warning", func(w *Writer) { w.Warningf("oops") }, "⚠️ Warning: oops\n"}, - {"Error", func(w *Writer) { w.Errorf("boom") }, "🛑 ERROR: boom\n"}, - {"Progress", func(w *Writer) { w.Progressf("p") }, " 》 p\n"}, - {"Detail", func(w *Writer) { w.Detailf("d") }, " 》 d\n"}, - {"Banner", func(w *Writer) { w.Bannerf("hello") }, "hello\n"}, // plain mode, no bold + {"Begin", func(w *ui.Writer) { w.Beginf("starting %s", "thing") }, + "》 starting thing\n"}, + {"Complete", func(w *ui.Writer) { w.Completef("done %s", "thing") }, + "》 done thing\n"}, + {"Info", func(w *ui.Writer) { w.Infof("status") }, "》 status\n"}, + {"Notice", func(w *ui.Writer) { w.Noticef("note") }, "》 note\n"}, + {"Warning", func(w *ui.Writer) { w.Warningf("oops") }, "⚠️ Warning: oops\n"}, + {"Error", func(w *ui.Writer) { w.Errorf("boom") }, "🛑 ERROR: boom\n"}, + {"Progress", func(w *ui.Writer) { w.Progressf("p") }, " 》 p\n"}, + {"Detail", func(w *ui.Writer) { w.Detailf("d") }, " 》 d\n"}, + {"Banner", func(w *ui.Writer) { w.Bannerf("hello") }, "hello\n"}, // plain, no bold } for _, tt := range tests { t.Run(tt.method, func(t *testing.T) { + t.Parallel() + w, buf := newTestWriter(false) tt.fn(w) @@ -43,6 +51,8 @@ func TestMessageMethodsPlain(t *testing.T) { } func TestWarningErrorCounters(t *testing.T) { + t.Parallel() + w, _ := newTestWriter(false) if w.WarningCount() != 0 || w.ErrorCount() != 0 { t.Fatalf("expected fresh writer to have zero counts") @@ -63,6 +73,8 @@ func TestWarningErrorCounters(t *testing.T) { } func TestColorOutputContainsANSI(t *testing.T) { + t.Parallel() + w, buf := newTestWriter(true) w.Errorf("boom") @@ -77,6 +89,8 @@ func TestColorOutputContainsANSI(t *testing.T) { } func TestBannerBoldWhenColor(t *testing.T) { + t.Parallel() + w, buf := newTestWriter(true) w.Bannerf("hello") @@ -87,6 +101,8 @@ func TestBannerBoldWhenColor(t *testing.T) { } func TestValueFormattersPlain(t *testing.T) { + t.Parallel() + w, _ := newTestWriter(false) if got := w.Hex("0123456789abcdef0123"); got != "0123456789ab..." { @@ -127,18 +143,26 @@ func TestValueFormattersPlain(t *testing.T) { } // Time format: today → HH:MM:SS, other day → YYYY-MM-DD HH:MM:SS. - today := time.Date(time.Now().Year(), time.Now().Month(), time.Now().Day(), 14, 30, 45, 0, time.Local) + // These construct local-zone times on purpose: Writer.Time displays + // in the process's local zone. + now := time.Now() + + today := time.Date(now.Year(), now.Month(), now.Day(), + 14, 30, 45, 0, time.Local) //nolint:gosmopolitan // local display if got := w.Time(today); got != "14:30:45" { t.Errorf("Time today: got %q, want 14:30:45", got) } - other := time.Date(2030, 1, 2, 3, 4, 5, 0, time.Local) + other := time.Date(2030, 1, 2, 3, 4, 5, 0, + time.Local) //nolint:gosmopolitan // local display if got := w.Time(other); got != "2030-01-02 03:04:05" { t.Errorf("Time other day: got %q", got) } } func TestValueFormattersColored(t *testing.T) { + t.Parallel() + w, _ := newTestWriter(true) hex := w.Hex("0123456789abcdef0123") diff --git a/internal/vaultik/blob_fetch.go b/internal/vaultik/blob_fetch.go index 1bd5931..fde9eb2 100644 --- a/internal/vaultik/blob_fetch.go +++ b/internal/vaultik/blob_fetch.go @@ -14,6 +14,10 @@ import ( "sneak.berlin/go/vaultik/internal/log" ) +// errBlobHashMismatch is returned when a fetched blob's content hash does +// not match the expected double-SHA-256 hash. +var errBlobHashMismatch = errors.New("blob hash mismatch") + // hashVerifyReader wraps a blobgen.Reader and verifies the double-SHA-256 hash // of decrypted plaintext when Close is called. It reuses the hash that // blobgen.Reader already computes internally via its TeeReader, avoiding @@ -46,7 +50,8 @@ func (h *hashVerifyReader) Close() error { actualHashHex := hex.EncodeToString(secondHasher.Sum(nil)) if actualHashHex != h.blobHash { - return fmt.Errorf("blob hash mismatch: expected %s, got %s", h.blobHash[:16], actualHashHex[:16]) + return fmt.Errorf("%w: expected %s, got %s", + errBlobHashMismatch, h.blobHash[:16], actualHashHex[:16]) } } @@ -61,7 +66,9 @@ func (h *hashVerifyReader) Close() error { // returns a streaming reader that computes the double-SHA-256 hash on the fly. // The hash is verified when the returned reader is closed (after fully reading). // This avoids buffering the entire blob in memory. -func (v *Vaultik) FetchAndDecryptBlob(ctx context.Context, blobHash string, expectedSize int64, identity age.Identity) (io.ReadCloser, error) { +func (v *Vaultik) FetchAndDecryptBlob( + ctx context.Context, blobHash string, expectedSize int64, identity age.Identity, +) (io.ReadCloser, error) { rc, _, err := v.FetchBlob(ctx, blobHash, expectedSize) if err != nil { return nil, err @@ -85,7 +92,9 @@ func (v *Vaultik) FetchAndDecryptBlob(ctx context.Context, blobHash string, expe // Times the Storage.Get and Storage.Stat round-trips separately at // debug level so we can see whether the size-only Stat (which is an // extra request on every fetch) is hurting throughput. -func (v *Vaultik) FetchBlob(ctx context.Context, blobHash string, expectedSize int64) (io.ReadCloser, int64, error) { +func (v *Vaultik) FetchBlob( + ctx context.Context, blobHash string, expectedSize int64, +) (io.ReadCloser, int64, error) { blobPath := fmt.Sprintf("blobs/%s/%s/%s", blobHash[:2], blobHash[2:4], blobHash) t0 := time.Now() diff --git a/internal/vaultik/blob_fetch_hash_test.go b/internal/vaultik/blob_fetch_hash_test.go index 6dee0b6..8334eea 100644 --- a/internal/vaultik/blob_fetch_hash_test.go +++ b/internal/vaultik/blob_fetch_hash_test.go @@ -14,35 +14,33 @@ import ( "sneak.berlin/go/vaultik/internal/vaultik" ) -// TestFetchAndDecryptBlobVerifiesHash verifies that FetchAndDecryptBlob checks -// the double-SHA-256 hash of the decrypted plaintext against the expected blob hash. -func TestFetchAndDecryptBlobVerifiesHash(t *testing.T) { - identity, err := age.GenerateX25519Identity() - if err != nil { - t.Fatalf("generating identity: %v", err) - } - - // Create test data and encrypt it using blobgen.Writer - plaintext := []byte("hello world test data for blob hash verification") +// buildHashTestBlob encrypts plaintext with blobgen.Writer and returns +// the encrypted bytes plus the expected double-SHA-256 hash. +func buildHashTestBlob( + t *testing.T, identity *age.X25519Identity, plaintext []byte, +) ([]byte, string) { + t.Helper() var encBuf bytes.Buffer - writer, err := blobgen.NewWriter(&encBuf, 1, []string{identity.Recipient().String()}) + writer, err := blobgen.NewWriter(&encBuf, 1, + []string{identity.Recipient().String()}) if err != nil { t.Fatalf("creating blobgen writer: %v", err) } - if _, err := writer.Write(plaintext); err != nil { + _, err = writer.Write(plaintext) + if err != nil { t.Fatalf("writing plaintext: %v", err) } - if err := writer.Close(); err != nil { + err = writer.Close() + if err != nil { t.Fatalf("closing writer: %v", err) } - encryptedData := encBuf.Bytes() - - // Compute correct double-SHA-256 hash of the plaintext (matches blobgen.Writer.Sum256) + // Compute the double-SHA-256 hash of the plaintext (matches + // blobgen.Writer.Sum256). firstHash := sha256.Sum256(plaintext) secondHash := sha256.Sum256(firstHash[:]) correctHash := hex.EncodeToString(secondHash[:]) @@ -50,12 +48,30 @@ func TestFetchAndDecryptBlobVerifiesHash(t *testing.T) { // Verify our hash matches what blobgen.Writer produces writerHash := hex.EncodeToString(writer.Sum256()) if correctHash != writerHash { - t.Fatalf("hash computation mismatch: manual=%s, writer=%s", correctHash, writerHash) + t.Fatalf("hash computation mismatch: manual=%s, writer=%s", + correctHash, writerHash) } + return encBuf.Bytes(), correctHash +} + +// TestFetchAndDecryptBlobVerifiesHash verifies that FetchAndDecryptBlob checks +// the double-SHA-256 hash of the decrypted plaintext against the expected blob hash. +func TestFetchAndDecryptBlobVerifiesHash(t *testing.T) { + t.Parallel() + + identity, err := age.GenerateX25519Identity() + if err != nil { + t.Fatalf("generating identity: %v", err) + } + + plaintext := []byte("hello world test data for blob hash verification") + encryptedData, correctHash := buildHashTestBlob(t, identity, plaintext) + // Set up mock storage with the blob at the correct path mockStorage := NewMockStorer() - blobPath := "blobs/" + correctHash[:2] + "/" + correctHash[2:4] + "/" + correctHash + blobPath := "blobs/" + correctHash[:2] + "/" + + correctHash[2:4] + "/" + correctHash mockStorage.mu.Lock() mockStorage.data[blobPath] = encryptedData @@ -65,7 +81,10 @@ func TestFetchAndDecryptBlobVerifiesHash(t *testing.T) { ctx := context.Background() t.Run("correct hash succeeds", func(t *testing.T) { - rc, err := tv.FetchAndDecryptBlob(ctx, correctHash, int64(len(encryptedData)), identity) + t.Parallel() + + rc, err := tv.FetchAndDecryptBlob( + ctx, correctHash, int64(len(encryptedData)), identity) if err != nil { t.Fatalf("expected success, got error: %v", err) } @@ -75,7 +94,8 @@ func TestFetchAndDecryptBlobVerifiesHash(t *testing.T) { t.Fatalf("reading stream: %v", err) } - if err := rc.Close(); err != nil { + err = rc.Close() + if err != nil { t.Fatalf("close (hash verification) failed: %v", err) } @@ -85,6 +105,8 @@ func TestFetchAndDecryptBlobVerifiesHash(t *testing.T) { }) t.Run("wrong hash fails", func(t *testing.T) { + t.Parallel() + // Use a fake hash that doesn't match the actual plaintext fakeHash := strings.Repeat("ab", 32) // 64 hex chars fakePath := "blobs/" + fakeHash[:2] + "/" + fakeHash[2:4] + "/" + fakeHash @@ -93,7 +115,8 @@ func TestFetchAndDecryptBlobVerifiesHash(t *testing.T) { mockStorage.data[fakePath] = encryptedData mockStorage.mu.Unlock() - rc, err := tv.FetchAndDecryptBlob(ctx, fakeHash, int64(len(encryptedData)), identity) + rc, err := tv.FetchAndDecryptBlob( + ctx, fakeHash, int64(len(encryptedData)), identity) if err != nil { t.Fatalf("unexpected error opening stream: %v", err) } diff --git a/internal/vaultik/blobcache.go b/internal/vaultik/blobcache.go index 2a28f18..f6a3bd6 100644 --- a/internal/vaultik/blobcache.go +++ b/internal/vaultik/blobcache.go @@ -1,6 +1,7 @@ package vaultik import ( + "errors" "fmt" "io" "os" @@ -8,6 +9,15 @@ import ( "sync" ) +// Sentinel errors for blob cache lookups. +var ( + errCacheKeyMissing = errors.New("key not in cache") + errCacheReadBeyondBlob = errors.New("read beyond blob size") +) + +// blobCacheFileMode is the permission mode for cached blob files. +const blobCacheFileMode = 0o600 + // blobDiskCacheEntry tracks a cached blob on disk. type blobDiskCacheEntry struct { key string @@ -61,54 +71,8 @@ func newBlobDiskCache(maxBytes int64) (*blobDiskCache, error) { }, nil } -func (c *blobDiskCache) path(key string) string { - return filepath.Join(c.dir, key) -} - -func (c *blobDiskCache) unlink(e *blobDiskCacheEntry) { - if e.prev != nil { - e.prev.next = e.next - } else { - c.head = e.next - } - - if e.next != nil { - e.next.prev = e.prev - } else { - c.tail = e.prev - } - - e.prev = nil - e.next = nil -} - -func (c *blobDiskCache) pushFront(e *blobDiskCacheEntry) { - e.prev = nil - - e.next = c.head - if c.head != nil { - c.head.prev = e - } - - c.head = e - if c.tail == nil { - c.tail = e - } -} - -func (c *blobDiskCache) evictLRU() { - if c.tail == nil { - return - } - - victim := c.tail - c.unlink(victim) - delete(c.items, victim.key) - c.curBytes -= victim.size - _ = os.Remove(c.path(victim.key)) -} - -// Put writes blob data to disk cache. Entries larger than maxBytes are silently skipped. +// Put writes blob data to disk cache. Entries larger than maxBytes are +// silently skipped. func (c *blobDiskCache) Put(key string, data []byte) error { entrySize := int64(len(data)) @@ -127,7 +91,7 @@ func (c *blobDiskCache) Put(key string, data []byte) error { delete(c.items, key) } - err := os.WriteFile(c.path(key), data, 0600) + err := os.WriteFile(c.path(key), data, blobCacheFileMode) if err != nil { return fmt.Errorf("writing blob to cache: %w", err) } @@ -166,7 +130,8 @@ func (c *blobDiskCache) PutFromReader(key string, r io.Reader) (int64, error) { } c.mu.Unlock() - f, err := os.OpenFile(c.path(key), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600) + f, err := os.OpenFile( + c.path(key), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, blobCacheFileMode) if err != nil { return 0, fmt.Errorf("creating cache file: %w", err) } @@ -255,13 +220,14 @@ func (c *blobDiskCache) ReadAt(key string, offset, length int64) ([]byte, error) if !ok { c.mu.Unlock() - return nil, fmt.Errorf("key %q not in cache", key) + return nil, fmt.Errorf("%w: %q", errCacheKeyMissing, key) } if offset+length > e.size { c.mu.Unlock() - return nil, fmt.Errorf("read beyond blob size: offset=%d length=%d size=%d", offset, length, e.size) + return nil, fmt.Errorf("%w: offset=%d length=%d size=%d", + errCacheReadBeyondBlob, offset, length, e.size) } c.unlink(e) @@ -275,7 +241,9 @@ func (c *blobDiskCache) ReadAt(key string, offset, length int64) ([]byte, error) defer func() { _ = f.Close() }() buf := make([]byte, length) - if _, err := f.ReadAt(buf, offset); err != nil { + + _, err = f.ReadAt(buf, offset) + if err != nil { return nil, err } @@ -377,3 +345,50 @@ func (c *blobDiskCache) Close() error { return os.RemoveAll(c.dir) } + +func (c *blobDiskCache) path(key string) string { + return filepath.Join(c.dir, key) +} + +func (c *blobDiskCache) unlink(e *blobDiskCacheEntry) { + if e.prev != nil { + e.prev.next = e.next + } else { + c.head = e.next + } + + if e.next != nil { + e.next.prev = e.prev + } else { + c.tail = e.prev + } + + e.prev = nil + e.next = nil +} + +func (c *blobDiskCache) pushFront(e *blobDiskCacheEntry) { + e.prev = nil + + e.next = c.head + if c.head != nil { + c.head.prev = e + } + + c.head = e + if c.tail == nil { + c.tail = e + } +} + +func (c *blobDiskCache) evictLRU() { + if c.tail == nil { + return + } + + victim := c.tail + c.unlink(victim) + delete(c.items, victim.key) + c.curBytes -= victim.size + _ = os.Remove(c.path(victim.key)) +} diff --git a/internal/vaultik/blobcache_test.go b/internal/vaultik/blobcache_test.go index 7467eff..5d61872 100644 --- a/internal/vaultik/blobcache_test.go +++ b/internal/vaultik/blobcache_test.go @@ -1,4 +1,4 @@ -package vaultik +package vaultik //nolint:testpackage // exercises unexported blobDiskCache import ( "bytes" @@ -8,6 +8,8 @@ import ( ) func TestBlobDiskCache_BasicGetPut(t *testing.T) { + t.Parallel() + cache, err := newBlobDiskCache(1 << 20) if err != nil { t.Fatal(err) @@ -15,7 +17,9 @@ func TestBlobDiskCache_BasicGetPut(t *testing.T) { defer func() { _ = cache.Close() }() data := []byte("hello world") - if err := cache.Put("key1", data); err != nil { + + err = cache.Put("key1", data) + if err != nil { t.Fatal(err) } @@ -35,6 +39,8 @@ func TestBlobDiskCache_BasicGetPut(t *testing.T) { } func TestBlobDiskCache_EvictionUnderPressure(t *testing.T) { + t.Parallel() + maxBytes := int64(1000) cache, err := newBlobDiskCache(maxBytes) @@ -47,7 +53,7 @@ func TestBlobDiskCache_EvictionUnderPressure(t *testing.T) { for i := range 5 { data := make([]byte, 300) - err := cache.Put(fmt.Sprintf("key%d", i), data) + err = cache.Put(fmt.Sprintf("key%d", i), data) if err != nil { t.Fatal(err) } @@ -67,6 +73,8 @@ func TestBlobDiskCache_EvictionUnderPressure(t *testing.T) { } func TestBlobDiskCache_OversizedEntryRejected(t *testing.T) { + t.Parallel() + cache, err := newBlobDiskCache(100) if err != nil { t.Fatal(err) @@ -74,7 +82,9 @@ func TestBlobDiskCache_OversizedEntryRejected(t *testing.T) { defer func() { _ = cache.Close() }() data := make([]byte, 200) - if err := cache.Put("big", data); err != nil { + + err = cache.Put("big", data) + if err != nil { t.Fatal(err) } @@ -84,17 +94,21 @@ func TestBlobDiskCache_OversizedEntryRejected(t *testing.T) { } func TestBlobDiskCache_UpdateInPlace(t *testing.T) { + t.Parallel() + cache, err := newBlobDiskCache(1 << 20) if err != nil { t.Fatal(err) } defer func() { _ = cache.Close() }() - if err := cache.Put("key1", []byte("v1")); err != nil { + err = cache.Put("key1", []byte("v1")) + if err != nil { t.Fatal(err) } - if err := cache.Put("key1", []byte("version2")); err != nil { + err = cache.Put("key1", []byte("version2")) + if err != nil { t.Fatal(err) } @@ -117,6 +131,8 @@ func TestBlobDiskCache_UpdateInPlace(t *testing.T) { } func TestBlobDiskCache_ReadAt(t *testing.T) { + t.Parallel() + cache, err := newBlobDiskCache(1 << 20) if err != nil { t.Fatal(err) @@ -124,11 +140,14 @@ func TestBlobDiskCache_ReadAt(t *testing.T) { defer func() { _ = cache.Close() }() data := make([]byte, 1024) - if _, err := rand.Read(data); err != nil { + + _, err = rand.Read(data) + if err != nil { t.Fatal(err) } - if err := cache.Put("blob1", data); err != nil { + err = cache.Put("blob1", data) + if err != nil { t.Fatal(err) } @@ -153,21 +172,27 @@ func TestBlobDiskCache_ReadAt(t *testing.T) { } func TestBlobDiskCache_Close(t *testing.T) { + t.Parallel() + cache, err := newBlobDiskCache(1 << 20) if err != nil { t.Fatal(err) } - if err := cache.Put("key1", []byte("data")); err != nil { + err = cache.Put("key1", []byte("data")) + if err != nil { t.Fatal(err) } - if err := cache.Close(); err != nil { + err = cache.Close() + if err != nil { t.Fatal(err) } } func TestBlobDiskCache_LRUOrder(t *testing.T) { + t.Parallel() + cache, err := newBlobDiskCache(200) if err != nil { t.Fatal(err) @@ -175,11 +200,14 @@ func TestBlobDiskCache_LRUOrder(t *testing.T) { defer func() { _ = cache.Close() }() d := make([]byte, 100) - if err := cache.Put("a", d); err != nil { + + err = cache.Put("a", d) + if err != nil { t.Fatal(err) } - if err := cache.Put("b", d); err != nil { + err = cache.Put("b", d) + if err != nil { t.Fatal(err) } @@ -187,7 +215,8 @@ func TestBlobDiskCache_LRUOrder(t *testing.T) { cache.Get("a") // Adding "c" should evict "b" (LRU), not "a" - if err := cache.Put("c", d); err != nil { + err = cache.Put("c", d) + if err != nil { t.Fatal(err) } diff --git a/internal/vaultik/helpers.go b/internal/vaultik/helpers.go index b10e161..cc57832 100644 --- a/internal/vaultik/helpers.go +++ b/internal/vaultik/helpers.go @@ -8,13 +8,56 @@ import ( "strings" "time" + "github.com/dustin/go-humanize" "sneak.berlin/go/vaultik/internal/types" ) +// percentScale converts a 0..1 ratio into a percentage. +const percentScale = 100 + +// progressLogEvery is how many processed items pass between progress +// log lines in long-running loops. +const progressLogEvery = 100 + +// ubytes renders a byte count with humanize.Bytes, clamping negative +// values to zero so the int64→uint64 conversion cannot overflow. +func ubytes(n int64) string { + if n < 0 { + n = 0 + } + + return humanize.Bytes(uint64(n)) +} + +// Sentinel errors for snapshot ID and duration parsing. +var ( + errMalformedSnapshotID = errors.New( + "invalid snapshot ID format: expected hostname_snapshotname_timestamp") + errInvalidDuration = errors.New("invalid duration") + errUnknownTimeUnit = errors.New("unknown time unit") +) + +// Time-unit lengths used by parseDuration. +const ( + day = 24 * time.Hour + week = 7 * day + month = 30 * day + year = 365 * day +) + +// Snapshot IDs split on "_" into hostname, optional name parts, and a +// trailing timestamp. +const ( + minSnapshotIDParts = 2 + minSnapshotIDNameParts = 3 +) + // SnapshotInfo contains information about a snapshot. // UncompressedSize and NewChunkSize are populated only when the snapshot // is present in the local database; LocallyTracked indicates whether // those values are meaningful. +// +//nolint:tagliatelle // snake_case is the established output format type SnapshotInfo struct { ID types.SnapshotID `json:"id"` Timestamp time.Time `json:"timestamp"` @@ -44,8 +87,8 @@ func formatBytes(bytes int64) string { // Format: hostname_snapshotname_2026-01-12T14:41:15Z func parseSnapshotTimestamp(snapshotID string) (time.Time, error) { parts := strings.Split(snapshotID, "_") - if len(parts) < 2 { - return time.Time{}, errors.New("invalid snapshot ID format: expected hostname_snapshotname_timestamp") + if len(parts) < minSnapshotIDParts { + return time.Time{}, errMalformedSnapshotID } // Last part is the RFC3339 timestamp @@ -65,7 +108,7 @@ func parseSnapshotTimestamp(snapshotID string) (time.Time, error) { // Returns the snapshot name, or empty string if the ID is malformed. func parseSnapshotName(snapshotID string) string { parts := strings.Split(snapshotID, "_") - if len(parts) < 3 { + if len(parts) < minSnapshotIDNameParts { // Format: hostname_timestamp — no snapshot name return "" } @@ -79,7 +122,8 @@ func parseSnapshotName(snapshotID string) string { // d/day/days, w/week/weeks, mo/month/months, y/year/years, plus standard Go // duration units (h, m, s). func parseDuration(s string) (time.Duration, error) { - if d, err := time.ParseDuration(s); err == nil { + d, err := time.ParseDuration(s) + if err == nil { return d, nil } @@ -87,7 +131,7 @@ func parseDuration(s string) (time.Duration, error) { matches := re.FindAllStringSubmatch(s, -1) if len(matches) == 0 { - return 0, fmt.Errorf("invalid duration: %q", s) + return 0, fmt.Errorf("%w: %q", errInvalidDuration, s) } var total time.Duration @@ -101,15 +145,15 @@ func parseDuration(s string) (time.Duration, error) { unit := strings.ToLower(match[2]) switch unit { case "d", "day", "days": - total += time.Duration(n) * 24 * time.Hour + total += time.Duration(n) * day case "w", "week", "weeks": - total += time.Duration(n) * 7 * 24 * time.Hour + total += time.Duration(n) * week case "mo", "month", "months": - total += time.Duration(n) * 30 * 24 * time.Hour + total += time.Duration(n) * month case "y", "year", "years": - total += time.Duration(n) * 365 * 24 * time.Hour + total += time.Duration(n) * year default: - return 0, fmt.Errorf("unknown time unit %q", unit) + return 0, fmt.Errorf("%w %q", errUnknownTimeUnit, unit) } } diff --git a/internal/vaultik/helpers_test.go b/internal/vaultik/helpers_test.go index e648bec..53e6746 100644 --- a/internal/vaultik/helpers_test.go +++ b/internal/vaultik/helpers_test.go @@ -1,4 +1,4 @@ -package vaultik +package vaultik //nolint:testpackage // exercises unexported parse helpers import ( "testing" @@ -6,6 +6,8 @@ import ( ) func TestParseSnapshotName(t *testing.T) { + t.Parallel() + tests := []struct { name string snapshotID string @@ -30,15 +32,20 @@ func TestParseSnapshotName(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + t.Parallel() + got := parseSnapshotName(tt.snapshotID) if got != tt.want { - t.Errorf("parseSnapshotName(%q) = %q, want %q", tt.snapshotID, got, tt.want) + t.Errorf("parseSnapshotName(%q) = %q, want %q", + tt.snapshotID, got, tt.want) } }) } } func TestParseDuration(t *testing.T) { + t.Parallel() + tests := []struct { input string want time.Duration @@ -56,6 +63,8 @@ func TestParseDuration(t *testing.T) { for _, tt := range tests { t.Run(tt.input, func(t *testing.T) { + t.Parallel() + got, err := parseDuration(tt.input) if tt.err { if err == nil { @@ -77,6 +86,8 @@ func TestParseDuration(t *testing.T) { } func TestParseSnapshotTimestamp(t *testing.T) { + t.Parallel() + tests := []struct { name string snapshotID string @@ -106,9 +117,12 @@ func TestParseSnapshotTimestamp(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + t.Parallel() + _, err := parseSnapshotTimestamp(tt.snapshotID) if (err != nil) != tt.wantErr { - t.Errorf("parseSnapshotTimestamp(%q) error = %v, wantErr %v", tt.snapshotID, err, tt.wantErr) + t.Errorf("parseSnapshotTimestamp(%q) error = %v, wantErr %v", + tt.snapshotID, err, tt.wantErr) } }) } diff --git a/internal/vaultik/info.go b/internal/vaultik/info.go index 2046bf2..59d41fc 100644 --- a/internal/vaultik/info.go +++ b/internal/vaultik/info.go @@ -22,10 +22,28 @@ func (v *Vaultik) ShowInfo() error { v.stdoutf("Go Version: %s\n", runtime.Version()) v.printlnStdout() - // Storage Configuration. The backend is selected by storage_url - // (s3://, file://, rclone://); the legacy s3.* fields are only - // printed when they're actually populated, since the URL scheme - // is the primary configuration. + v.showStorageConfig() + v.showBackupSettings() + + // Encryption Configuration + v.stdoutf("=== Encryption Configuration ===\n") + v.stdoutf("Recipients:\n") + + for _, recipient := range v.Config.AgeRecipients { + v.stdoutf(" - %s\n", recipient) + } + + v.printlnStdout() + v.showLocalDatabase() + + return nil +} + +// showStorageConfig prints the storage configuration section. The +// backend is selected by storage_url (s3://, file://, rclone://); the +// legacy s3.* fields are only printed when they're actually populated, +// since the URL scheme is the primary configuration. +func (v *Vaultik) showStorageConfig() { v.stdoutf("=== Storage Configuration ===\n") storageInfo := v.Storage.Info() v.stdoutf("Type: %s\n", storageInfo.Type) @@ -52,8 +70,11 @@ func (v *Vaultik) ShowInfo() error { } v.printlnStdout() +} - // Backup Settings +// showBackupSettings prints the configured snapshots, exclude patterns, +// and chunking/compression settings. +func (v *Vaultik) showBackupSettings() { v.stdoutf("=== Backup Settings ===\n") // Show configured snapshots @@ -78,65 +99,76 @@ func (v *Vaultik) ShowInfo() error { } v.stdoutf("Compression: zstd level %d\n", v.Config.CompressionLevel) - v.stdoutf("Chunk Size: %s\n", humanize.Bytes(uint64(v.Config.ChunkSize))) - v.stdoutf("Blob Size Limit: %s\n", humanize.Bytes(uint64(v.Config.BlobSizeLimit))) + v.stdoutf("Chunk Size: %s\n", ubytes(int64(v.Config.ChunkSize))) + v.stdoutf("Blob Size Limit: %s\n", ubytes(int64(v.Config.BlobSizeLimit))) v.printlnStdout() +} - // Encryption Configuration - v.stdoutf("=== Encryption Configuration ===\n") - v.stdoutf("Recipients:\n") - - for _, recipient := range v.Config.AgeRecipients { - v.stdoutf(" - %s\n", recipient) - } - - v.printlnStdout() - - // Local Database +// showLocalDatabase prints the local index database section, including +// record counts when the index exists. +func (v *Vaultik) showLocalDatabase() { v.stdoutf("=== Local Database ===\n") v.stdoutf("Index Path: %s\n", v.Config.IndexPath) // Check if index file exists and get its size - if info, err := v.Fs.Stat(v.Config.IndexPath); err == nil { - v.stdoutf("Index Size: %s\n", humanize.Bytes(uint64(info.Size()))) - - // Get snapshot count from database - query := `SELECT COUNT(*) FROM snapshots WHERE completed_at IS NOT NULL` - - var snapshotCount int - - err := v.DB.Conn().QueryRowContext(v.ctx, query).Scan(&snapshotCount) - if err == nil { - v.stdoutf("Snapshots: %d\n", snapshotCount) - } - - // Get blob count from database - query = `SELECT COUNT(*) FROM blobs` - - var blobCount int - - err = v.DB.Conn().QueryRowContext(v.ctx, query).Scan(&blobCount) - if err == nil { - v.stdoutf("Blobs: %d\n", blobCount) - } - - // Get file count from database - query = `SELECT COUNT(*) FROM files` - - var fileCount int - - err = v.DB.Conn().QueryRowContext(v.ctx, query).Scan(&fileCount) - if err == nil { - v.stdoutf("Files: %d\n", fileCount) - } - } else { + info, err := v.Fs.Stat(v.Config.IndexPath) + if err != nil { v.stdoutf("Index Size: (not created)\n") + + return } - return nil + v.stdoutf("Index Size: %s\n", ubytes(info.Size())) + + // Get snapshot count from database + query := `SELECT COUNT(*) FROM snapshots WHERE completed_at IS NOT NULL` + + var snapshotCount int + + err = v.DB.Conn().QueryRowContext(v.ctx, query).Scan(&snapshotCount) + if err == nil { + v.stdoutf("Snapshots: %d\n", snapshotCount) + } + + // Get blob count from database + query = `SELECT COUNT(*) FROM blobs` + + var blobCount int + + err = v.DB.Conn().QueryRowContext(v.ctx, query).Scan(&blobCount) + if err == nil { + v.stdoutf("Blobs: %d\n", blobCount) + } + + // Get file count from database + query = `SELECT COUNT(*) FROM files` + + var fileCount int + + err = v.DB.Conn().QueryRowContext(v.ctx, query).Scan(&fileCount) + if err == nil { + v.stdoutf("Files: %d\n", fileCount) + } } +// Table layout constants for the human-readable remote info output. +const ( + // snapshotIDColWidth is the SNAPSHOT column width in the remote + // info table. + snapshotIDColWidth = 45 + + // metadataKeyParts is the minimum "/"-separated segment count of a + // metadata object key (metadata//). + metadataKeyParts = 3 + + // blobKeyParts is the minimum "/"-separated segment count of a blob + // object key (blobs///). + blobKeyParts = 4 +) + // SnapshotMetadataInfo contains information about a single snapshot's metadata +// +//nolint:tagliatelle // snake_case is the established JSON output format type SnapshotMetadataInfo struct { SnapshotID string `json:"snapshot_id"` ManifestSize int64 `json:"manifest_size"` @@ -147,6 +179,8 @@ type SnapshotMetadataInfo struct { } // RemoteInfoResult contains all remote storage information +// +//nolint:tagliatelle // snake_case is the established JSON output format type RemoteInfoResult struct { // Storage info StorageType string `json:"storage_type"` @@ -201,7 +235,8 @@ func (v *Vaultik) RemoteInfo(jsonOutput bool) error { v.populateRemoteInfoResult(result, snapshotMetadata, snapshotIDs, referencedBlobs) - if err := v.scanRemoteBlobStorage(result, referencedBlobs, jsonOutput); err != nil { + err = v.scanRemoteBlobStorage(result, referencedBlobs, jsonOutput) + if err != nil { return err } @@ -223,8 +258,11 @@ func (v *Vaultik) RemoteInfo(jsonOutput bool) error { return nil } -// collectSnapshotMetadata scans remote metadata and returns per-snapshot info and sorted IDs -func (v *Vaultik) collectSnapshotMetadata() (map[string]*SnapshotMetadataInfo, []string, error) { +// collectSnapshotMetadata scans remote metadata and returns +// per-snapshot info and sorted IDs. +func (v *Vaultik) collectSnapshotMetadata() ( + map[string]*SnapshotMetadataInfo, []string, error, +) { snapshotMetadata := make(map[string]*SnapshotMetadataInfo) metadataCh := v.Storage.ListStream(v.ctx, "metadata/") @@ -234,7 +272,7 @@ func (v *Vaultik) collectSnapshotMetadata() (map[string]*SnapshotMetadataInfo, [ } parts := strings.Split(obj.Key, "/") - if len(parts) < 3 { + if len(parts) < metadataKeyParts { continue } @@ -266,8 +304,11 @@ func (v *Vaultik) collectSnapshotMetadata() (map[string]*SnapshotMetadataInfo, [ return snapshotMetadata, snapshotIDs, nil } -// collectReferencedBlobsFromManifests downloads manifests and returns referenced blob hashes with sizes -func (v *Vaultik) collectReferencedBlobsFromManifests(snapshotIDs []string, snapshotMetadata map[string]*SnapshotMetadataInfo) map[string]int64 { +// collectReferencedBlobsFromManifests downloads manifests and returns +// referenced blob hashes with sizes. +func (v *Vaultik) collectReferencedBlobsFromManifests( + snapshotIDs []string, snapshotMetadata map[string]*SnapshotMetadataInfo, +) map[string]int64 { referencedBlobs := make(map[string]int64) for _, snapshotID := range snapshotIDs { @@ -305,8 +346,14 @@ func (v *Vaultik) collectReferencedBlobsFromManifests(snapshotIDs []string, snap return referencedBlobs } -// populateRemoteInfoResult fills in the result's snapshot and referenced blob stats -func (v *Vaultik) populateRemoteInfoResult(result *RemoteInfoResult, snapshotMetadata map[string]*SnapshotMetadataInfo, snapshotIDs []string, referencedBlobs map[string]int64) { +// populateRemoteInfoResult fills in the result's snapshot and +// referenced blob stats. +func (v *Vaultik) populateRemoteInfoResult( + result *RemoteInfoResult, + snapshotMetadata map[string]*SnapshotMetadataInfo, + snapshotIDs []string, + referencedBlobs map[string]int64, +) { var totalMetadataSize int64 for _, id := range snapshotIDs { @@ -325,7 +372,9 @@ func (v *Vaultik) populateRemoteInfoResult(result *RemoteInfoResult, snapshotMet } // scanRemoteBlobStorage lists all blobs on remote and computes orphan stats -func (v *Vaultik) scanRemoteBlobStorage(result *RemoteInfoResult, referencedBlobs map[string]int64, jsonOutput bool) error { +func (v *Vaultik) scanRemoteBlobStorage( + result *RemoteInfoResult, referencedBlobs map[string]int64, jsonOutput bool, +) error { if !jsonOutput { v.stdoutf("Scanning blobs...\n") } @@ -339,7 +388,7 @@ func (v *Vaultik) scanRemoteBlobStorage(result *RemoteInfoResult, referencedBlob } parts := strings.Split(obj.Key, "/") - if len(parts) < 4 { + if len(parts) < blobKeyParts { continue } @@ -361,51 +410,74 @@ func (v *Vaultik) scanRemoteBlobStorage(result *RemoteInfoResult, referencedBlob // printRemoteInfoTable renders the human-readable remote info output func (v *Vaultik) printRemoteInfoTable(result *RemoteInfoResult) { + const ( + rowFormat = "%-45s %12s %12s %12s %10s %12s\n" + sizeColWidth = 12 + countColWidth = 10 + ) + v.stdoutf("\n=== Snapshot Metadata ===\n") if len(result.Snapshots) == 0 { v.stdoutf("No snapshots found\n") } else { - v.stdoutf("%-45s %12s %12s %12s %10s %12s\n", "SNAPSHOT", "MANIFEST", "DATABASE", "TOTAL", "BLOBS", "BLOB SIZE") - v.stdoutf("%-45s %12s %12s %12s %10s %12s\n", strings.Repeat("-", 45), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 10), strings.Repeat("-", 12)) + separator := fmt.Sprintf(rowFormat, + strings.Repeat("-", snapshotIDColWidth), + strings.Repeat("-", sizeColWidth), + strings.Repeat("-", sizeColWidth), + strings.Repeat("-", sizeColWidth), + strings.Repeat("-", countColWidth), + strings.Repeat("-", sizeColWidth)) + + v.stdoutf(rowFormat, + "SNAPSHOT", "MANIFEST", "DATABASE", "TOTAL", "BLOBS", "BLOB SIZE") + v.stdoutf("%s", separator) for _, info := range result.Snapshots { - v.stdoutf("%-45s %12s %12s %12s %10s %12s\n", - truncateString(info.SnapshotID, 45), - humanize.Bytes(uint64(info.ManifestSize)), - humanize.Bytes(uint64(info.DatabaseSize)), - humanize.Bytes(uint64(info.TotalSize)), + v.stdoutf(rowFormat, + truncateString(info.SnapshotID, snapshotIDColWidth), + ubytes(info.ManifestSize), + ubytes(info.DatabaseSize), + ubytes(info.TotalSize), humanize.Comma(int64(info.BlobCount)), - humanize.Bytes(uint64(info.BlobsSize)), + ubytes(info.BlobsSize), ) } - v.stdoutf("%-45s %12s %12s %12s %10s %12s\n", strings.Repeat("-", 45), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 10), strings.Repeat("-", 12)) - v.stdoutf("%-45s %12s %12s %12s\n", fmt.Sprintf("Total (%d snapshots)", result.TotalMetadataCount), "", "", humanize.Bytes(uint64(result.TotalMetadataSize))) + v.stdoutf("%s", separator) + v.stdoutf("%-45s %12s %12s %12s\n", + fmt.Sprintf("Total (%d snapshots)", result.TotalMetadataCount), + "", "", ubytes(result.TotalMetadataSize)) } v.stdoutf("\n=== Blob Storage ===\n") v.stdoutf("Total blobs on remote: %s (%s)\n", - humanize.Comma(int64(result.TotalBlobCount)), humanize.Bytes(uint64(result.TotalBlobSize))) + humanize.Comma(int64(result.TotalBlobCount)), + ubytes(result.TotalBlobSize)) v.stdoutf("Referenced by snapshots: %s (%s)\n", - humanize.Comma(int64(result.ReferencedBlobCount)), humanize.Bytes(uint64(result.ReferencedBlobSize))) + humanize.Comma(int64(result.ReferencedBlobCount)), + ubytes(result.ReferencedBlobSize)) v.stdoutf("Orphaned (unreferenced): %s (%s)\n", - humanize.Comma(int64(result.OrphanedBlobCount)), humanize.Bytes(uint64(result.OrphanedBlobSize))) + humanize.Comma(int64(result.OrphanedBlobCount)), + ubytes(result.OrphanedBlobSize)) if result.OrphanedBlobCount > 0 { v.stdoutf("\nRun 'vaultik prune' to remove orphaned blobs.\n") } } +// ellipsis is appended by truncateString when it shortens a string. +const ellipsis = "..." + // truncateString truncates a string to maxLen, adding "..." if truncated func truncateString(s string, maxLen int) string { if len(s) <= maxLen { return s } - if maxLen <= 3 { + if maxLen <= len(ellipsis) { return s[:maxLen] } - return s[:maxLen-3] + "..." + return s[:maxLen-len(ellipsis)] + ellipsis } diff --git a/internal/vaultik/integration_test.go b/internal/vaultik/integration_test.go index 78a1c18..0e51afd 100644 --- a/internal/vaultik/integration_test.go +++ b/internal/vaultik/integration_test.go @@ -24,6 +24,16 @@ import ( "sneak.berlin/go/vaultik/internal/vaultik" ) +// Shared fixture values used across the vaultik integration tests. +const ( + testLabel = "test" + testHostname = "test-host" + testAgePublicKey = "age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05g" + + "l0sjq9q9wjg" + testAgeSecretKey = "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKU" + + "T68TXSFPK7APHXA2QS2NJA5" +) + // MockStorer implements storage.Storer for testing type MockStorer struct { mu sync.Mutex @@ -38,7 +48,7 @@ func NewMockStorer() *MockStorer { } } -func (m *MockStorer) Put(ctx context.Context, key string, reader io.Reader) error { +func (m *MockStorer) Put(_ context.Context, key string, reader io.Reader) error { m.mu.Lock() defer m.mu.Unlock() @@ -54,11 +64,14 @@ func (m *MockStorer) Put(ctx context.Context, key string, reader io.Reader) erro return nil } -func (m *MockStorer) PutWithProgress(ctx context.Context, key string, reader io.Reader, size int64, progress storage.ProgressCallback) error { +func (m *MockStorer) PutWithProgress( + ctx context.Context, key string, reader io.Reader, + _ int64, _ storage.ProgressCallback, +) error { return m.Put(ctx, key, reader) } -func (m *MockStorer) Get(ctx context.Context, key string) (io.ReadCloser, error) { +func (m *MockStorer) Get(_ context.Context, key string) (io.ReadCloser, error) { m.mu.Lock() defer m.mu.Unlock() @@ -72,7 +85,7 @@ func (m *MockStorer) Get(ctx context.Context, key string) (io.ReadCloser, error) return io.NopCloser(bytes.NewReader(data)), nil } -func (m *MockStorer) Stat(ctx context.Context, key string) (*storage.ObjectInfo, error) { +func (m *MockStorer) Stat(_ context.Context, key string) (*storage.ObjectInfo, error) { m.mu.Lock() defer m.mu.Unlock() @@ -89,7 +102,7 @@ func (m *MockStorer) Stat(ctx context.Context, key string) (*storage.ObjectInfo, }, nil } -func (m *MockStorer) Delete(ctx context.Context, key string) error { +func (m *MockStorer) Delete(_ context.Context, key string) error { m.mu.Lock() defer m.mu.Unlock() @@ -99,7 +112,7 @@ func (m *MockStorer) Delete(ctx context.Context, key string) error { return nil } -func (m *MockStorer) List(ctx context.Context, prefix string) ([]string, error) { +func (m *MockStorer) List(_ context.Context, prefix string) ([]string, error) { m.mu.Lock() defer m.mu.Unlock() @@ -108,7 +121,8 @@ func (m *MockStorer) List(ctx context.Context, prefix string) ([]string, error) var keys []string for key := range m.data { - if len(prefix) == 0 || (len(key) >= len(prefix) && key[:len(prefix)] == prefix) { + if len(prefix) == 0 || + (len(key) >= len(prefix) && key[:len(prefix)] == prefix) { keys = append(keys, key) } } @@ -116,7 +130,9 @@ func (m *MockStorer) List(ctx context.Context, prefix string) ([]string, error) return keys, nil } -func (m *MockStorer) ListStream(ctx context.Context, prefix string) <-chan storage.ObjectInfo { +func (m *MockStorer) ListStream( + _ context.Context, prefix string, +) <-chan storage.ObjectInfo { ch := make(chan storage.ObjectInfo) go func() { defer close(ch) @@ -125,7 +141,8 @@ func (m *MockStorer) ListStream(ctx context.Context, prefix string) <-chan stora defer m.mu.Unlock() for key, data := range m.data { - if len(prefix) == 0 || (len(key) >= len(prefix) && key[:len(prefix)] == prefix) { + if len(prefix) == 0 || + (len(key) >= len(prefix) && key[:len(prefix)] == prefix) { ch <- storage.ObjectInfo{ Key: key, Size: int64(len(data)), @@ -137,8 +154,8 @@ func (m *MockStorer) ListStream(ctx context.Context, prefix string) <-chan stora return ch } -func (m *MockStorer) Info() storage.StorageInfo { - return storage.StorageInfo{ +func (m *MockStorer) Info() storage.Info { + return storage.Info{ Type: "mock", Location: "memory", } @@ -163,123 +180,55 @@ func (m *MockStorer) GetStorageSize() int { return len(m.data) } -// TestEndToEndBackup tests the full backup workflow with mocked dependencies -func TestEndToEndBackup(t *testing.T) { - // Initialize logger - log.Initialize(log.Config{}) +// writeTestFileTree creates each file (and its parent directory) in fs. +func writeTestFileTree(t *testing.T, fs afero.Fs, files map[string]string) { + t.Helper() - // Create in-memory filesystem - fs := afero.NewMemMapFs() + for path, content := range files { + dir := filepath.Dir(path) - // Create test directory structure and files - testFiles := map[string]string{ - "/home/user/documents/file1.txt": "This is file 1 content", - "/home/user/documents/file2.txt": "This is file 2 content with more data", - "/home/user/pictures/photo1.jpg": "Binary photo data here...", - "/home/user/code/main.go": "package main\n\nfunc main() {\n\tprintln(\"Hello, World!\")\n}", - } - - // Create all directories first - dirs := []string{ - "/home/user/documents", - "/home/user/pictures", - "/home/user/code", - } - for _, dir := range dirs { err := fs.MkdirAll(dir, 0755) if err != nil { t.Fatalf("failed to create directory %s: %v", dir, err) } - } - // Create test files - for path, content := range testFiles { - err := afero.WriteFile(fs, path, []byte(content), 0644) + err = afero.WriteFile(fs, path, []byte(content), 0644) if err != nil { t.Fatalf("failed to create test file %s: %v", path, err) } } +} - // Create mock storage - mockStorage := NewMockStorer() +// createTestSnapshotRecord inserts a snapshot row so scans have a +// snapshot to attach to. +func createTestSnapshotRecord( + t *testing.T, repos *database.Repositories, snapshotID string, +) { + t.Helper() - // Create test configuration - cfg := &config.Config{ - Snapshots: map[string]config.SnapshotConfig{ - "test": { - Paths: []string{"/home/user"}, - }, - }, - Exclude: []string{"*.tmp", "*.log"}, - ChunkSize: config.Size(16 * 1024), // 16KB chunks - BlobSizeLimit: config.Size(100 * 1024), // 100KB blobs - CompressionLevel: 3, - AgeRecipients: []string{"age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05gl0sjq9q9wjg"}, // Test public key - AgeSecretKey: "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5", // Test private key - S3: config.S3Config{ - Endpoint: "http://localhost:9000", // MinIO endpoint for testing - Region: "us-east-1", - Bucket: "test-bucket", - AccessKeyID: "test-access", - SecretAccessKey: "test-secret", - }, - IndexPath: ":memory:", // In-memory SQLite database - } - - // For a true end-to-end test, we'll create a simpler test that focuses on - // the core backup logic using the scanner directly with our mock storage ctx := context.Background() - // Create in-memory database - db, err := database.New(ctx, ":memory:") - - require.NoError(t, err) - defer func() { - err := db.Close() - if err != nil { - t.Errorf("failed to close database: %v", err) - } - }() - - repos := database.NewRepositories(db) - - // Create scanner with mock storage - scanner := snapshot.NewScanner(snapshot.ScannerConfig{ - FS: fs, - ChunkSize: cfg.ChunkSize.Int64(), - Repositories: repos, - Storage: mockStorage, - MaxBlobSize: cfg.BlobSizeLimit.Int64(), - CompressionLevel: cfg.CompressionLevel, - AgeRecipients: cfg.AgeRecipients, - EnableProgress: false, - }) - - // Create a snapshot record - snapshotID := "test-snapshot-001" - err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { - snapshot := &database.Snapshot{ + err := repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { + snap := &database.Snapshot{ ID: types.SnapshotID(snapshotID), - Hostname: "test-host", + Hostname: testHostname, VaultikVersion: "test-version", StartedAt: time.Now(), } - return repos.Snapshots.Create(ctx, tx, snapshot) + return repos.Snapshots.Create(ctx, tx, snap) }) require.NoError(t, err) +} - // Run the backup scan - result, err := scanner.Scan(ctx, "/home/user", snapshotID) - require.NoError(t, err) - - // Verify scan results - // The scanner counts both files and directories, so we have: - // 4 files + 4 directories (/home, /home/user, /home/user/documents, /home/user/pictures, /home/user/code) - assert.GreaterOrEqual(t, result.FilesScanned, 4, "Should scan at least 4 files") - assert.Positive(t, result.BytesScanned, "Should scan some bytes") - assert.Positive(t, result.ChunksCreated, "Should create chunks") - assert.Positive(t, result.BlobsCreated, "Should create blobs") +// TestEndToEndBackup tests the full backup workflow with mocked dependencies +// verifyEndToEndBackupState checks storage upload calls, database file +// rows, and chunk mappings after the end-to-end backup scan. +func verifyEndToEndBackupState( + ctx context.Context, t *testing.T, + repos *database.Repositories, mockStorage *MockStorer, +) { + t.Helper() // Verify storage operations calls := mockStorage.GetCalls() @@ -313,9 +262,108 @@ func TestEndToEndBackup(t *testing.T) { assert.Equal(t, 4, regularFiles, "Should have 4 regular files in database") // Verify chunks were created by checking a specific file - fileChunks, err := repos.FileChunks.GetByPath(ctx, "/home/user/documents/file1.txt") + fileChunks, err := repos.FileChunks.GetByPath(ctx, + "/home/user/documents/file1.txt") require.NoError(t, err) assert.NotEmpty(t, fileChunks, "Should have chunks for file1.txt") +} + +// newEndToEndTestConfig builds the standard config used by the +// end-to-end backup test. +func newEndToEndTestConfig() *config.Config { + return &config.Config{ + Snapshots: map[string]config.SnapshotConfig{ + testLabel: { + Paths: []string{"/home/user"}, + }, + }, + Exclude: []string{"*.tmp", "*.log"}, + ChunkSize: config.Size(16 * 1024), // 16KB chunks + BlobSizeLimit: config.Size(100 * 1024), // 100KB blobs + CompressionLevel: 3, + AgeRecipients: []string{testAgePublicKey}, + AgeSecretKey: testAgeSecretKey, + S3: config.S3Config{ + Endpoint: "http://localhost:9000", // MinIO endpoint for testing + Region: "us-east-1", + Bucket: "test-bucket", + AccessKeyID: "test-access", + SecretAccessKey: "test-secret", + }, + IndexPath: ":memory:", // In-memory SQLite database + } +} + +func TestEndToEndBackup(t *testing.T) { + // Initialize logger + log.Initialize(log.Config{}) + t.Parallel() + + // Create in-memory filesystem + fs := afero.NewMemMapFs() + + // Create test directory structure and files + testFiles := map[string]string{ + "/home/user/documents/file1.txt": "This is file 1 content", + "/home/user/documents/file2.txt": "This is file 2 content with more data", + "/home/user/pictures/photo1.jpg": "Binary photo data here...", + "/home/user/code/main.go": "package main\n\nfunc main() {\n" + + "\tprintln(\"Hello, World!\")\n}", + } + + writeTestFileTree(t, fs, testFiles) + + // Create mock storage + mockStorage := NewMockStorer() + + cfg := newEndToEndTestConfig() + + // For a true end-to-end test, we'll create a simpler test that focuses on + // the core backup logic using the scanner directly with our mock storage + ctx := context.Background() + + // Create in-memory database + db, err := database.New(ctx, ":memory:") + + require.NoError(t, err) + defer func() { + err := db.Close() + if err != nil { + t.Errorf("failed to close database: %v", err) + } + }() + + repos := database.NewRepositories(db) + + // Create scanner with mock storage + scanner := snapshot.NewScanner(snapshot.ScannerConfig{ + FS: fs, + ChunkSize: cfg.ChunkSize.Int64(), + Repositories: repos, + Storage: mockStorage, + MaxBlobSize: cfg.BlobSizeLimit.Int64(), + CompressionLevel: cfg.CompressionLevel, + AgeRecipients: cfg.AgeRecipients, + EnableProgress: false, + }) + + // Create a snapshot record + snapshotID := "test-snapshot-001" + createTestSnapshotRecord(t, repos, snapshotID) + + // Run the backup scan + result, err := scanner.Scan(ctx, "/home/user", snapshotID) + require.NoError(t, err) + + // Verify scan results. The scanner counts both files and + // directories: 4 files + directories (/home, /home/user, + // /home/user/documents, /home/user/pictures, /home/user/code). + assert.GreaterOrEqual(t, result.FilesScanned, 4, "Should scan at least 4 files") + assert.Positive(t, result.BytesScanned, "Should scan some bytes") + assert.Positive(t, result.ChunksCreated, "Should create chunks") + assert.Positive(t, result.BlobsCreated, "Should create blobs") + + verifyEndToEndBackupState(ctx, t, repos, mockStorage) // Verify blobs were uploaded to storage assert.Positive(t, mockStorage.GetStorageSize(), "Should have blobs in storage") @@ -343,6 +391,7 @@ func TestEndToEndBackup(t *testing.T) { func TestBackupAndVerify(t *testing.T) { // Initialize logger log.Initialize(log.Config{}) + t.Parallel() // Create in-memory filesystem fs := afero.NewMemMapFs() @@ -379,22 +428,12 @@ func TestBackupAndVerify(t *testing.T) { Storage: mockStorage, MaxBlobSize: int64(1024 * 1024), // 1MB blobs CompressionLevel: 3, - AgeRecipients: []string{"age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05gl0sjq9q9wjg"}, // Test public key + AgeRecipients: []string{testAgePublicKey}, }) // Create a snapshot snapshotID := "test-snapshot-001" - err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { - snapshot := &database.Snapshot{ - ID: types.SnapshotID(snapshotID), - Hostname: "test-host", - VaultikVersion: "test-version", - StartedAt: time.Now(), - } - - return repos.Snapshots.Create(ctx, tx, snapshot) - }) - require.NoError(t, err) + createTestSnapshotRecord(t, repos, snapshotID) // Run the backup result, err := scanner.Scan(ctx, "/data", snapshotID) @@ -402,12 +441,14 @@ func TestBackupAndVerify(t *testing.T) { // Verify backup created blobs assert.Positive(t, result.BlobsCreated, "Should create at least one blob") - assert.Equal(t, mockStorage.GetStorageSize(), result.BlobsCreated, "Storage should have the blobs") + assert.Equal(t, mockStorage.GetStorageSize(), result.BlobsCreated, + "Storage should have the blobs") // Verify we can retrieve the blob from storage objects, err := mockStorage.List(ctx, "blobs/") require.NoError(t, err) - assert.Len(t, objects, result.BlobsCreated, "Should have correct number of blobs in storage") + assert.Len(t, objects, result.BlobsCreated, + "Should have correct number of blobs in storage") // Get the first blob and verify it exists if len(objects) > 0 { @@ -438,66 +479,15 @@ func TestBackupAndVerify(t *testing.T) { // TestBackupAndRestore tests the full backup and restore workflow // This test verifies that the restore code correctly handles the binary SQLite // database format that is exported by the snapshot manager. -func TestBackupAndRestore(t *testing.T) { - // Initialize logger - log.Initialize(log.Config{}) - - // Create real temp directory for the database (SQLite needs real filesystem) - realTempDir, err := os.MkdirTemp("", "vaultik-test-") - require.NoError(t, err) - - defer func() { _ = os.RemoveAll(realTempDir) }() - - // Use real OS filesystem for this test - fs := afero.NewOsFs() - - // Create test directory structure and files - dataDir := filepath.Join(realTempDir, "data") - testFiles := map[string]string{ - filepath.Join(dataDir, "file1.txt"): "This is file 1 content", - filepath.Join(dataDir, "file2.txt"): "This is file 2 content with more data", - filepath.Join(dataDir, "subdir", "file3.txt"): "This is file 3 in a subdirectory", - } - - // Create directories and files - for path, content := range testFiles { - dir := filepath.Dir(path) - - err := fs.MkdirAll(dir, 0755) - if err != nil { - t.Fatalf("failed to create directory %s: %v", dir, err) - } - - err = afero.WriteFile(fs, path, []byte(content), 0644) - if err != nil { - t.Fatalf("failed to create test file %s: %v", path, err) - } - } - - ctx := context.Background() - - // Create mock storage - mockStorage := NewMockStorer() - - // Test keypair - agePublicKey := "age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05gl0sjq9q9wjg" - ageSecretKey := "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5" - - // Create database file - dbPath := filepath.Join(realTempDir, "test.db") - db, err := database.New(ctx, dbPath) - require.NoError(t, err) - - defer func() { _ = db.Close() }() - - repos := database.NewRepositories(db) - - // Create config for snapshot manager - cfg := &config.Config{ - AgeSecretKey: ageSecretKey, - AgeRecipients: []string{agePublicKey}, - CompressionLevel: 3, - } +// runBackupPhase performs the backup half of the round-trip test: +// create the snapshot, scan the data directory, complete the snapshot, +// and export its metadata, verifying the metadata upload. +func runBackupPhase( + ctx context.Context, t *testing.T, fs afero.Fs, + repos *database.Repositories, mockStorage *MockStorer, + cfg *config.Config, dataDir, dbPath, agePublicKey string, +) string { + t.Helper() // Create snapshot manager sm := snapshot.NewSnapshotManager(snapshot.SnapshotManagerParams{ @@ -519,14 +509,16 @@ func TestBackupAndRestore(t *testing.T) { }) // Create a snapshot - snapshotID, err := sm.CreateSnapshot(ctx, "test-host", "test-version", "test-git") + snapshotID, err := sm.CreateSnapshotWithName( + ctx, testHostname, "", "test-version", "test-git") require.NoError(t, err) t.Logf("Created snapshot: %s", snapshotID) // Run the backup (scan) result, err := scanner.Scan(ctx, dataDir, snapshotID) require.NoError(t, err) - t.Logf("Scan complete: %d files, %d blobs", result.FilesScanned, result.BlobsCreated) + t.Logf("Scan complete: %d files, %d blobs", + result.FilesScanned, result.BlobsCreated) // Complete the snapshot err = sm.CompleteSnapshot(ctx, snapshotID) @@ -541,7 +533,63 @@ func TestBackupAndRestore(t *testing.T) { keys, err := mockStorage.List(ctx, "metadata/") require.NoError(t, err) t.Logf("Metadata keys: %v", keys) - assert.GreaterOrEqual(t, len(keys), 2, "Should have at least db.zst.age and manifest.json.zst") + assert.GreaterOrEqual(t, len(keys), 2, + "Should have at least db.zst.age and manifest.json.zst") + + return snapshotID +} + +func TestBackupAndRestore(t *testing.T) { + // Initialize logger + log.Initialize(log.Config{}) + t.Parallel() + + // Create real temp directory for the database (SQLite needs real filesystem) + realTempDir := t.TempDir() + + // Use real OS filesystem for this test + fs := afero.NewOsFs() + + // Create test directory structure and files + dataDir := filepath.Join(realTempDir, "data") + testFiles := map[string]string{ + filepath.Join(dataDir, "file1.txt"): "This is file 1 content", + filepath.Join(dataDir, "file2.txt"): "This is file 2 content " + + "with more data", + filepath.Join(dataDir, "subdir", "file3.txt"): "This is file 3 " + + "in a subdirectory", + } + + // Create directories and files + writeTestFileTree(t, fs, testFiles) + + ctx := context.Background() + + // Create mock storage + mockStorage := NewMockStorer() + + // Test keypair + agePublicKey := testAgePublicKey + ageSecretKey := testAgeSecretKey + + // Create database file + dbPath := filepath.Join(realTempDir, "test.db") + db, err := database.New(ctx, dbPath) + require.NoError(t, err) + + defer func() { _ = db.Close() }() + + repos := database.NewRepositories(db) + + // Create config for snapshot manager + cfg := &config.Config{ + AgeSecretKey: ageSecretKey, + AgeRecipients: []string{agePublicKey}, + CompressionLevel: 3, + } + + snapshotID := runBackupPhase( + ctx, t, fs, repos, mockStorage, cfg, dataDir, dbPath, agePublicKey) // Close the source database err = db.Close() @@ -570,49 +618,32 @@ func TestBackupAndRestore(t *testing.T) { for origPath, expectedContent := range testFiles { restoredPath := filepath.Join(restoreDir, origPath) restoredContent, err := afero.ReadFile(fs, restoredPath) - require.NoError(t, err, "Should be able to read restored file: %s", restoredPath) - assert.Equal(t, expectedContent, string(restoredContent), "Restored content should match original for: %s", origPath) + require.NoError(t, err, + "Should be able to read restored file: %s", restoredPath) + assert.Equal(t, expectedContent, string(restoredContent), + "Restored content should match original for: %s", origPath) } t.Log("Backup and restore test completed successfully") } -// TestEndToEndFileStorage exercises the full backup → restore loop against the -// real `file://` storage backend (FileStorer) on a real OS filesystem. This is -// the closest local approximation of a production backup: encrypted blobs get -// written to disk, the metadata SQLite database is exported through the same -// blobgen pipeline as a real backup, and restoration reads them back through -// the public Vaultik.Restore entrypoint. It is the canonical end-to-end smoke -// test for 1.0. -func TestEndToEndFileStorage(t *testing.T) { - log.Initialize(log.Config{}) - - // Real OS filesystem (SQLite + FileStorer both need it). - fs := afero.NewOsFs() - tempDir, err := os.MkdirTemp("", "vaultik-e2e-") - require.NoError(t, err) - - defer func() { _ = os.RemoveAll(tempDir) }() - - dataDir := filepath.Join(tempDir, "source") - storeDir := filepath.Join(tempDir, "remote") - restoreDir := filepath.Join(tempDir, "restored") - dbPath := filepath.Join(tempDir, "index.sqlite") - - // Write a representative mix of file sizes: - // - empty file - // - tiny text file - // - file just under chunk boundary - // - file forcing multiple chunks - // - nested subdirectories - chunkSize := int64(64 * 1024) - maxBlobSize := int64(512 * 1024) +// setupE2ESourceTree writes a representative mix of file sizes (empty +// file, tiny text file, file under the chunk boundary, multi-chunk +// file, nested subdirectories), a permission-restricted file, an empty +// directory, and a symlink. It returns the content map keyed by path +// plus the restricted-file, empty-dir, and symlink paths. +func setupE2ESourceTree( + t *testing.T, fs afero.Fs, dataDir string, chunkSize int64, +) (map[string][]byte, string, string, string) { + t.Helper() testFiles := map[string][]byte{ - filepath.Join(dataDir, "empty.txt"): {}, - filepath.Join(dataDir, "small.txt"): []byte("hello vaultik"), - filepath.Join(dataDir, "subdir", "medium.bin"): bytesPattern("medium-", int(chunkSize/2)), - filepath.Join(dataDir, "subdir", "large.bin"): bytesPattern("large-", int(chunkSize*4)), + filepath.Join(dataDir, "empty.txt"): {}, + filepath.Join(dataDir, "small.txt"): []byte("hello vaultik"), + filepath.Join(dataDir, "subdir", "medium.bin"): bytesPattern( + "medium-", int(chunkSize/2)), + filepath.Join(dataDir, "subdir", "large.bin"): bytesPattern( + "large-", int(chunkSize*4)), filepath.Join(dataDir, "deep", "nest", "leaf.txt"): []byte("leaf"), } @@ -624,6 +655,7 @@ func TestEndToEndFileStorage(t *testing.T) { // Create a file with non-default permissions. restrictedPath := filepath.Join(dataDir, "restricted.txt") require.NoError(t, afero.WriteFile(fs, restrictedPath, []byte("secret"), 0o600)) + testFiles[restrictedPath] = []byte("secret") // Create an empty directory (should survive round-trip). @@ -634,22 +666,38 @@ func TestEndToEndFileStorage(t *testing.T) { symlinkPath := filepath.Join(dataDir, "link-to-small") require.NoError(t, os.Symlink("small.txt", symlinkPath)) + return testFiles, restrictedPath, emptyDir, symlinkPath +} + +// TestEndToEndFileStorage exercises the full backup → restore loop against the +// real `file://` storage backend (FileStorer) on a real OS filesystem. This is +// the closest local approximation of a production backup: encrypted blobs get +// written to disk, the metadata SQLite database is exported through the same +// blobgen pipeline as a real backup, and restoration reads them back through +// the public Vaultik.Restore entrypoint. It is the canonical end-to-end smoke +// test for 1.0. +// runFileStorageBackup performs the backup half of the file-storage +// end-to-end test against a real on-disk FileStorer, verifies the +// on-disk layout, and closes the index database so the restore half +// runs from remote bytes only. +func runFileStorageBackup( + ctx context.Context, t *testing.T, fs afero.Fs, + dataDir, storeDir, dbPath string, + chunkSize, maxBlobSize int64, +) (*config.Config, *storage.FileStorer, string) { + t.Helper() + // FileStorer is the real-world local-disk backend. storer, err := storage.NewFileStorer(storeDir) require.NoError(t, err) - agePublicKey := "age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05gl0sjq9q9wjg" - ageSecretKey := "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5" - cfg := &config.Config{ - AgeRecipients: []string{agePublicKey}, - AgeSecretKey: ageSecretKey, + AgeRecipients: []string{testAgePublicKey}, + AgeSecretKey: testAgeSecretKey, CompressionLevel: 3, - Hostname: "test-host", + Hostname: testHostname, } - ctx := context.Background() - db, err := database.New(ctx, dbPath) require.NoError(t, err) @@ -674,7 +722,8 @@ func TestEndToEndFileStorage(t *testing.T) { Repositories: repos, }) - snapshotID, err := sm.CreateSnapshotWithName(ctx, cfg.Hostname, "e2e", "test-version", "test-git") + snapshotID, err := sm.CreateSnapshotWithName( + ctx, cfg.Hostname, "e2e", "test-version", "test-git") require.NoError(t, err) scanResult, err := scanner.Scan(ctx, dataDir, snapshotID) @@ -692,7 +741,8 @@ func TestEndToEndFileStorage(t *testing.T) { require.NoError(t, err) require.True(t, blobInfo.IsDir()) - metaInfo, err := os.Stat(filepath.Join(storeDir, "metadata", snapshot.RemoteSnapshotKey(snapshotID))) + metaInfo, err := os.Stat(filepath.Join( + storeDir, "metadata", snapshot.RemoteSnapshotKey(snapshotID))) require.NoError(t, err) require.True(t, metaInfo.IsDir()) @@ -700,6 +750,39 @@ func TestEndToEndFileStorage(t *testing.T) { // the remote bytes plus the secret key, with no help from the local index. require.NoError(t, db.Close()) + return cfg, storer, snapshotID +} + +func TestEndToEndFileStorage(t *testing.T) { + log.Initialize(log.Config{}) + t.Parallel() + + // Real OS filesystem (SQLite + FileStorer both need it). + fs := afero.NewOsFs() + tempDir := t.TempDir() + + dataDir := filepath.Join(tempDir, "source") + storeDir := filepath.Join(tempDir, "remote") + restoreDir := filepath.Join(tempDir, "restored") + dbPath := filepath.Join(tempDir, "index.sqlite") + + // Write a representative mix of file sizes: + // - empty file + // - tiny text file + // - file just under chunk boundary + // - file forcing multiple chunks + // - nested subdirectories + chunkSize := int64(64 * 1024) + maxBlobSize := int64(512 * 1024) + + testFiles, restrictedPath, emptyDir, symlinkPath := + setupE2ESourceTree(t, fs, dataDir, chunkSize) + + ctx := context.Background() + + cfg, storer, snapshotID := runFileStorageBackup( + ctx, t, fs, dataDir, storeDir, dbPath, chunkSize, maxBlobSize) + restoreVaultik := &vaultik.Vaultik{ Config: cfg, Storage: storer, @@ -716,6 +799,19 @@ func TestEndToEndFileStorage(t *testing.T) { Verify: true, })) + verifyE2ERestoredTree(t, fs, restoreDir, testFiles, + restrictedPath, emptyDir, symlinkPath) +} + +// verifyE2ERestoredTree byte-compares every restored file and checks the +// restricted-permission file, empty directory, and symlink special cases. +func verifyE2ERestoredTree( + t *testing.T, fs afero.Fs, restoreDir string, + testFiles map[string][]byte, + restrictedPath, emptyDir, symlinkPath string, +) { + t.Helper() + // Byte-equality compare every original against its restored copy. for origPath, expected := range testFiles { restoredPath := filepath.Join(restoreDir, origPath) @@ -751,22 +847,22 @@ func TestEndToEndFileStorage(t *testing.T) { // regression where snapshot_blobs was populated only for blobs uploaded // during the snapshot, leaving fully-deduplicated snapshots unrestorable // with "chunk X not found in any blob" errors. -func TestDedupOnlySnapshotRestores(t *testing.T) { - log.Initialize(log.Config{}) +// dedupBackupEnv bundles the moving parts of the dedup round-trip test. +type dedupBackupEnv struct { + cfg *config.Config + storer *storage.FileStorer + db *database.DB + repos *database.Repositories + sm *snapshot.SnapshotManager + makeScanner func() *snapshot.Scanner +} - fs := afero.NewOsFs() - tempDir, err := os.MkdirTemp("", "vaultik-dedup-") - require.NoError(t, err) - - defer func() { _ = os.RemoveAll(tempDir) }() - - dataDir := filepath.Join(tempDir, "source") - storeDir := filepath.Join(tempDir, "remote") - restoreDir := filepath.Join(tempDir, "restored") - dbPath := filepath.Join(tempDir, "index.sqlite") - - chunkSize := int64(64 * 1024) - maxBlobSize := int64(512 * 1024) +// writeDedupSourceFiles writes the two deterministic source files used +// by the dedup test and returns their expected contents by path. +func writeDedupSourceFiles( + t *testing.T, fs afero.Fs, dataDir string, chunkSize int64, +) map[string][]byte { + t.Helper() testFiles := map[string][]byte{ filepath.Join(dataDir, "a.bin"): bytesPattern("a-", int(chunkSize*3)), @@ -777,25 +873,30 @@ func TestDedupOnlySnapshotRestores(t *testing.T) { require.NoError(t, afero.WriteFile(fs, path, content, 0o644)) } + return testFiles +} + +// setupDedupBackupEnv creates the storer, config, database, snapshot +// manager, and scanner factory for the dedup round-trip test. +func setupDedupBackupEnv( + ctx context.Context, t *testing.T, fs afero.Fs, + storeDir, dbPath string, chunkSize, maxBlobSize int64, +) *dedupBackupEnv { + t.Helper() + storer, err := storage.NewFileStorer(storeDir) require.NoError(t, err) - agePublicKey := "age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05gl0sjq9q9wjg" - ageSecretKey := "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5" - cfg := &config.Config{ - AgeRecipients: []string{agePublicKey}, - AgeSecretKey: ageSecretKey, + AgeRecipients: []string{testAgePublicKey}, + AgeSecretKey: testAgeSecretKey, CompressionLevel: 3, - Hostname: "test-host", + Hostname: testHostname, } - ctx := context.Background() db, err := database.New(ctx, dbPath) require.NoError(t, err) - defer func() { _ = db.Close() }() - repos := database.NewRepositories(db) makeScanner := func() *snapshot.Scanner { @@ -814,31 +915,85 @@ func TestDedupOnlySnapshotRestores(t *testing.T) { }) sm.SetFilesystem(fs) + return &dedupBackupEnv{ + cfg: cfg, + storer: storer, + db: db, + repos: repos, + sm: sm, + makeScanner: makeScanner, + } +} + +// runDedupSnapshot creates a "dedup" snapshot, scans dataDir into it, +// completes it, and exports its metadata, returning the snapshot ID and +// scan result. +func runDedupSnapshot( + ctx context.Context, t *testing.T, + sm *snapshot.SnapshotManager, scanner *snapshot.Scanner, + hostname, dataDir, dbPath string, +) (string, *snapshot.ScanResult) { + t.Helper() + + id, err := sm.CreateSnapshotWithName(ctx, hostname, "dedup", "v", "g") + require.NoError(t, err) + + result, err := scanner.Scan(ctx, dataDir, id) + require.NoError(t, err) + + require.NoError(t, sm.CompleteSnapshot(ctx, id)) + require.NoError(t, sm.ExportSnapshotMetadata(ctx, dbPath, id)) + + return id, result +} + +func TestDedupOnlySnapshotRestores(t *testing.T) { + log.Initialize(log.Config{}) + t.Parallel() + + fs := afero.NewOsFs() + tempDir := t.TempDir() + + dataDir := filepath.Join(tempDir, "source") + storeDir := filepath.Join(tempDir, "remote") + restoreDir := filepath.Join(tempDir, "restored") + dbPath := filepath.Join(tempDir, "index.sqlite") + + chunkSize := int64(64 * 1024) + maxBlobSize := int64(512 * 1024) + + ctx := context.Background() + + testFiles := writeDedupSourceFiles(t, fs, dataDir, chunkSize) + + env := setupDedupBackupEnv( + ctx, t, fs, storeDir, dbPath, chunkSize, maxBlobSize) + defer func() { _ = env.db.Close() }() + + cfg, storer, repos, sm := env.cfg, env.storer, env.repos, env.sm + makeScanner := env.makeScanner + db := env.db + // First snapshot — uploads all blobs. - id1, err := sm.CreateSnapshotWithName(ctx, cfg.Hostname, "dedup", "v", "g") - require.NoError(t, err) - r1, err := makeScanner().Scan(ctx, dataDir, id1) - require.NoError(t, err) - require.Positive(t, r1.BlobsCreated, "first snapshot should upload at least one blob") - require.NoError(t, sm.CompleteSnapshot(ctx, id1)) - require.NoError(t, sm.ExportSnapshotMetadata(ctx, dbPath, id1)) + _, r1 := runDedupSnapshot(ctx, t, sm, makeScanner(), + cfg.Hostname, dataDir, dbPath) + require.Positive(t, r1.BlobsCreated, + "first snapshot should upload at least one blob") // Second snapshot — same data, every chunk dedups. Sleep past the // second-precision timestamp so the snapshot IDs differ. time.Sleep(1100 * time.Millisecond) - id2, err := sm.CreateSnapshotWithName(ctx, cfg.Hostname, "dedup", "v", "g") - require.NoError(t, err) - r2, err := makeScanner().Scan(ctx, dataDir, id2) - require.NoError(t, err) - require.Equal(t, 0, r2.BlobsCreated, "second snapshot should upload zero new blobs (fully dedup'd)") - require.NoError(t, sm.CompleteSnapshot(ctx, id2)) - require.NoError(t, sm.ExportSnapshotMetadata(ctx, dbPath, id2)) + id2, r2 := runDedupSnapshot(ctx, t, sm, makeScanner(), + cfg.Hostname, dataDir, dbPath) + require.Equal(t, 0, r2.BlobsCreated, + "second snapshot should upload zero new blobs (fully dedup'd)") // snapshot_blobs for id2 must be populated despite no uploads. blobHashes, err := repos.Snapshots.GetBlobHashes(ctx, id2) require.NoError(t, err) - require.NotEmpty(t, blobHashes, "snapshot_blobs for fully-dedup'd snapshot must reference blobs uploaded by prior snapshot") + require.NotEmpty(t, blobHashes, "snapshot_blobs for fully-dedup'd snapshot "+ + "must reference blobs uploaded by prior snapshot") require.NoError(t, db.Close()) @@ -871,7 +1026,7 @@ func TestDedupOnlySnapshotRestores(t *testing.T) { func bytesPattern(tag string, n int) []byte { out := make([]byte, n) for i := range out { - out[i] = byte(tag[i%len(tag)] ^ byte(i&0xff)) + out[i] = tag[i%len(tag)] ^ byte(i&0xff) } return out diff --git a/internal/vaultik/prune.go b/internal/vaultik/prune.go index 67cd06c..328b0b2 100644 --- a/internal/vaultik/prune.go +++ b/internal/vaultik/prune.go @@ -6,7 +6,6 @@ import ( "fmt" "strings" - "github.com/dustin/go-humanize" "sneak.berlin/go/vaultik/internal/log" ) @@ -16,6 +15,14 @@ type PruneOptions struct { JSON bool } +// errNukeRequiresForce guards the destructive remote nuke operation. +var errNukeRequiresForce = errors.New( + "nuke requires --force (this deletes ALL remote snapshots and blobs)") + +// metadataDirName is the top-level remote directory holding snapshot +// metadata. +const metadataDirName = "metadata" + // NukeRemote deletes every snapshot's metadata and every blob from remote // storage. After this returns successfully the bucket prefix is empty and // the next backup starts from scratch. @@ -24,18 +31,19 @@ type PruneOptions struct { // confirming with the user. func (v *Vaultik) NukeRemote(force bool) error { if !force { - return errors.New("nuke requires --force (this deletes ALL remote snapshots and blobs)") + return errNukeRequiresForce } v.UI.Beginf("Removing all snapshot metadata from backup destination store.") - if _, err := v.RemoveAllSnapshots(&RemoveOptions{Force: true}); err != nil { + _, err := v.RemoveAllSnapshots(&RemoveOptions{Force: true}) + if err != nil { return fmt.Errorf("removing all snapshots: %w", err) } v.UI.Beginf("Removing any blobs still present in backup destination store.") - err := v.PruneBlobs(&PruneOptions{Force: true}) + err = v.PruneBlobs(&PruneOptions{Force: true}) if err != nil { return fmt.Errorf("pruning blobs: %w", err) } @@ -46,6 +54,8 @@ func (v *Vaultik) NukeRemote(force bool) error { } // PruneBlobsResult contains the result of a blob prune operation +// +//nolint:tagliatelle // snake_case is the established JSON output format type PruneBlobsResult struct { BlobsFound int `json:"blobs_found"` BlobsDeleted int `json:"blobs_deleted"` @@ -74,7 +84,8 @@ func (v *Vaultik) Prune(opts *PruneOptions) error { return fmt.Errorf("reconciling local snapshots with remote: %w", err) } - if _, err := v.PruneDatabase(); err != nil { + _, err = v.PruneDatabase() + if err != nil { return fmt.Errorf("pruning local database: %w", err) } @@ -111,20 +122,24 @@ func (v *Vaultik) PruneBlobs(opts *PruneOptions) error { return nil } - log.Info("Found unreferenced blobs", "count", len(unreferencedBlobs), "total_size", humanize.Bytes(uint64(totalSize))) + log.Info("Found unreferenced blobs", + "count", len(unreferencedBlobs), "total_size", ubytes(totalSize)) if !opts.JSON { - v.stdoutf("Found %d unreferenced blob(s) totaling %s\n", len(unreferencedBlobs), humanize.Bytes(uint64(totalSize))) + v.stdoutf("Found %d unreferenced blob(s) totaling %s\n", + len(unreferencedBlobs), ubytes(totalSize)) } if !opts.Force && !opts.JSON { v.stdoutf("\nDelete %d unreferenced blob(s)? [y/N] ", len(unreferencedBlobs)) var confirm string - if _, err := v.scanStdin(&confirm); err != nil { + + _, err = v.scanStdin(&confirm) + if err != nil { v.printlnStdout("Cancelled") - return nil + return nil //nolint:nilerr // read failure means no confirmation } if strings.ToLower(confirm) != "y" { @@ -140,7 +155,8 @@ func (v *Vaultik) PruneBlobs(opts *PruneOptions) error { return v.outputPruneBlobsJSON(result) } - v.stdoutf("\nDeleted %d blob(s) totaling %s\n", result.BlobsDeleted, humanize.Bytes(uint64(result.BytesFreed))) + v.stdoutf("\nDeleted %d blob(s) totaling %s\n", + result.BlobsDeleted, ubytes(result.BytesFreed)) if result.BlobsFailed > 0 { v.stdoutf("Failed to delete %d blob(s)\n", result.BlobsFailed) @@ -149,7 +165,8 @@ func (v *Vaultik) PruneBlobs(opts *PruneOptions) error { return nil } -// collectReferencedBlobs downloads all manifests and returns the set of referenced blob hashes +// collectReferencedBlobs downloads all manifests and returns the set of +// referenced blob hashes. func (v *Vaultik) collectReferencedBlobs() (map[string]bool, error) { log.Info("Listing remote snapshots") // IDs returned by listUniqueSnapshotIDs are remote keys (hashed @@ -181,7 +198,8 @@ func (v *Vaultik) collectReferencedBlobs() (map[string]bool, error) { manifestCount++ } - log.Info("Processed manifests", "count", manifestCount, "unique_blobs_referenced", len(allBlobsReferenced)) + log.Info("Processed manifests", + "count", manifestCount, "unique_blobs_referenced", len(allBlobsReferenced)) return allBlobsReferenced, nil } @@ -199,8 +217,10 @@ func (v *Vaultik) listUniqueSnapshotIDs() ([]string, error) { } parts := strings.Split(object.Key, "/") - if len(parts) >= 2 && parts[0] == "metadata" && parts[1] != "" { - if strings.HasSuffix(object.Key, "/") || strings.Contains(object.Key, "/manifest.json.zst") { + if len(parts) >= minSnapshotIDParts && + parts[0] == metadataDirName && parts[1] != "" { + if strings.HasSuffix(object.Key, "/") || + strings.Contains(object.Key, "/manifest.json.zst") { snapshotID := parts[1] if !seen[snapshotID] { seen[snapshotID] = true @@ -226,7 +246,7 @@ func (v *Vaultik) listAllRemoteBlobs() (map[string]int64, error) { } parts := strings.Split(object.Key, "/") - if len(parts) == 4 && parts[0] == "blobs" { + if len(parts) == blobKeyParts && parts[0] == "blobs" { allBlobs[parts[3]] = object.Size } } @@ -236,8 +256,11 @@ func (v *Vaultik) listAllRemoteBlobs() (map[string]int64, error) { return allBlobs, nil } -// findUnreferencedBlobs returns blob hashes not referenced by any manifest and their total size -func (v *Vaultik) findUnreferencedBlobs(allBlobs map[string]int64, referenced map[string]bool) ([]string, int64) { +// findUnreferencedBlobs returns blob hashes not referenced by any +// manifest and their total size. +func (v *Vaultik) findUnreferencedBlobs( + allBlobs map[string]int64, referenced map[string]bool, +) ([]string, int64) { var ( unreferenced []string totalSize int64 @@ -253,8 +276,11 @@ func (v *Vaultik) findUnreferencedBlobs(allBlobs map[string]int64, referenced ma return unreferenced, totalSize } -// deleteUnreferencedBlobs deletes the given blobs from storage and populates the result -func (v *Vaultik) deleteUnreferencedBlobs(unreferencedBlobs []string, allBlobs map[string]int64, result *PruneBlobsResult) { +// deleteUnreferencedBlobs deletes the given blobs from storage and +// populates the result. +func (v *Vaultik) deleteUnreferencedBlobs( + unreferencedBlobs []string, allBlobs map[string]int64, result *PruneBlobsResult, +) { log.Info("Deleting unreferenced blobs") for i, hash := range unreferencedBlobs { @@ -270,11 +296,12 @@ func (v *Vaultik) deleteUnreferencedBlobs(unreferencedBlobs []string, allBlobs m result.BlobsDeleted++ result.BytesFreed += allBlobs[hash] - if (i+1)%100 == 0 || i == len(unreferencedBlobs)-1 { + if (i+1)%progressLogEvery == 0 || i == len(unreferencedBlobs)-1 { log.Info("Deletion progress", "deleted", i+1, "total", len(unreferencedBlobs), - "percent", fmt.Sprintf("%.1f%%", float64(i+1)/float64(len(unreferencedBlobs))*100), + "percent", fmt.Sprintf("%.1f%%", + float64(i+1)/float64(len(unreferencedBlobs))*percentScale), ) } } @@ -283,7 +310,7 @@ func (v *Vaultik) deleteUnreferencedBlobs(unreferencedBlobs []string, allBlobs m log.Info("Prune complete", "deleted_count", result.BlobsDeleted, - "deleted_size", humanize.Bytes(uint64(result.BytesFreed)), + "deleted_size", ubytes(result.BytesFreed), "failed", result.BlobsFailed, ) } diff --git a/internal/vaultik/purge_per_name_test.go b/internal/vaultik/purge_per_name_test.go index b8c4782..318f27c 100644 --- a/internal/vaultik/purge_per_name_test.go +++ b/internal/vaultik/purge_per_name_test.go @@ -16,12 +16,19 @@ import ( "sneak.berlin/go/vaultik/internal/vaultik" ) +// Snapshot IDs reused across the purge tests. +const ( + snapSystemT0 = "testhost_system_2026-01-01T00:00:00Z" + snapHomeT0 = "testhost_home_2026-01-01T00:00:00Z" + snapHomeT1 = "testhost_home_2026-01-01T01:00:00Z" + snapHomeT3 = "testhost_home_2026-01-01T03:00:00Z" +) + // setupPurgeTest creates a Vaultik instance with an in-memory database and mock // storage pre-populated with the given snapshot IDs. Each snapshot is marked as // completed. Remote metadata stubs are created so syncWithRemote keeps them. func setupPurgeTest(t *testing.T, snapshotIDs []string) *vaultik.Vaultik { t.Helper() - log.Initialize(log.Config{}) ctx := context.Background() db, err := database.New(ctx, ":memory:") @@ -44,7 +51,7 @@ func setupPurgeTest(t *testing.T, snapshotIDs []string) *vaultik.Vaultik { snap := &database.Snapshot{ ID: types.SnapshotID(id), Hostname: "testhost", - VaultikVersion: "test", + VaultikVersion: testLabel, StartedAt: startedAt, CompletedAt: &completedAt, } @@ -96,13 +103,16 @@ func listRemainingSnapshots(t *testing.T, v *vaultik.Vaultik) []string { } func TestPurgeKeepLatest_PerName(t *testing.T) { + log.Initialize(log.Config{}) + t.Parallel() + // Create snapshots for two different names: "home" and "system". // With per-name --keep-latest, the latest of each should be kept. snapshotIDs := []string{ - "testhost_system_2026-01-01T00:00:00Z", - "testhost_home_2026-01-01T01:00:00Z", + snapSystemT0, + snapHomeT1, "testhost_system_2026-01-01T02:00:00Z", - "testhost_home_2026-01-01T03:00:00Z", + snapHomeT3, "testhost_system_2026-01-01T04:00:00Z", } @@ -118,15 +128,19 @@ func TestPurgeKeepLatest_PerName(t *testing.T) { // Should keep the latest of each name assert.Len(t, remaining, 2, "should keep exactly 2 snapshots (one per name)") - assert.Contains(t, remaining, "testhost_system_2026-01-01T04:00:00Z", "should keep latest system") - assert.Contains(t, remaining, "testhost_home_2026-01-01T03:00:00Z", "should keep latest home") + assert.Contains(t, remaining, "testhost_system_2026-01-01T04:00:00Z", + "should keep latest system") + assert.Contains(t, remaining, snapHomeT3, "should keep latest home") } func TestPurgeKeepLatest_SingleName(t *testing.T) { + log.Initialize(log.Config{}) + t.Parallel() + // All snapshots have the same name — keep-latest should keep exactly one. snapshotIDs := []string{ - "testhost_home_2026-01-01T00:00:00Z", - "testhost_home_2026-01-01T01:00:00Z", + snapHomeT0, + snapHomeT1, "testhost_home_2026-01-01T02:00:00Z", } @@ -140,17 +154,21 @@ func TestPurgeKeepLatest_SingleName(t *testing.T) { remaining := listRemainingSnapshots(t, v) assert.Len(t, remaining, 1) - assert.Contains(t, remaining, "testhost_home_2026-01-01T02:00:00Z", "should keep the newest") + assert.Contains(t, remaining, "testhost_home_2026-01-01T02:00:00Z", + "should keep the newest") } func TestPurgeKeepLatest_WithNameFilter(t *testing.T) { + log.Initialize(log.Config{}) + t.Parallel() + // Use --name to filter purge to only "home" snapshots. // "system" snapshots should be untouched. snapshotIDs := []string{ - "testhost_system_2026-01-01T00:00:00Z", - "testhost_home_2026-01-01T01:00:00Z", + snapSystemT0, + snapHomeT1, "testhost_system_2026-01-01T02:00:00Z", - "testhost_home_2026-01-01T03:00:00Z", + snapHomeT3, "testhost_home_2026-01-01T04:00:00Z", } @@ -167,12 +185,15 @@ func TestPurgeKeepLatest_WithNameFilter(t *testing.T) { // 2 system snapshots untouched + 1 latest home = 3 assert.Len(t, remaining, 3) - assert.Contains(t, remaining, "testhost_system_2026-01-01T00:00:00Z") + assert.Contains(t, remaining, snapSystemT0) assert.Contains(t, remaining, "testhost_system_2026-01-01T02:00:00Z") assert.Contains(t, remaining, "testhost_home_2026-01-01T04:00:00Z") } func TestPurgeKeepLatest_NoSnapshots(t *testing.T) { + log.Initialize(log.Config{}) + t.Parallel() + v := setupPurgeTest(t, nil) err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{ @@ -183,8 +204,11 @@ func TestPurgeKeepLatest_NoSnapshots(t *testing.T) { } func TestPurgeKeepLatest_NameFilterNoMatch(t *testing.T) { + log.Initialize(log.Config{}) + t.Parallel() + snapshotIDs := []string{ - "testhost_system_2026-01-01T00:00:00Z", + snapSystemT0, "testhost_system_2026-01-01T01:00:00Z", } @@ -203,13 +227,16 @@ func TestPurgeKeepLatest_NameFilterNoMatch(t *testing.T) { } func TestPurgeOlderThan_WithNameFilter(t *testing.T) { + log.Initialize(log.Config{}) + t.Parallel() + // Snapshots with different names and timestamps. // --older-than should apply only to the named subset when --name is used. snapshotIDs := []string{ "testhost_system_2020-01-01T00:00:00Z", "testhost_home_2020-01-01T00:00:00Z", - "testhost_system_2026-01-01T00:00:00Z", - "testhost_home_2026-01-01T00:00:00Z", + snapSystemT0, + snapHomeT0, } v := setupPurgeTest(t, snapshotIDs) @@ -227,14 +254,17 @@ func TestPurgeOlderThan_WithNameFilter(t *testing.T) { // Old system stays (not filtered by name), old home deleted, recent ones stay assert.Len(t, remaining, 3) assert.Contains(t, remaining, "testhost_system_2020-01-01T00:00:00Z") - assert.Contains(t, remaining, "testhost_system_2026-01-01T00:00:00Z") - assert.Contains(t, remaining, "testhost_home_2026-01-01T00:00:00Z") + assert.Contains(t, remaining, snapSystemT0) + assert.Contains(t, remaining, snapHomeT0) } func TestPurgeKeepLatest_ThreeNames(t *testing.T) { + log.Initialize(log.Config{}) + t.Parallel() + // Three different snapshot names with multiple snapshots each. snapshotIDs := []string{ - "testhost_home_2026-01-01T00:00:00Z", + snapHomeT0, "testhost_system_2026-01-01T01:00:00Z", "testhost_media_2026-01-01T02:00:00Z", "testhost_home_2026-01-01T03:00:00Z", diff --git a/internal/vaultik/remove_snapshot_test.go b/internal/vaultik/remove_snapshot_test.go index 2bbf33f..bed9ace 100644 --- a/internal/vaultik/remove_snapshot_test.go +++ b/internal/vaultik/remove_snapshot_test.go @@ -29,7 +29,7 @@ func newTestStorer() *testStorer { } } -func (s *testStorer) Put(ctx context.Context, key string, reader io.Reader) error { +func (s *testStorer) Put(_ context.Context, key string, reader io.Reader) error { s.mu.Lock() defer s.mu.Unlock() @@ -43,11 +43,14 @@ func (s *testStorer) Put(ctx context.Context, key string, reader io.Reader) erro return nil } -func (s *testStorer) PutWithProgress(ctx context.Context, key string, reader io.Reader, size int64, progress storage.ProgressCallback) error { +func (s *testStorer) PutWithProgress( + ctx context.Context, key string, reader io.Reader, + _ int64, _ storage.ProgressCallback, +) error { return s.Put(ctx, key, reader) } -func (s *testStorer) Get(ctx context.Context, key string) (io.ReadCloser, error) { +func (s *testStorer) Get(_ context.Context, key string) (io.ReadCloser, error) { s.mu.Lock() defer s.mu.Unlock() @@ -59,7 +62,7 @@ func (s *testStorer) Get(ctx context.Context, key string) (io.ReadCloser, error) return io.NopCloser(bytes.NewReader(data)), nil } -func (s *testStorer) Stat(ctx context.Context, key string) (*storage.ObjectInfo, error) { +func (s *testStorer) Stat(_ context.Context, key string) (*storage.ObjectInfo, error) { s.mu.Lock() defer s.mu.Unlock() @@ -74,7 +77,7 @@ func (s *testStorer) Stat(ctx context.Context, key string) (*storage.ObjectInfo, }, nil } -func (s *testStorer) Delete(ctx context.Context, key string) error { +func (s *testStorer) Delete(_ context.Context, key string) error { s.mu.Lock() defer s.mu.Unlock() @@ -83,7 +86,7 @@ func (s *testStorer) Delete(ctx context.Context, key string) error { return nil } -func (s *testStorer) List(ctx context.Context, prefix string) ([]string, error) { +func (s *testStorer) List(_ context.Context, prefix string) ([]string, error) { s.mu.Lock() defer s.mu.Unlock() @@ -98,7 +101,9 @@ func (s *testStorer) List(ctx context.Context, prefix string) ([]string, error) return keys, nil } -func (s *testStorer) ListStream(ctx context.Context, prefix string) <-chan storage.ObjectInfo { +func (s *testStorer) ListStream( + _ context.Context, prefix string, +) <-chan storage.ObjectInfo { ch := make(chan storage.ObjectInfo) go func() { @@ -120,6 +125,13 @@ func (s *testStorer) ListStream(ctx context.Context, prefix string) <-chan stora return ch } +func (s *testStorer) Info() storage.Info { + return storage.Info{ + Type: testLabel, + Location: "memory", + } +} + func (s *testStorer) hasKey(key string) bool { s.mu.Lock() defer s.mu.Unlock() @@ -136,17 +148,16 @@ func (s *testStorer) keyCount() int { return len(s.data) } -func (s *testStorer) Info() storage.StorageInfo { - return storage.StorageInfo{ - Type: "test", - Location: "memory", - } -} +// testBlobHashA is a fixture blob hash reused across the remove tests. +const testBlobHashA = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" // addManifest creates a compressed manifest in storage at the same // hashed path the production code uses. snapshotID is the human ID; // the storage path uses RemoteSnapshotKey(id). -func addManifest(t *testing.T, store *testStorer, snapshotID string, blobHashes []string) { +func addManifest( + t *testing.T, store *testStorer, snapshotID string, blobHashes []string, +) { t.Helper() blobs := make([]snapshot.BlobInfo, len(blobHashes)) @@ -173,11 +184,12 @@ func addManifest(t *testing.T, store *testStorer, snapshotID string, blobHashes } // remoteKeyPath returns the storage-relative path to a snapshot's -// metadata directory or manifest under the hashed remote-key scheme. -// Tests use this in hasKey/asserts to avoid scattering RemoteSnapshotKey -// calls throughout. -func remoteKeyPath(snapshotID, suffix string) string { - return "metadata/" + snapshot.RemoteSnapshotKey(snapshotID) + "/" + suffix +// manifest under the hashed remote-key scheme. Tests use this in +// hasKey/asserts to avoid scattering RemoteSnapshotKey calls +// throughout. +func remoteKeyPath(snapshotID string) string { + return "metadata/" + snapshot.RemoteSnapshotKey(snapshotID) + + "/manifest.json.zst" } // addBlob adds a fake blob to storage @@ -206,10 +218,11 @@ func addBlob(t *testing.T, store *testStorer, hash string) { // untouched. func TestRemoveSnapshot_LocalOnly_PreservesRemote(t *testing.T) { log.Initialize(log.Config{}) + t.Parallel() store := newTestStorer() - blobA := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + blobA := testBlobHashA addManifest(t, store, "snapshot-001", []string{blobA}) addBlob(t, store, blobA) @@ -223,9 +236,10 @@ func TestRemoveSnapshot_LocalOnly_PreservesRemote(t *testing.T) { assert.False(t, result.RemoteRemoved) assert.True(t, store.hasKey("blobs/aa/aa/"+blobA)) - assert.True(t, store.hasKey(remoteKeyPath("snapshot-001", "manifest.json.zst"))) + assert.True(t, store.hasKey(remoteKeyPath("snapshot-001"))) - assert.Contains(t, tv.Stdout.String(), "Removed snapshot 'snapshot-001' from local database") + assert.Contains(t, tv.Stdout.String(), + "Removed snapshot 'snapshot-001' from local database") } // TestRemoveSnapshot_DefaultRemovesMetadataNotBlobs is the canonical @@ -235,10 +249,11 @@ func TestRemoveSnapshot_LocalOnly_PreservesRemote(t *testing.T) { // remaining remote manifest, and the output prints that exact command. func TestRemoveSnapshot_DefaultRemovesMetadataNotBlobs(t *testing.T) { log.Initialize(log.Config{}) + t.Parallel() store := newTestStorer() - blobUnique := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + blobUnique := testBlobHashA blobShared := "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" addManifest(t, store, "snapshot-001", []string{blobUnique, blobShared}) @@ -255,8 +270,8 @@ func TestRemoveSnapshot_DefaultRemovesMetadataNotBlobs(t *testing.T) { assert.Equal(t, "snapshot-001", result.SnapshotID) assert.True(t, result.RemoteRemoved) - assert.False(t, store.hasKey(remoteKeyPath("snapshot-001", "manifest.json.zst"))) - assert.True(t, store.hasKey(remoteKeyPath("snapshot-002", "manifest.json.zst"))) + assert.False(t, store.hasKey(remoteKeyPath("snapshot-001"))) + assert.True(t, store.hasKey(remoteKeyPath("snapshot-002"))) // Blobs are intentionally NOT touched — that's what `vaultik prune` // is for. assert.True(t, store.hasKey("blobs/aa/aa/"+blobUnique)) @@ -270,10 +285,11 @@ func TestRemoveSnapshot_DefaultRemovesMetadataNotBlobs(t *testing.T) { func TestRemoveSnapshot_DryRun(t *testing.T) { log.Initialize(log.Config{}) + t.Parallel() store := newTestStorer() - blobA := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + blobA := testBlobHashA addManifest(t, store, "snapshot-001", []string{blobA}) addBlob(t, store, blobA) @@ -289,13 +305,14 @@ func TestRemoveSnapshot_DryRun(t *testing.T) { assert.Equal(t, initialCount, store.keyCount()) assert.True(t, store.hasKey("blobs/aa/aa/"+blobA)) - assert.True(t, store.hasKey(remoteKeyPath("snapshot-001", "manifest.json.zst"))) + assert.True(t, store.hasKey(remoteKeyPath("snapshot-001"))) assert.Contains(t, tv.Stdout.String(), "[Dry run - no changes made]") } func TestRemoveAllSnapshots_RequiresForce(t *testing.T) { log.Initialize(log.Config{}) + t.Parallel() store := newTestStorer() addManifest(t, store, "snapshot-001", []string{}) @@ -306,16 +323,17 @@ func TestRemoveAllSnapshots_RequiresForce(t *testing.T) { opts := &vaultik.RemoveOptions{} // No Force _, err := tv.RemoveAllSnapshots(opts) - assert.Error(t, err) + require.Error(t, err) assert.Contains(t, err.Error(), "--all requires --force") } func TestRemoveAllSnapshots_WithForce(t *testing.T) { log.Initialize(log.Config{}) + t.Parallel() store := newTestStorer() - blobA := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + blobA := testBlobHashA addManifest(t, store, "snapshot-001", []string{blobA}) addManifest(t, store, "snapshot-002", []string{blobA}) addBlob(t, store, blobA) @@ -331,8 +349,8 @@ func TestRemoveAllSnapshots_WithForce(t *testing.T) { // Blobs intentionally preserved — that's prune's job. assert.True(t, store.hasKey("blobs/aa/aa/"+blobA)) - assert.False(t, store.hasKey(remoteKeyPath("snapshot-001", "manifest.json.zst"))) - assert.False(t, store.hasKey(remoteKeyPath("snapshot-002", "manifest.json.zst"))) + assert.False(t, store.hasKey(remoteKeyPath("snapshot-001"))) + assert.False(t, store.hasKey(remoteKeyPath("snapshot-002"))) out := tv.Stdout.String() assert.Contains(t, out, "Removed 2 snapshot(s)") @@ -342,6 +360,7 @@ func TestRemoveAllSnapshots_WithForce(t *testing.T) { func TestRemoveAllSnapshots_DryRun(t *testing.T) { log.Initialize(log.Config{}) + t.Parallel() store := newTestStorer() addManifest(t, store, "snapshot-001", []string{}) @@ -368,6 +387,7 @@ func TestRemoveAllSnapshots_DryRun(t *testing.T) { func TestRemoveAllSnapshots_NoSnapshots(t *testing.T) { log.Initialize(log.Config{}) + t.Parallel() store := newTestStorer() // No snapshots added diff --git a/internal/vaultik/restore.go b/internal/vaultik/restore.go index 7eebf4f..36a632e 100644 --- a/internal/vaultik/restore.go +++ b/internal/vaultik/restore.go @@ -14,7 +14,6 @@ import ( "time" "filippo.io/age" - "github.com/dustin/go-humanize" "github.com/spf13/afero" "sneak.berlin/go/vaultik/internal/blobgen" "sneak.berlin/go/vaultik/internal/database" @@ -23,6 +22,34 @@ import ( "sneak.berlin/go/vaultik/internal/types" ) +// Sentinel errors for restore failures. +var ( + errFilesFailedRestore = errors.New("file(s) failed to restore") + errFilesFailedVerify = errors.New("files failed verification") + errDecryptionKeyRequired = errors.New( + "decryption key required for restore\n\n" + + "Set the VAULTIK_AGE_SECRET_KEY environment variable to your " + + "age private key:\n" + + " export VAULTIK_AGE_SECRET_KEY='AGE-SECRET-KEY-...'") + errBlobMissingFromIndex = errors.New("blob hash missing from blob index") + errChunkNotInAnyBlob = errors.New("chunk not found in any blob") + errBlobIDNotInHashIndex = errors.New("blob id missing from hash index") + errShortChunkRead = errors.New("short read") +) + +// restoreDirMode is the permission mode for directories created while +// restoring (parent directories and the target root; restored +// directories themselves get their stored mode). +const restoreDirMode = 0o755 + +// sweepIntervalDivisor sets the sweeper threshold to one N-th of the +// configured blob size limit. +const sweepIntervalDivisor = 100 + +// restoreStatusInterval is how often periodic progress lines are +// printed during restore and verify. +const restoreStatusInterval = 15 * time.Second + // RestoreOptions contains options for the restore operation type RestoreOptions struct { SnapshotID string @@ -100,7 +127,8 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error { v.UI.Infof("Found %s files to restore.", v.UI.Count(len(files))) // Step 3: Create target directory - if err := v.Fs.MkdirAll(opts.TargetDir, 0755); err != nil { + err = v.Fs.MkdirAll(opts.TargetDir, restoreDirMode) + if err != nil { return fmt.Errorf("creating target directory: %w", err) } @@ -120,9 +148,9 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error { log.Info("Restore complete", "files_restored", result.FilesRestored, - "bytes_restored", humanize.Bytes(uint64(result.BytesRestored)), + "bytes_restored", ubytes(result.BytesRestored), "blobs_downloaded", result.BlobsDownloaded, - "bytes_downloaded", humanize.Bytes(uint64(result.BytesDownloaded)), + "bytes_downloaded", ubytes(result.BytesDownloaded), "duration", result.Duration, ) @@ -132,8 +160,21 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error { v.UI.Duration(result.Duration), ) + return v.finishRestore(repos, files, opts, result) +} + +// finishRestore emits the post-restore warnings, runs optional +// verification, and converts any failed-file count into an error. +func (v *Vaultik) finishRestore( + repos *database.Repositories, + files []*database.File, + opts *RestoreOptions, + result *RestoreResult, +) error { if os.Geteuid() != 0 { - v.UI.Warningf("Restore did not preserve file ownership: chown(2) requires root. Re-run as root (e.g. with sudo) if you need original UID/GID preserved.") + v.UI.Warningf("Restore did not preserve file ownership: chown(2) " + + "requires root. Re-run as root (e.g. with sudo) if you need " + + "original UID/GID preserved.") } if result.FilesFailed > 0 { @@ -153,16 +194,19 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error { } if result.FilesFailed > 0 { - return fmt.Errorf("%d file(s) failed to restore", result.FilesFailed) + return fmt.Errorf("%d %w", result.FilesFailed, errFilesFailedRestore) } return nil } -// prepareRestoreIdentity validates that an age secret key is configured and parses it +// prepareRestoreIdentity validates that an age secret key is configured +// and parses it. +// +//nolint:ireturn // age.Identity is the decryption abstraction by design func (v *Vaultik) prepareRestoreIdentity() (age.Identity, error) { if v.Config.AgeSecretKey == "" { - return nil, errors.New("decryption key required for restore\n\nSet the VAULTIK_AGE_SECRET_KEY environment variable to your age private key:\n export VAULTIK_AGE_SECRET_KEY='AGE-SECRET-KEY-...'") + return nil, errDecryptionKeyRequired } identity, err := age.ParseX25519Identity(v.Config.AgeSecretKey) @@ -212,22 +256,12 @@ func (v *Vaultik) restoreAllFiles( // Per-restore sweep state: every blob_size_limit/100 bytes written, // scan the cache and delete any blob whose remaining file references // are all already restored. - sweeper := newRestoreSweeper(v.ctx, repos, blobCache, v.Config.BlobSizeLimit.Int64()/100) + sweeper := newRestoreSweeper(v.ctx, repos, blobCache, + v.Config.BlobSizeLimit.Int64()/sweepIntervalDivisor) - // Pre-fetch every blob row once so chunk extraction can map a - // blob_id to its hash without a DB round-trip per chunk. - blobsByID, err := repos.Blobs.GetAll(v.ctx) + blobByHash, blobIDToHash, err := v.buildBlobIndexes(repos) if err != nil { - return nil, fmt.Errorf("fetching blob index: %w", err) - } - - blobIDToHash := make(map[string]string, len(blobsByID)) - - blobByHash := make(map[string]*database.Blob, len(blobsByID)) - for id, blob := range blobsByID { - hash := blob.Hash.String() - blobIDToHash[id] = hash - blobByHash[hash] = blob + return nil, err } plan, err := newRestorePlan(v.ctx, repos, files, chunkToBlobMap, blobIDToHash) @@ -235,18 +269,7 @@ func (v *Vaultik) restoreAllFiles( return nil, fmt.Errorf("building restore plan: %w", err) } - // Index files by ID so the loop can look them up by the IDs the - // plan hands back. - filesByID := make(map[types.FileID]*database.File, len(files)) - for _, f := range files { - filesByID[f.ID] = f - } - - // Calculate total bytes expected for percentage / ETA arithmetic. - var totalBytesExpected int64 - for _, file := range files { - totalBytesExpected += file.Size - } + filesByID, totalBytesExpected := indexRestoreFiles(files) v.UI.Beginf("Restoring %s files (%s) to %s.", v.UI.Count(len(files)), @@ -268,52 +291,41 @@ func (v *Vaultik) restoreAllFiles( runningAsRoot: os.Geteuid() == 0, } + err = v.runRestoreLoop(session, plan, filesByID, totalBytesExpected) + if err != nil { + return nil, err + } + + return result, nil +} + +// runRestoreLoop drains the restore plan: restore files as their blobs +// become available, download the next blob set when nothing is ready, +// and emit periodic progress. +func (v *Vaultik) runRestoreLoop( + session *restoreSession, plan *restorePlan, + filesByID map[types.FileID]*database.File, totalBytesExpected int64, +) error { // Periodic progress output, matching the snapshot create cadence. startTime := time.Now() lastStatusTime := startTime - - const statusInterval = 15 * time.Second - processed := 0 + totalFiles := len(filesByID) for plan.hasPending() { if v.ctx.Err() != nil { - return nil, v.ctx.Err() + return v.ctx.Err() } fileID, ready := plan.popReady() if !ready { - // No file is fully cache-served. First free any blobs - // whose file sets are exhausted — without this, the - // blob whose last file we just finished would still be - // cached when we Put the next one, briefly pushing - // peak occupancy from 1 to 2. - sweeper.sweep() - - // Pick the pending file with the smallest uncached - // blob set and download its blobs. After each blob - // lands, the plan moves any pending file whose set - // just emptied onto the ready queue. - next := plan.pickNextDownload() - if next.IsZero() { - break + downloaded, err := session.downloadNextBlobSet(plan) + if err != nil { + return err } - for _, hash := range plan.blobsNeeded(next) { - blob, ok := blobByHash[hash] - if !ok { - return nil, fmt.Errorf("blob hash %s missing from blob index", hash[:16]) - } - - err := session.downloadBlobToCache(hash, blob.CompressedSize) - if err != nil { - return nil, fmt.Errorf("downloading blob %s: %w", hash[:16], err) - } - - result.BlobsDownloaded++ - result.BytesDownloaded += blob.CompressedSize - - plan.markBlobCached(hash) + if !downloaded { + break } continue @@ -323,54 +335,173 @@ func (v *Vaultik) restoreAllFiles( err := session.restoreFile(file) if err != nil { - log.Error("Failed to restore file", "path", file.Path, "error", err) - - if !opts.SkipErrors { - return nil, fmt.Errorf("restoring %s: %w (pass --skip-errors to continue past restore failures)", file.Path, err) + err = v.handleRestoreFileError( + plan, session.opts, session.result, file, fileID, err) + if err != nil { + return err } - v.UI.Errorf("Failed to restore %s: %v. Skipping (--skip-errors).", v.UI.Path(file.Path.String()), err) - - result.FilesFailed++ - result.FailedFiles = append(result.FailedFiles, file.Path.String()) - - plan.finishFile(fileID) - continue } // Record the file as restored so the sweeper can free blobs // once all referencing files are done, and drop it from the // plan's indexes so future picks ignore it. - sweeper.fileRestored(fileID.String()) + session.sweeper.fileRestored(fileID.String()) plan.finishFile(fileID) processed++ - if time.Since(lastStatusTime) >= statusInterval { - v.printRestoreProgress(processed, len(files), result.BytesRestored, totalBytesExpected, startTime) - - lastStatusTime = time.Now() - } - - // Structured progress log for --verbose / JSON consumers. - if processed%100 == 0 || processed == len(files) { - log.Info("Restore progress", - "files", fmt.Sprintf("%d/%d", processed, len(files)), - "bytes", humanize.Bytes(uint64(result.BytesRestored)), - ) - } + v.restoreProgressTick(processed, totalFiles, + session.result.BytesRestored, + totalBytesExpected, startTime, &lastStatusTime) } - return result, nil + return nil +} + +// downloadNextBlobSet is invoked when no file is fully cache-served. +// It first frees any blobs whose file sets are exhausted — without +// this, the blob whose last file we just finished would still be +// cached when we Put the next one, briefly pushing peak occupancy from +// 1 to 2. It then picks the pending file with the smallest uncached +// blob set and downloads its blobs; after each blob lands, the plan +// moves any pending file whose set just emptied onto the ready queue. +// Returns false when nothing is pending download (the caller stops). +func (s *restoreSession) downloadNextBlobSet(plan *restorePlan) (bool, error) { + s.sweeper.sweep() + + next := plan.pickNextDownload() + if next.IsZero() { + return false, nil + } + + for _, hash := range plan.blobsNeeded(next) { + blob, ok := s.blobByHash[hash] + if !ok { + return false, fmt.Errorf("%w: %s", errBlobMissingFromIndex, hash[:16]) + } + + err := s.downloadBlobToCache(hash, blob.CompressedSize) + if err != nil { + return false, fmt.Errorf("downloading blob %s: %w", hash[:16], err) + } + + s.result.BlobsDownloaded++ + s.result.BytesDownloaded += blob.CompressedSize + + plan.markBlobCached(hash) + } + + return true, nil +} + +// indexRestoreFiles indexes files by ID for plan lookups and sums the +// expected byte total for percentage / ETA arithmetic. +func indexRestoreFiles( + files []*database.File, +) (map[types.FileID]*database.File, int64) { + filesByID := make(map[types.FileID]*database.File, len(files)) + + var totalBytesExpected int64 + + for _, f := range files { + filesByID[f.ID] = f + totalBytesExpected += f.Size + } + + return filesByID, totalBytesExpected +} + +// buildBlobIndexes pre-fetches every blob row once so chunk extraction +// can map a blob_id to its hash without a DB round-trip per chunk. +func (v *Vaultik) buildBlobIndexes( + repos *database.Repositories, +) (map[string]*database.Blob, map[string]string, error) { + blobsByID, err := repos.Blobs.GetAll(v.ctx) + if err != nil { + return nil, nil, fmt.Errorf("fetching blob index: %w", err) + } + + blobIDToHash := make(map[string]string, len(blobsByID)) + + blobByHash := make(map[string]*database.Blob, len(blobsByID)) + for id, blob := range blobsByID { + hash := blob.Hash.String() + blobIDToHash[id] = hash + blobByHash[hash] = blob + } + + return blobByHash, blobIDToHash, nil +} + +// restoreProgressTick emits the periodic UI status line and structured +// progress log during the restore loop. +func (v *Vaultik) restoreProgressTick( + processed, totalFiles int, bytesRestored, totalBytesExpected int64, + startTime time.Time, lastStatusTime *time.Time, +) { + if time.Since(*lastStatusTime) >= restoreStatusInterval { + v.printRestoreProgress( + processed, totalFiles, bytesRestored, + totalBytesExpected, startTime) + + *lastStatusTime = time.Now() + } + + // Structured progress log for --verbose / JSON consumers. + if processed%progressLogEvery == 0 || processed == totalFiles { + log.Info("Restore progress", + "files", fmt.Sprintf("%d/%d", processed, totalFiles), + "bytes", ubytes(bytesRestored), + ) + } +} + +// handleRestoreFileError records a per-file restore failure: fatal unless +// --skip-errors is set, in which case the file is counted as failed and +// dropped from the plan. +func (v *Vaultik) handleRestoreFileError( + plan *restorePlan, opts *RestoreOptions, result *RestoreResult, + file *database.File, fileID types.FileID, err error, +) error { + log.Error("Failed to restore file", "path", file.Path, "error", err) + + if !opts.SkipErrors { + return fmt.Errorf( + "restoring %s: %w (pass --skip-errors to continue past "+ + "restore failures)", file.Path, err) + } + + v.UI.Errorf("Failed to restore %s: %v. Skipping (--skip-errors).", + v.UI.Path(file.Path.String()), err) + + result.FilesFailed++ + result.FailedFiles = append(result.FailedFiles, file.Path.String()) + + plan.finishFile(fileID) + + return nil } // printRestoreProgress emits a periodic restore-phase status line via // the UI writer, mirroring scanner.printProcessingProgress so the two // long-running commands have the same on-screen rhythm. -func (v *Vaultik) printRestoreProgress(filesDone, totalFiles int, bytesDone, totalBytes int64, startTime time.Time) { +func (v *Vaultik) printRestoreProgress( + filesDone, totalFiles int, bytesDone, totalBytes int64, startTime time.Time, +) { + v.printPhaseProgress("Restore", "restore", + filesDone, totalFiles, bytesDone, totalBytes, startTime) +} + +// printPhaseProgress emits a periodic status line for a long-running +// phase (restore or verify) so user-facing pacing is uniform. +func (v *Vaultik) printPhaseProgress( + title, phase string, + filesDone, totalFiles int, bytesDone, totalBytes int64, startTime time.Time, +) { elapsed := time.Since(startTime) - pct := float64(bytesDone) / float64(totalBytes) * 100 + pct := float64(bytesDone) / float64(totalBytes) * percentScale byteRate := float64(bytesDone) / elapsed.Seconds() fileRate := float64(filesDone) / elapsed.Seconds() @@ -382,7 +513,9 @@ func (v *Vaultik) printRestoreProgress(filesDone, totalFiles int, bytesDone, tot } if eta > 0 { - v.UI.Progressf("Restore: %s/%s files (%s), %s/%s, %s, %.0f files/sec, restore elapsed: %s, restore ETA: %s (est remain %s).", + v.UI.Progressf("%s: %s/%s files (%s), %s/%s, %s, %.0f files/sec, "+ + "%s elapsed: %s, %s ETA: %s (est remain %s).", + title, v.UI.Count(filesDone), v.UI.Count(totalFiles), v.UI.Percent(pct), @@ -390,14 +523,18 @@ func (v *Vaultik) printRestoreProgress(filesDone, totalFiles int, bytesDone, tot v.UI.Size(totalBytes), v.UI.Speed(byteRate), fileRate, + phase, v.UI.Duration(elapsed), + phase, v.UI.Time(time.Now().Add(eta)), v.UI.Duration(eta)) return } - v.UI.Progressf("Restore: %s/%s files (%s), %s/%s, %s, %.0f files/sec, restore elapsed: %s.", + v.UI.Progressf("%s: %s/%s files (%s), %s/%s, %s, %.0f files/sec, "+ + "%s elapsed: %s.", + title, v.UI.Count(filesDone), v.UI.Count(totalFiles), v.UI.Percent(pct), @@ -405,6 +542,7 @@ func (v *Vaultik) printRestoreProgress(filesDone, totalFiles int, bytesDone, tot v.UI.Size(totalBytes), v.UI.Speed(byteRate), fileRate, + phase, v.UI.Duration(elapsed)) } @@ -428,7 +566,7 @@ func (v *Vaultik) handleRestoreVerification( v.UI.Detailf("%s", v.UI.Path(path)) } - return fmt.Errorf("%d files failed verification", result.FilesFailed) + return fmt.Errorf("%d %w", result.FilesFailed, errFilesFailedVerify) } v.UI.Completef("Verified %s files (%s).", @@ -441,9 +579,12 @@ func (v *Vaultik) handleRestoreVerification( // downloadSnapshotDB downloads and decrypts the snapshot metadata // database. The snapshotID is the human ID; we hash it to the remote // key for the storage path. -func (v *Vaultik) downloadSnapshotDB(snapshotID string, identity age.Identity) (*database.DB, error) { +func (v *Vaultik) downloadSnapshotDB( + snapshotID string, identity age.Identity, +) (*database.DB, error) { // Download encrypted database from storage - dbKey := fmt.Sprintf("metadata/%s/db.zst.age", snapshot.RemoteSnapshotKey(snapshotID)) + dbKey := fmt.Sprintf("metadata/%s/db.zst.age", + snapshot.RemoteSnapshotKey(snapshotID)) reader, err := v.Storage.Get(v.ctx, dbKey) if err != nil { @@ -457,7 +598,8 @@ func (v *Vaultik) downloadSnapshotDB(snapshotID string, identity age.Identity) ( return nil, fmt.Errorf("reading encrypted data: %w", err) } - log.Debug("Downloaded encrypted database", "size", humanize.Bytes(uint64(len(encryptedData)))) + log.Debug("Downloaded encrypted database", + "size", ubytes(int64(len(encryptedData)))) // Decrypt and decompress using blobgen.Reader blobReader, err := blobgen.NewReader(bytes.NewReader(encryptedData), identity) @@ -472,7 +614,7 @@ func (v *Vaultik) downloadSnapshotDB(snapshotID string, identity age.Identity) ( return nil, fmt.Errorf("decrypting and decompressing: %w", err) } - log.Debug("Decrypted database", "size", humanize.Bytes(uint64(len(dbData)))) + log.Debug("Decrypted database", "size", ubytes(int64(len(dbData)))) // Create a temporary database file and write the binary SQLite data directly tempFile, err := afero.TempFile(v.Fs, "", "vaultik-restore-*.db") @@ -483,14 +625,16 @@ func (v *Vaultik) downloadSnapshotDB(snapshotID string, identity age.Identity) ( tempPath := tempFile.Name() // Write the binary SQLite database directly - if _, err := tempFile.Write(dbData); err != nil { + _, err = tempFile.Write(dbData) + if err != nil { _ = tempFile.Close() _ = v.Fs.Remove(tempPath) return nil, fmt.Errorf("writing database file: %w", err) } - if err := tempFile.Close(); err != nil { + err = tempFile.Close() + if err != nil { _ = v.Fs.Remove(tempPath) return nil, fmt.Errorf("closing temp file: %w", err) @@ -508,7 +652,9 @@ func (v *Vaultik) downloadSnapshotDB(snapshotID string, identity age.Identity) ( } // getFilesToRestore returns the list of files to restore based on path filters -func (v *Vaultik) getFilesToRestore(ctx context.Context, repos *database.Repositories, pathFilters []string) ([]*database.File, error) { +func (v *Vaultik) getFilesToRestore( + ctx context.Context, repos *database.Repositories, pathFilters []string, +) ([]*database.File, error) { // If no filters, get all files if len(pathFilters) == 0 { return repos.Files.ListAll(ctx) @@ -541,7 +687,9 @@ func (v *Vaultik) getFilesToRestore(ctx context.Context, repos *database.Reposit } // buildChunkToBlobMap creates a mapping from chunk hash to blob information -func (v *Vaultik) buildChunkToBlobMap(ctx context.Context, repos *database.Repositories) (map[string]*database.BlobChunk, error) { +func (v *Vaultik) buildChunkToBlobMap( + ctx context.Context, repos *database.Repositories, +) (map[string]*database.BlobChunk, error) { // Query all blob_chunks query := `SELECT blob_id, chunk_hash, offset, length FROM blob_chunks` @@ -559,7 +707,9 @@ func (v *Vaultik) buildChunkToBlobMap(ctx context.Context, repos *database.Repos bc database.BlobChunk blobIDStr, chunkHashStr string ) - if err := rows.Scan(&blobIDStr, &chunkHashStr, &bc.Offset, &bc.Length); err != nil { + + err = rows.Scan(&blobIDStr, &chunkHashStr, &bc.Offset, &bc.Length) + if err != nil { return nil, fmt.Errorf("scanning blob_chunk: %w", err) } @@ -584,7 +734,7 @@ func (v *Vaultik) buildChunkToBlobMap(ctx context.Context, repos *database.Repos // readable: restoreFile(file) instead of a ten-argument helper. type restoreSession struct { v *Vaultik - ctx context.Context + ctx context.Context //nolint:containedctx // per-restore state by design repos *database.Repositories opts *RestoreOptions identity age.Identity @@ -609,7 +759,7 @@ func (s *restoreSession) restoreFile(file *database.File) error { parentDir := filepath.Dir(targetPath) - err := s.v.Fs.MkdirAll(parentDir, 0755) + err := s.v.Fs.MkdirAll(parentDir, restoreDirMode) if err != nil { return fmt.Errorf("creating parent directory: %w", err) } @@ -636,7 +786,8 @@ func (s *restoreSession) restoreSymlink(file *database.File, targetPath string) return fmt.Errorf("creating symlink: %w", err) } } else { - log.Debug("Symlink creation not supported on this filesystem", "path", file.Path, "target", file.LinkTarget) + log.Debug("Symlink creation not supported on this filesystem", + "path", file.Path, "target", file.LinkTarget) } s.result.FilesRestored++ @@ -648,34 +799,51 @@ func (s *restoreSession) restoreSymlink(file *database.File, targetPath string) // restoreDirectory restores a directory with its permissions, mtime, // and (on real filesystems, with sufficient privileges) ownership. -func (s *restoreSession) restoreDirectory(file *database.File, targetPath string) error { +func (s *restoreSession) restoreDirectory( + file *database.File, targetPath string, +) error { err := s.v.Fs.MkdirAll(targetPath, os.FileMode(file.Mode)) if err != nil { return fmt.Errorf("creating directory: %w", err) } - err = s.v.Fs.Chmod(targetPath, os.FileMode(file.Mode)) + s.applyFileMetadata(file, targetPath) + + s.result.FilesRestored++ + + return nil +} + +// applyFileMetadata applies stored permissions, ownership (when running +// as root on a real filesystem), and mtime to a restored path. Failures +// are logged at debug level and do not abort the restore. +func (s *restoreSession) applyFileMetadata(file *database.File, targetPath string) { + err := s.v.Fs.Chmod(targetPath, os.FileMode(file.Mode)) if err != nil { - log.Debug("Failed to set directory permissions", "path", targetPath, "error", err) + log.Debug("Failed to set permissions", "path", targetPath, "error", err) } if s.runningAsRoot { if _, ok := s.v.Fs.(*afero.OsFs); ok { - err := os.Chown(targetPath, int(file.UID), int(file.GID)) + err = os.Chown(targetPath, int(file.UID), int(file.GID)) if err != nil { - log.Debug("Failed to set directory ownership", "path", targetPath, "error", err) + log.Debug("Failed to set ownership", "path", targetPath, "error", err) } } } err = s.v.Fs.Chtimes(targetPath, file.MTime, file.MTime) if err != nil { - log.Debug("Failed to set directory mtime", "path", targetPath, "error", err) + log.Debug("Failed to set mtime", "path", targetPath, "error", err) } +} - s.result.FilesRestored++ - - return nil +// chunkWriteTimings accumulates per-phase durations while writing a +// file's chunks out of the blob cache. Debug instrumentation only. +type chunkWriteTimings struct { + readAt time.Duration + write time.Duration + sweeper time.Duration } // restoreRegularFile reconstructs a regular file by reading chunks @@ -683,7 +851,9 @@ func (s *restoreSession) restoreDirectory(file *database.File, targetPath string // method runs is that every blob this file needs is already in the // disk cache — the planner guarantees that by only marking files // "ready" once their full blob set is on disk. -func (s *restoreSession) restoreRegularFile(file *database.File, targetPath string) error { +func (s *restoreSession) restoreRegularFile( + file *database.File, targetPath string, +) error { fileStart := time.Now() t0 := time.Now() @@ -705,49 +875,9 @@ func (s *restoreSession) restoreRegularFile(file *database.File, targetPath stri defer func() { _ = outFile.Close() }() - var ( - readAtDur time.Duration - writeDur time.Duration - sweeperDur time.Duration - bytesWritten int64 - ) - - for _, fc := range fileChunks { - chunkHashStr := fc.ChunkHash.String() - - blobChunk, ok := s.chunkToBlobMap[chunkHashStr] - if !ok { - return fmt.Errorf("chunk %s not found in any blob", chunkHashStr[:16]) - } - - blobHash, ok := s.blobIDToHash[blobChunk.BlobID.String()] - if !ok { - return fmt.Errorf("blob id %s missing from hash index", blobChunk.BlobID) - } - - t0 = time.Now() - chunkData, err := s.blobCache.ReadAt(blobHash, blobChunk.Offset, blobChunk.Length) - readAtDur += time.Since(t0) - - if err != nil { - return fmt.Errorf("reading chunk %s from cached blob %s: %w", fc.ChunkHash[:16], blobHash[:16], err) - } - - t0 = time.Now() - n, err := outFile.Write(chunkData) - writeDur += time.Since(t0) - - if err != nil { - return fmt.Errorf("writing chunk: %w", err) - } - - bytesWritten += int64(n) - - t0 = time.Now() - - s.sweeper.chunkRestored(int64(n)) - - sweeperDur += time.Since(t0) + bytesWritten, timings, err := s.writeFileChunks(outFile, fileChunks) + if err != nil { + return err } log.Debug("Restored regular file (timings)", @@ -757,47 +887,92 @@ func (s *restoreSession) restoreRegularFile(file *database.File, targetPath stri "ms_total", time.Since(fileStart).Milliseconds(), "ms_file_chunks_query", fileChunksQueryDur.Milliseconds(), "ms_create", createDur.Milliseconds(), - "ms_readat", readAtDur.Milliseconds(), - "ms_writes", writeDur.Milliseconds(), - "ms_sweeper", sweeperDur.Milliseconds(), + "ms_readat", timings.readAt.Milliseconds(), + "ms_writes", timings.write.Milliseconds(), + "ms_sweeper", timings.sweeper.Milliseconds(), ) - if err := outFile.Close(); err != nil { + err = outFile.Close() + if err != nil { return fmt.Errorf("closing output file: %w", err) } - if err := s.v.Fs.Chmod(targetPath, os.FileMode(file.Mode)); err != nil { - log.Debug("Failed to set file permissions", "path", targetPath, "error", err) - } - - if s.runningAsRoot { - if _, ok := s.v.Fs.(*afero.OsFs); ok { - err := os.Chown(targetPath, int(file.UID), int(file.GID)) - if err != nil { - log.Debug("Failed to set file ownership", "path", targetPath, "error", err) - } - } - } - - if err := s.v.Fs.Chtimes(targetPath, file.MTime, file.MTime); err != nil { - log.Debug("Failed to set file mtime", "path", targetPath, "error", err) - } + s.applyFileMetadata(file, targetPath) s.result.FilesRestored++ s.result.BytesRestored += bytesWritten - log.Debug("Restored file", "path", file.Path, "size", humanize.Bytes(uint64(bytesWritten))) + log.Debug("Restored file", "path", file.Path, "size", ubytes(bytesWritten)) return nil } +// writeFileChunks streams each of the file's chunks from the blob disk +// cache into outFile, crediting restored bytes to the sweeper as it +// goes. Returns the bytes written plus per-phase timing accumulators. +func (s *restoreSession) writeFileChunks( + outFile afero.File, fileChunks []*database.FileChunk, +) (int64, chunkWriteTimings, error) { + var ( + timings chunkWriteTimings + bytesWritten int64 + ) + + for _, fc := range fileChunks { + chunkHashStr := fc.ChunkHash.String() + + blobChunk, ok := s.chunkToBlobMap[chunkHashStr] + if !ok { + return bytesWritten, timings, fmt.Errorf( + "%w: %s", errChunkNotInAnyBlob, chunkHashStr[:16]) + } + + blobHash, ok := s.blobIDToHash[blobChunk.BlobID.String()] + if !ok { + return bytesWritten, timings, fmt.Errorf( + "%w: %s", errBlobIDNotInHashIndex, blobChunk.BlobID) + } + + t0 := time.Now() + chunkData, err := s.blobCache.ReadAt( + blobHash, blobChunk.Offset, blobChunk.Length) + timings.readAt += time.Since(t0) + + if err != nil { + return bytesWritten, timings, fmt.Errorf( + "reading chunk %s from cached blob %s: %w", + fc.ChunkHash[:16], blobHash[:16], err) + } + + t0 = time.Now() + n, err := outFile.Write(chunkData) + timings.write += time.Since(t0) + + if err != nil { + return bytesWritten, timings, fmt.Errorf("writing chunk: %w", err) + } + + bytesWritten += int64(n) + + t0 = time.Now() + + s.sweeper.chunkRestored(int64(n)) + + timings.sweeper += time.Since(t0) + } + + return bytesWritten, timings, nil +} + // downloadBlobToCache streams a blob from remote storage straight into // the disk cache, decrypting and decompressing on the fly. The // plaintext never lives fully in memory — io.Copy through // blobDiskCache.PutFromReader uses a 32 KiB buffer regardless of blob // size, which is what makes multi-GB blobs tractable on machines with // less RAM than the blob. -func (s *restoreSession) downloadBlobToCache(blobHash string, expectedSize int64) error { +func (s *restoreSession) downloadBlobToCache( + blobHash string, expectedSize int64, +) error { start := time.Now() t0 := time.Now() @@ -833,7 +1008,8 @@ func (s *restoreSession) downloadBlobToCache(blobHash string, expectedSize int64 return nil } -// verifyRestoredFiles verifies that all restored files match their expected chunk hashes +// verifyRestoredFiles verifies that all restored files match their +// expected chunk hashes. func (v *Vaultik) verifyRestoredFiles( ctx context.Context, repos *database.Repositories, @@ -863,7 +1039,7 @@ func (v *Vaultik) verifyRestoredFiles( log.Info("Verifying restored files", "files", len(regularFiles), - "bytes", humanize.Bytes(uint64(totalBytes)), + "bytes", ubytes(totalBytes), ) v.UI.Beginf("Verifying %s files (%s).", v.UI.Count(len(regularFiles)), @@ -872,8 +1048,6 @@ func (v *Vaultik) verifyRestoredFiles( startTime := time.Now() lastStatusTime := startTime - const statusInterval = 15 * time.Second - var bytesProcessed int64 for i, file := range regularFiles { @@ -896,8 +1070,9 @@ func (v *Vaultik) verifyRestoredFiles( bytesProcessed += file.Size - if time.Since(lastStatusTime) >= statusInterval { - v.printVerifyProgress(i+1, len(regularFiles), bytesProcessed, totalBytes, startTime) + if time.Since(lastStatusTime) >= restoreStatusInterval { + v.printVerifyProgress( + i+1, len(regularFiles), bytesProcessed, totalBytes, startTime) lastStatusTime = time.Now() } @@ -905,7 +1080,7 @@ func (v *Vaultik) verifyRestoredFiles( log.Info("Verification complete", "files_verified", result.FilesVerified, - "bytes_verified", humanize.Bytes(uint64(result.BytesVerified)), + "bytes_verified", ubytes(result.BytesVerified), "files_failed", result.FilesFailed, ) @@ -915,44 +1090,11 @@ func (v *Vaultik) verifyRestoredFiles( // printVerifyProgress emits a periodic verify-phase status line. Same // shape as the restore progress line so user-facing pacing is uniform // across the two phases. -func (v *Vaultik) printVerifyProgress(filesDone, totalFiles int, bytesDone, totalBytes int64, startTime time.Time) { - elapsed := time.Since(startTime) - pct := float64(bytesDone) / float64(totalBytes) * 100 - byteRate := float64(bytesDone) / elapsed.Seconds() - fileRate := float64(filesDone) / elapsed.Seconds() - - remainingBytes := totalBytes - bytesDone - - var eta time.Duration - if byteRate > 0 && remainingBytes > 0 { - eta = time.Duration(float64(remainingBytes)/byteRate) * time.Second - } - - if eta > 0 { - v.UI.Progressf("Verify: %s/%s files (%s), %s/%s, %s, %.0f files/sec, verify elapsed: %s, verify ETA: %s (est remain %s).", - v.UI.Count(filesDone), - v.UI.Count(totalFiles), - v.UI.Percent(pct), - v.UI.Size(bytesDone), - v.UI.Size(totalBytes), - v.UI.Speed(byteRate), - fileRate, - v.UI.Duration(elapsed), - v.UI.Time(time.Now().Add(eta)), - v.UI.Duration(eta)) - - return - } - - v.UI.Progressf("Verify: %s/%s files (%s), %s/%s, %s, %.0f files/sec, verify elapsed: %s.", - v.UI.Count(filesDone), - v.UI.Count(totalFiles), - v.UI.Percent(pct), - v.UI.Size(bytesDone), - v.UI.Size(totalBytes), - v.UI.Speed(byteRate), - fileRate, - v.UI.Duration(elapsed)) +func (v *Vaultik) printVerifyProgress( + filesDone, totalFiles int, bytesDone, totalBytes int64, startTime time.Time, +) { + v.printPhaseProgress("Verify", "verify", + filesDone, totalFiles, bytesDone, totalBytes, startTime) } // verifyFile verifies a single restored file by checking its chunk hashes @@ -982,7 +1124,8 @@ func (v *Vaultik) verifyFile( // Get chunk size from database chunk, err := repos.Chunks.GetByHash(ctx, fc.ChunkHash.String()) if err != nil { - return bytesVerified, fmt.Errorf("getting chunk %s: %w", fc.ChunkHash.String()[:16], err) + return bytesVerified, fmt.Errorf("getting chunk %s: %w", + fc.ChunkHash.String()[:16], err) } // Read chunk data from file @@ -994,7 +1137,8 @@ func (v *Vaultik) verifyFile( } if int64(n) != chunk.Size { - return bytesVerified, fmt.Errorf("short read: expected %d bytes, got %d", chunk.Size, n) + return bytesVerified, fmt.Errorf("%w: expected %d bytes, got %d", + errShortChunkRead, chunk.Size, n) } // Calculate hash and compare @@ -1003,14 +1147,15 @@ func (v *Vaultik) verifyFile( expectedHash := fc.ChunkHash.String() if actualHash != expectedHash { - return bytesVerified, fmt.Errorf("chunk %d hash mismatch: expected %s, got %s", - fc.Idx, expectedHash[:16], actualHash[:16]) + return bytesVerified, fmt.Errorf("%w: chunk %d: expected %s, got %s", + errChunkHashMismatch, fc.Idx, expectedHash[:16], actualHash[:16]) } bytesVerified += int64(n) } - log.Debug("File verified", "path", file.Path, "bytes", bytesVerified, "chunks", len(fileChunks)) + log.Debug("File verified", + "path", file.Path, "bytes", bytesVerified, "chunks", len(fileChunks)) return bytesVerified, nil } diff --git a/internal/vaultik/restore_locality_test.go b/internal/vaultik/restore_locality_test.go index 0e7c52d..67b1310 100644 --- a/internal/vaultik/restore_locality_test.go +++ b/internal/vaultik/restore_locality_test.go @@ -1,4 +1,4 @@ -package vaultik +package vaultik //nolint:testpackage // inspects unexported cache internals import ( "bytes" @@ -50,33 +50,28 @@ import ( // path-ordered names that interleave the blobs (a1, b1, c1, a2, b2, // c2, …) so naive path-order processing would touch every blob before // finishing any of them. -func TestRestoreLocalityAndReadAt(t *testing.T) { - log.Initialize(log.Config{}) +// localitySource is one 1 MiB fixture file used by the locality test. +type localitySource struct { + path string + data []byte +} - fs := afero.NewOsFs() - tempDir, err := os.MkdirTemp("", "vaultik-locality-") - require.NoError(t, err) +// localityCopy is a byte-for-byte clone of one fixture source with an +// interleaved name. +type localityCopy struct { + path string + data []byte +} - defer func() { _ = os.RemoveAll(tempDir) }() +// buildLocalityFixture writes the adversarial source layout described +// in TestRestoreLocalityAndReadAt: 15 sources of 1 MiB each (which the +// backup packs into 3 blobs of 5 chunks) plus 9 interleaved-name copies +// (3 per blob group). +func buildLocalityFixture( + t *testing.T, fs afero.Fs, dataDir string, +) ([]*localitySource, []localityCopy) { + t.Helper() - dataDir := filepath.Join(tempDir, "source") - storeDir := filepath.Join(tempDir, "remote") - restoreDir := filepath.Join(tempDir, "restored") - dbPath := filepath.Join(tempDir, "index.sqlite") - - require.NoError(t, fs.MkdirAll(dataDir, 0o755)) - - // Layout: 15 source files of exactly 1 MiB each. With - // chunkSize (avg) = 4 MiB the chunker's minSize is 1 MiB, so any - // file of 1 MiB becomes a single chunk. With a 5 MiB blob limit - // the packer fits exactly 5 chunks per blob, producing 3 blobs - // containing src-001..005, src-006..010, src-011..015. - // - // Then add 9 "copy" files — byte-for-byte clones of three of the - // sources (one from each blob group) — with interleaved names - // (cp-001-A, cp-002-B, cp-003-C, cp-004-A, …) so a naive - // path-ordered restore would touch all three blobs before - // finishing any of them. const ( srcBytes = 1024 * 1024 srcCount = 15 @@ -84,35 +79,24 @@ func TestRestoreLocalityAndReadAt(t *testing.T) { perBlob = srcCount / blobsCount ) - type source struct { - path string - data []byte - } - - sources := make([]*source, srcCount) + sources := make([]*localitySource, srcCount) for i := range srcCount { - s := &source{ + s := &localitySource{ path: fmt.Sprintf("src-%03d.bin", i+1), data: randomBytes(t, srcBytes), } sources[i] = s - require.NoError(t, afero.WriteFile(fs, filepath.Join(dataDir, s.path), s.data, 0o644)) + require.NoError(t, afero.WriteFile( + fs, filepath.Join(dataDir, s.path), s.data, 0o644)) } // Pick one representative source per blob group (src-001 → blob // 1, src-006 → blob 2, src-011 → blob 3) and create 3 copies of // each with interleaved alphabetical names. - type copyFile struct { - path string - data []byte - sourceBlob int // 0, 1, or 2 - sourceIndex int // index into sources slice - } - groupReps := []int{0, perBlob, 2 * perBlob} // 0, 5, 10 letters := []byte{'A', 'B', 'C'} - var copies []copyFile + copies := make([]localityCopy, 0, blobsCount*3) for i := range 3 { for j := range blobsCount { @@ -121,10 +105,24 @@ func TestRestoreLocalityAndReadAt(t *testing.T) { path := filepath.Join(dataDir, name) src := sources[groupReps[j]] require.NoError(t, afero.WriteFile(fs, path, src.data, 0o644)) - copies = append(copies, copyFile{path: path, data: src.data, sourceBlob: j, sourceIndex: groupReps[j]}) + + copies = append(copies, localityCopy{path: path, data: src.data}) } } + return sources, copies +} + +// setupLocalityBackup runs the backup half of the locality test: a +// snapshot of dataDir into a file storer with sizes tuned so 15 one-chunk +// files pack into 3 blobs. The index database is closed before returning +// so the restore half runs from remote bytes only. +func setupLocalityBackup( + ctx context.Context, t *testing.T, fs afero.Fs, + dataDir, storeDir, dbPath string, +) (*config.Config, *storage.FileStorer, string) { + t.Helper() + // chunkSize avg = 4 MiB makes minSize = 1 MiB, so a 1 MiB file // becomes one chunk. maxBlobSize = 5 MiB packs exactly 5 chunks // per blob, yielding 3 blobs from 15 source files. @@ -134,8 +132,10 @@ func TestRestoreLocalityAndReadAt(t *testing.T) { storer, err := storage.NewFileStorer(storeDir) require.NoError(t, err) - agePublicKey := "age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05gl0sjq9q9wjg" - ageSecretKey := "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5" + agePublicKey := "age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05g" + + "l0sjq9q9wjg" + ageSecretKey := "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKU" + + "T68TXSFPK7APHXA2QS2NJA5" cfg := &config.Config{ AgeRecipients: []string{agePublicKey}, @@ -145,8 +145,6 @@ func TestRestoreLocalityAndReadAt(t *testing.T) { BlobSizeLimit: config.Size(maxBlobSize), } - ctx := context.Background() - db, err := database.New(ctx, dbPath) require.NoError(t, err) @@ -171,7 +169,8 @@ func TestRestoreLocalityAndReadAt(t *testing.T) { Repositories: repos, }) - snapshotID, err := sm.CreateSnapshotWithName(ctx, cfg.Hostname, "locality", "test-version", "test-git") + snapshotID, err := sm.CreateSnapshotWithName( + ctx, cfg.Hostname, "locality", "test-version", "test-git") require.NoError(t, err) _, err = scanner.Scan(ctx, dataDir, snapshotID) @@ -182,10 +181,35 @@ func TestRestoreLocalityAndReadAt(t *testing.T) { blobsOnDisk := listBlobKeys(t, storeDir) t.Logf("backup produced %d blobs", len(blobsOnDisk)) - require.GreaterOrEqual(t, len(blobsOnDisk), 3, "expected at least 3 blobs from 3 filler groups") + require.GreaterOrEqual(t, len(blobsOnDisk), 3, + "expected at least 3 blobs from 3 filler groups") require.NoError(t, db.Close()) + return cfg, storer, snapshotID +} + +func TestRestoreLocalityAndReadAt(t *testing.T) { + log.Initialize(log.Config{}) + t.Parallel() + + fs := afero.NewOsFs() + tempDir := t.TempDir() + + dataDir := filepath.Join(tempDir, "source") + storeDir := filepath.Join(tempDir, "remote") + restoreDir := filepath.Join(tempDir, "restored") + dbPath := filepath.Join(tempDir, "index.sqlite") + + require.NoError(t, fs.MkdirAll(dataDir, 0o755)) + + sources, copies := buildLocalityFixture(t, fs, dataDir) + + ctx := context.Background() + + cfg, storer, snapshotID := setupLocalityBackup( + ctx, t, fs, dataDir, storeDir, dbPath) + // Wrap the storer so we can count downloads per blob key. counter := newCountingStorer(storer) @@ -215,20 +239,7 @@ func TestRestoreLocalityAndReadAt(t *testing.T) { require.NotNil(t, cacheRef, "restoreCacheObserver must fire during restore") - // Verify restored content matches. - for _, s := range sources { - restored := filepath.Join(restoreDir, dataDir, s.path) - got, err := afero.ReadFile(fs, restored) - require.NoErrorf(t, err, "source missing after restore: %s", s.path) - require.Truef(t, bytes.Equal(got, s.data), "byte mismatch for source %s", s.path) - } - - for _, c := range copies { - restored := filepath.Join(restoreDir, c.path) - got, err := afero.ReadFile(fs, restored) - require.NoErrorf(t, err, "copy missing after restore: %s", c.path) - require.Truef(t, bytes.Equal(got, c.data), "byte mismatch for copy %s", c.path) - } + verifyLocalityRestore(t, fs, restoreDir, dataDir, sources, copies) // (1) Each blob fetched exactly once. for key, n := range counter.snapshot() { @@ -242,18 +253,48 @@ func TestRestoreLocalityAndReadAt(t *testing.T) { // (2) Peak cache size ≤ 1. The sweeper plus locality-aware // ordering should free each blob before the next one downloads. assert.LessOrEqualf(t, cacheRef.PeakLen(), 1, - "peak cached blobs was %d; expected ≤ 1 with locality-ordered restore", cacheRef.PeakLen()) + "peak cached blobs was %d; expected ≤ 1 with locality-ordered restore", + cacheRef.PeakLen()) // (3) Cache.Get must never be called during restore — chunk // extraction has to go through ReadAt so we never read the whole // blob from disk to grab a few KB slice. assert.Equalf(t, 0, cacheRef.GetCalls(), - "blobDiskCache.Get was called %d times during restore; restore must use ReadAt exclusively", cacheRef.GetCalls()) + "blobDiskCache.Get was called %d times during restore; "+ + "restore must use ReadAt exclusively", cacheRef.GetCalls()) t.Logf("blob cache stats: peak_len=%d get_calls=%d readat_calls=%d", cacheRef.PeakLen(), cacheRef.GetCalls(), cacheRef.ReadAtCalls()) } +// verifyLocalityRestore byte-compares every restored source and copy +// against its original content. +func verifyLocalityRestore( + t *testing.T, + fs afero.Fs, + restoreDir, dataDir string, + sources []*localitySource, + copies []localityCopy, +) { + t.Helper() + + for _, s := range sources { + restored := filepath.Join(restoreDir, dataDir, s.path) + got, err := afero.ReadFile(fs, restored) + require.NoErrorf(t, err, "source missing after restore: %s", s.path) + require.Truef(t, bytes.Equal(got, s.data), + "byte mismatch for source %s", s.path) + } + + for _, c := range copies { + restored := filepath.Join(restoreDir, c.path) + got, err := afero.ReadFile(fs, restored) + require.NoErrorf(t, err, "copy missing after restore: %s", c.path) + require.Truef(t, bytes.Equal(got, c.data), + "byte mismatch for copy %s", c.path) + } +} + // randomBytes returns n bytes of random data. Used to make sure the // chunker picks non-degenerate FastCDC boundaries. func randomBytes(t *testing.T, n int) []byte { @@ -316,7 +357,9 @@ func newCountingStorer(inner storage.Storer) *countingStorerInternal { return &countingStorerInternal{Storer: inner, counts: make(map[string]int)} } -func (c *countingStorerInternal) Get(ctx context.Context, key string) (io.ReadCloser, error) { +func (c *countingStorerInternal) Get( + ctx context.Context, key string, +) (io.ReadCloser, error) { c.mu.Lock() c.counts[key]++ c.mu.Unlock() diff --git a/internal/vaultik/restore_plan.go b/internal/vaultik/restore_plan.go index c6cf5cb..e40afde 100644 --- a/internal/vaultik/restore_plan.go +++ b/internal/vaultik/restore_plan.go @@ -2,6 +2,7 @@ package vaultik import ( "context" + "errors" "fmt" "math" "os" @@ -10,6 +11,12 @@ import ( "sneak.berlin/go/vaultik/internal/types" ) +// Sentinel errors for restore planning index lookups. +var ( + errPlanChunkMissing = errors.New("chunk missing from blob map") + errPlanBlobIDMissing = errors.New("blob id missing from id-to-hash map") +) + // restorePlan orders restore-time file processing by blob locality. The // goal is to keep the blob disk cache occupancy as small as possible: // download one blob, drain every file referencing only that blob, let @@ -67,14 +74,14 @@ func newRestorePlan( for _, fc := range fileChunks { bc, ok := chunkToBlobMap[fc.ChunkHash.String()] if !ok { - return nil, fmt.Errorf("planning %s: chunk %s missing from blob map", - f.Path, fc.ChunkHash.String()[:16]) + return nil, fmt.Errorf("planning %s: %w: %s", + f.Path, errPlanChunkMissing, fc.ChunkHash.String()[:16]) } hash, ok := blobIDToHash[bc.BlobID.String()] if !ok { - return nil, fmt.Errorf("planning %s: blob id %s missing from id-to-hash map", - f.Path, bc.BlobID) + return nil, fmt.Errorf("planning %s: %w: %s", + f.Path, errPlanBlobIDMissing, bc.BlobID) } blobs[hash] = struct{}{} diff --git a/internal/vaultik/restore_sweeper.go b/internal/vaultik/restore_sweeper.go index f3a47f0..e363d60 100644 --- a/internal/vaultik/restore_sweeper.go +++ b/internal/vaultik/restore_sweeper.go @@ -28,7 +28,7 @@ import ( // which is local, indexed, and not under contention — the queries are // cheap and run at most once per blob per sweep interval. type restoreSweeper struct { - ctx context.Context + ctx context.Context //nolint:containedctx // ctx bound at construction by design repos *database.Repositories cache *blobDiskCache threshold int64 @@ -38,7 +38,12 @@ type restoreSweeper struct { // newRestoreSweeper returns a sweeper that triggers eviction every // `threshold` bytes restored. Callers should pass blob_size_limit/100. -func newRestoreSweeper(ctx context.Context, repos *database.Repositories, cache *blobDiskCache, threshold int64) *restoreSweeper { +func newRestoreSweeper( + ctx context.Context, + repos *database.Repositories, + cache *blobDiskCache, + threshold int64, +) *restoreSweeper { if threshold <= 0 { threshold = 1 } @@ -78,7 +83,8 @@ func (s *restoreSweeper) sweep() { for _, blobHash := range s.cache.Keys() { needed, err := s.blobStillNeeded(blobHash) if err != nil { - log.Debug("sweeper referencing-files query failed", "blob_hash", blobHash[:16], "error", err) + log.Debug("sweeper referencing-files query failed", + "blob_hash", blobHash[:16], "error", err) continue } @@ -119,7 +125,8 @@ func (s *restoreSweeper) blobStillNeeded(blobHash string) (bool, error) { } } - if err := rows.Err(); err != nil { + err = rows.Err() + if err != nil { return true, err } diff --git a/internal/vaultik/restore_sweeper_integration_test.go b/internal/vaultik/restore_sweeper_integration_test.go index d5b21e0..13cc5cc 100644 --- a/internal/vaultik/restore_sweeper_integration_test.go +++ b/internal/vaultik/restore_sweeper_integration_test.go @@ -40,33 +40,20 @@ import ( // originals; the sweeper must keep each blob alive until BOTH the // original AND every duplicate referencing its chunks have been // restored. -func TestRestoreSweeperEvictsBlobs(t *testing.T) { - log.Initialize(log.Config{}) +// buildSweeperFixture writes the source layout for the sweeper test: 30 +// unique 1 MB random files plus 10 duplicates of a random subset. The +// PRNG seed is fixed so failures are reproducible; the entropy is what +// matters here — the FastCDC chunker needs realistic-looking data to +// pick chunk boundaries naturally. Returns the expected content by +// path. +func buildSweeperFixture( + t *testing.T, fs afero.Fs, dataDir string, uniqueFiles, duplicateFiles int, +) map[string][]byte { + t.Helper() - fs := afero.NewOsFs() - tempDir, err := os.MkdirTemp("", "vaultik-sweeper-") - require.NoError(t, err) + const fileSize = 1 * 1024 * 1024 - defer func() { _ = os.RemoveAll(tempDir) }() - - dataDir := filepath.Join(tempDir, "source") - storeDir := filepath.Join(tempDir, "remote") - restoreDir := filepath.Join(tempDir, "restored") - dbPath := filepath.Join(tempDir, "index.sqlite") - - require.NoError(t, fs.MkdirAll(dataDir, 0o755)) - - // Generate 30 unique 1 MB random files. The PRNG seed is fixed so - // failures are reproducible; the entropy is what matters here — the - // FastCDC chunker needs realistic-looking data to pick chunk - // boundaries naturally. - const ( - uniqueFiles = 30 - duplicateFiles = 10 - fileSize = 1 * 1024 * 1024 - ) - - rng := rand.New(rand.NewSource(42)) + rng := rand.New(rand.NewSource(42)) //nolint:gosec // G404: fixture data only type sourceFile struct { path string @@ -87,33 +74,58 @@ func TestRestoreSweeperEvictsBlobs(t *testing.T) { expected[path] = data } - // Pick 10 of the originals and copy each to a fresh path so the - // chunker dedups them against the originals' blobs. + // Copy a subset of the originals to fresh paths so the chunker + // dedups them against the originals' blobs. for i, idx := range rng.Perm(uniqueFiles)[:duplicateFiles] { src := uniques[idx] dstPath := filepath.Join(dataDir, fmt.Sprintf("dup-%02d.bin", i)) require.NoError(t, afero.WriteFile(fs, dstPath, src.data, 0o644)) + expected[dstPath] = src.data } - chunkSize := int64(64 * 1024) - maxBlobSize := int64(10 * 1024 * 1024) + return expected +} - storer, err := storage.NewFileStorer(storeDir) - require.NoError(t, err) +// verifySingleDownloadPerBlob asserts each blob on disk was fetched +// exactly once. >1 means the sweeper evicted a still-needed blob; 0 +// means the cache silently stopped being consulted. +func verifySingleDownloadPerBlob( + t *testing.T, counter *countingStorer, storeDir string, +) { + t.Helper() - agePublicKey := "age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05gl0sjq9q9wjg" - ageSecretKey := "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5" + blobDownloads := 0 - cfg := &config.Config{ - AgeRecipients: []string{agePublicKey}, - AgeSecretKey: ageSecretKey, - CompressionLevel: 3, - Hostname: "test-host", - BlobSizeLimit: config.Size(maxBlobSize), + for key, count := range counter.snapshot() { + if !strings.HasPrefix(key, "blobs/") { + continue + } + + assert.Equalf(t, 1, count, + "blob %s should have been downloaded exactly once during "+ + "restore, got %d", key, count) + + blobDownloads++ } - ctx := context.Background() + blobCount := countBlobsOnDisk(t, storeDir) + assert.Equal(t, blobCount, blobDownloads, + "every blob on disk should have been fetched exactly once during restore") + t.Logf("restore downloaded %d blobs, each exactly once", blobDownloads) +} + +// runSweeperBackup performs the backup half of the sweeper test and +// closes the index database so the restore half runs from remote bytes +// only. Returns the snapshot ID. +func runSweeperBackup( + ctx context.Context, t *testing.T, fs afero.Fs, + storer *storage.FileStorer, cfg *config.Config, + dataDir, storeDir, dbPath string, + chunkSize, maxBlobSize int64, + uniqueFiles, duplicateFiles int, +) string { + t.Helper() db, err := database.New(ctx, dbPath) require.NoError(t, err) @@ -139,13 +151,15 @@ func TestRestoreSweeperEvictsBlobs(t *testing.T) { Repositories: repos, }) - snapshotID, err := sm.CreateSnapshotWithName(ctx, cfg.Hostname, "sweeper", "test-version", "test-git") + snapshotID, err := sm.CreateSnapshotWithName( + ctx, cfg.Hostname, "sweeper", "test-version", "test-git") require.NoError(t, err) scanResult, err := scanner.Scan(ctx, dataDir, snapshotID) require.NoError(t, err) require.Equal(t, uniqueFiles+duplicateFiles, scanResult.FilesScanned) - require.Greater(t, scanResult.BlobsCreated, 1, "30 MB of unique data at 10 MB blob size should yield multiple blobs") + require.Greater(t, scanResult.BlobsCreated, 1, + "30 MB of unique data at 10 MB blob size should yield multiple blobs") require.NoError(t, sm.CompleteSnapshot(ctx, snapshotID)) require.NoError(t, sm.ExportSnapshotMetadata(ctx, dbPath, snapshotID)) @@ -161,6 +175,50 @@ func TestRestoreSweeperEvictsBlobs(t *testing.T) { // as a real restore on a fresh machine would. require.NoError(t, db.Close()) + return snapshotID +} + +func TestRestoreSweeperEvictsBlobs(t *testing.T) { + log.Initialize(log.Config{}) + t.Parallel() + + fs := afero.NewOsFs() + tempDir := t.TempDir() + + dataDir := filepath.Join(tempDir, "source") + storeDir := filepath.Join(tempDir, "remote") + restoreDir := filepath.Join(tempDir, "restored") + dbPath := filepath.Join(tempDir, "index.sqlite") + + require.NoError(t, fs.MkdirAll(dataDir, 0o755)) + + const ( + uniqueFiles = 30 + duplicateFiles = 10 + ) + + expected := buildSweeperFixture(t, fs, dataDir, uniqueFiles, duplicateFiles) + + chunkSize := int64(64 * 1024) + maxBlobSize := int64(10 * 1024 * 1024) + + storer, err := storage.NewFileStorer(storeDir) + require.NoError(t, err) + + cfg := &config.Config{ + AgeRecipients: []string{testAgePublicKey}, + AgeSecretKey: testAgeSecretKey, + CompressionLevel: 3, + Hostname: testHostname, + BlobSizeLimit: config.Size(maxBlobSize), + } + + ctx := context.Background() + + snapshotID := runSweeperBackup(ctx, t, fs, storer, cfg, + dataDir, storeDir, dbPath, chunkSize, maxBlobSize, + uniqueFiles, duplicateFiles) + counter := newCountingStorer(storer) restoreVaultik := &vaultik.Vaultik{ @@ -186,25 +244,7 @@ func TestRestoreSweeperEvictsBlobs(t *testing.T) { require.Equalf(t, want, got, "byte mismatch for %s", origPath) } - // Each blob must have been downloaded exactly once. >1 means the - // sweeper evicted a still-needed blob; 0 means the cache silently - // stopped being consulted. - blobDownloads := 0 - - for key, count := range counter.snapshot() { - if !strings.HasPrefix(key, "blobs/") { - continue - } - - assert.Equalf(t, 1, count, - "blob %s should have been downloaded exactly once during restore, got %d", key, count) - - blobDownloads++ - } - - assert.Equal(t, blobCount, blobDownloads, - "every blob on disk should have been fetched exactly once during restore") - t.Logf("restore downloaded %d blobs, each exactly once", blobDownloads) + verifySingleDownloadPerBlob(t, counter, storeDir) } // countingStorer wraps a Storer and records the number of Get calls per @@ -221,7 +261,9 @@ func newCountingStorer(inner storage.Storer) *countingStorer { return &countingStorer{Storer: inner, counts: make(map[string]int)} } -func (c *countingStorer) Get(ctx context.Context, key string) (io.ReadCloser, error) { +func (c *countingStorer) Get( + ctx context.Context, key string, +) (io.ReadCloser, error) { c.mu.Lock() c.counts[key]++ c.mu.Unlock() diff --git a/internal/vaultik/snapshot.go b/internal/vaultik/snapshot.go index 3f9b040..4e6b98c 100644 --- a/internal/vaultik/snapshot.go +++ b/internal/vaultik/snapshot.go @@ -12,17 +12,32 @@ import ( "text/tabwriter" "time" - "github.com/dustin/go-humanize" "sneak.berlin/go/vaultik/internal/database" "sneak.berlin/go/vaultik/internal/log" "sneak.berlin/go/vaultik/internal/snapshot" ) +// Sentinel errors for snapshot management. +var ( + errSnapshotNotInConfig = errors.New("snapshot not found in config") + errNoSnapshotsInConfig = errors.New("no snapshots configured") + errBlobsMissing = errors.New("blobs are missing") + errSnapshotVerifyFailed = errors.New("verification failed") + errRemoveAllNeedsForce = errors.New("--all requires --force") + errInvalidTableName = errors.New("invalid table name") +) + +// listRecentLimit caps how many snapshot rows are fetched from the +// local index for listing and purge operations. +const listRecentLimit = 10000 + // SnapshotCreateOptions contains options for the snapshot create command type SnapshotCreateOptions struct { - Cron bool - Prune bool - KeepNewerThan string // With --prune: keep snapshots newer than this duration (e.g. "4w"); default: keep only latest + Cron bool + Prune bool + // KeepNewerThan is used with --prune: keep snapshots newer than this + // duration (e.g. "4w"); default: keep only latest. + KeepNewerThan string SkipErrors bool // Skip file read errors (log them loudly but continue) Snapshots []string // Optional list of snapshot names to process (empty = all) } @@ -56,7 +71,8 @@ func (v *Vaultik) CreateSnapshot(opts *SnapshotCreateOptions) error { // Prune the database before starting: delete incomplete snapshots and orphaned data. // This ensures the database is consistent before we start a new snapshot. // Since we use locking, only one vaultik instance accesses the DB at a time. - if _, err := v.PruneDatabase(); err != nil { + _, err = v.PruneDatabase() + if err != nil { return fmt.Errorf("prune database: %w", err) } @@ -68,18 +84,19 @@ func (v *Vaultik) CreateSnapshot(opts *SnapshotCreateOptions) error { // Validate requested snapshot names exist for _, name := range snapshotNames { if _, ok := v.Config.Snapshots[name]; !ok { - return fmt.Errorf("snapshot %q not found in config", name) + return fmt.Errorf("%w: %q", errSnapshotNotInConfig, name) } } } if len(snapshotNames) == 0 { - return errors.New("no snapshots configured") + return errNoSnapshotsInConfig } // Process each named snapshot for snapIdx, snapName := range snapshotNames { - err := v.createNamedSnapshot(opts, hostname, snapName, snapIdx+1, len(snapshotNames)) + err = v.createNamedSnapshot( + opts, hostname, snapName, snapIdx+1, len(snapshotNames)) if err != nil { return err } @@ -87,11 +104,12 @@ func (v *Vaultik) CreateSnapshot(opts *SnapshotCreateOptions) error { // Print overall summary if multiple snapshots if len(snapshotNames) > 1 { - v.UI.Completef("All %d snapshots completed in %s.", len(snapshotNames), v.UI.Duration(time.Since(overallStartTime))) + v.UI.Completef("All %d snapshots completed in %s.", + len(snapshotNames), v.UI.Duration(time.Since(overallStartTime))) } if opts.Prune { - err := v.runPostBackupPrune(snapshotNames, opts.KeepNewerThan) + err = v.runPostBackupPrune(snapshotNames, opts.KeepNewerThan) if err != nil { return fmt.Errorf("post-backup prune: %w", err) } @@ -103,7 +121,8 @@ func (v *Vaultik) CreateSnapshot(opts *SnapshotCreateOptions) error { // emit the summary via UI.Warning so cron actually delivers // something for the user to look at. if v.UI.WarningCount() > 0 { - v.UI.Warningf("Finished with %d warning(s) — review the output above.", v.UI.WarningCount()) + v.UI.Warningf("Finished with %d warning(s) — review the output above.", + v.UI.WarningCount()) } else { v.UI.Completef("Finished successfully.") } @@ -115,8 +134,11 @@ func (v *Vaultik) CreateSnapshot(opts *SnapshotCreateOptions) error { // orphan blobs from remote storage. If keepNewerThan is set (e.g. "4w"), // snapshots newer than that duration are kept. Otherwise only the latest // snapshot of each name is kept. -func (v *Vaultik) runPostBackupPrune(snapshotNames []string, keepNewerThan string) error { - log.Info("Running post-backup prune", "snapshots", snapshotNames, "keep_newer_than", keepNewerThan) +func (v *Vaultik) runPostBackupPrune( + snapshotNames []string, keepNewerThan string, +) error { + log.Info("Running post-backup prune", + "snapshots", snapshotNames, "keep_newer_than", keepNewerThan) v.UI.Beginf("Running post-backup prune.") purgeOpts := &SnapshotPurgeOptions{ @@ -160,7 +182,9 @@ type snapshotStats struct { } // createNamedSnapshot creates a single named snapshot -func (v *Vaultik) createNamedSnapshot(opts *SnapshotCreateOptions, hostname, snapName string, idx, total int) error { +func (v *Vaultik) createNamedSnapshot( + opts *SnapshotCreateOptions, hostname, snapName string, idx, total int, +) error { snapshotStartTime := time.Now() if total > 1 { @@ -180,7 +204,8 @@ func (v *Vaultik) createNamedSnapshot(opts *SnapshotCreateOptions, hostname, sna SkipErrors: opts.SkipErrors, }) - snapshotID, err := v.SnapshotManager.CreateSnapshotWithName(v.ctx, hostname, snapName, v.Globals.Version, v.Globals.Commit) + snapshotID, err := v.SnapshotManager.CreateSnapshotWithName( + v.ctx, hostname, snapName, v.Globals.Version, v.Globals.Commit) if err != nil { return fmt.Errorf("creating snapshot: %w", err) } @@ -195,7 +220,8 @@ func (v *Vaultik) createNamedSnapshot(opts *SnapshotCreateOptions, hostname, sna v.collectUploadStats(scanner, stats) - if err := v.finalizeSnapshotMetadata(snapshotID, stats); err != nil { + err = v.finalizeSnapshotMetadata(snapshotID, stats) + if err != nil { return err } @@ -212,7 +238,8 @@ func (v *Vaultik) createNamedSnapshot(opts *SnapshotCreateOptions, hostname, sna return nil } -// resolveSnapshotPaths resolves source directories to absolute paths with symlink resolution +// resolveSnapshotPaths resolves source directories to absolute paths +// with symlink resolution. func (v *Vaultik) resolveSnapshotPaths(snapName string) ([]string, error) { snapConfig := v.Config.Snapshots[snapName] resolvedDirs := make([]string, 0, len(snapConfig.Paths)) @@ -228,7 +255,8 @@ func (v *Vaultik) resolveSnapshotPaths(snapName string) ([]string, error) { if os.IsNotExist(err) { resolvedPath = absPath } else { - return nil, fmt.Errorf("failed to resolve symlinks for %s: %w", absPath, err) + return nil, fmt.Errorf( + "failed to resolve symlinks for %s: %w", absPath, err) } } @@ -238,8 +266,11 @@ func (v *Vaultik) resolveSnapshotPaths(snapName string) ([]string, error) { return resolvedDirs, nil } -// scanAllDirectories runs the scanner on each resolved directory and accumulates stats -func (v *Vaultik) scanAllDirectories(scanner *snapshot.Scanner, resolvedDirs []string, snapshotID string) (*snapshotStats, error) { +// scanAllDirectories runs the scanner on each resolved directory and +// accumulates stats. +func (v *Vaultik) scanAllDirectories( + scanner *snapshot.Scanner, resolvedDirs []string, snapshotID string, +) (*snapshotStats, error) { stats := &snapshotStats{} for i, dir := range resolvedDirs { @@ -252,7 +283,8 @@ func (v *Vaultik) scanAllDirectories(scanner *snapshot.Scanner, resolvedDirs []s } log.Info("Scanning directory", "path", dir) - v.UI.Beginf("Enumerating snapshot source files in %s (%d of %d).", v.UI.Path(dir), i+1, len(resolvedDirs)) + v.UI.Beginf("Enumerating snapshot source files in %s (%d of %d).", + v.UI.Path(dir), i+1, len(resolvedDirs)) result, err := scanner.Scan(v.ctx, dir, snapshotID) if err != nil { @@ -282,18 +314,22 @@ func (v *Vaultik) scanAllDirectories(scanner *snapshot.Scanner, resolvedDirs []s return stats, nil } -// collectUploadStats gathers upload statistics from the scanner's progress reporter +// collectUploadStats gathers upload statistics from the scanner's +// progress reporter. func (v *Vaultik) collectUploadStats(scanner *snapshot.Scanner, stats *snapshotStats) { if s := scanner.GetProgress(); s != nil { progressStats := s.GetStats() stats.totalBytesUploaded = progressStats.BytesUploaded.Load() stats.totalBlobsUploaded = int(progressStats.BlobsUploaded.Load()) - stats.uploadDuration = time.Duration(progressStats.UploadDurationMs.Load()) * time.Millisecond + stats.uploadDuration = time.Duration( + progressStats.UploadDurationMs.Load()) * time.Millisecond } } // finalizeSnapshotMetadata updates stats, marks complete, and exports metadata -func (v *Vaultik) finalizeSnapshotMetadata(snapshotID string, stats *snapshotStats) error { +func (v *Vaultik) finalizeSnapshotMetadata( + snapshotID string, stats *snapshotStats, +) error { extStats := snapshot.ExtendedBackupStats{ BackupStats: snapshot.BackupStats{ FilesScanned: stats.totalFiles, @@ -317,7 +353,8 @@ func (v *Vaultik) finalizeSnapshotMetadata(snapshotID string, stats *snapshotSta return fmt.Errorf("completing snapshot: %w", err) } - err = v.SnapshotManager.ExportSnapshotMetadata(v.ctx, v.Config.IndexPath, snapshotID) + err = v.SnapshotManager.ExportSnapshotMetadata( + v.ctx, v.Config.IndexPath, snapshotID) if err != nil { return fmt.Errorf("exporting snapshot metadata: %w", err) } @@ -336,17 +373,19 @@ func (v *Vaultik) uploadSpeed(bytesUploaded int64, duration time.Duration) strin } // printSnapshotSummary prints the comprehensive snapshot completion summary -func (v *Vaultik) printSnapshotSummary(snapshotID string, startTime time.Time, stats *snapshotStats) { +func (v *Vaultik) printSnapshotSummary( + snapshotID string, startTime time.Time, stats *snapshotStats, +) { snapshotDuration := time.Since(startTime) totalFilesChanged := stats.totalFiles - stats.totalFilesSkipped totalBytesAll := stats.totalBytes + stats.totalBytesSkipped // Get total blob sizes from database - totalBlobSizeCompressed, totalBlobSizeUncompressed := v.getSnapshotBlobSizes(snapshotID) + compressedSize, uncompressedSize := v.getSnapshotBlobSizes(snapshotID) var compressionRatio float64 - if totalBlobSizeUncompressed > 0 { - compressionRatio = float64(totalBlobSizeCompressed) / float64(totalBlobSizeUncompressed) + if uncompressedSize > 0 { + compressionRatio = float64(compressedSize) / float64(uncompressedSize) } else { compressionRatio = 1.0 } @@ -374,8 +413,8 @@ func (v *Vaultik) printSnapshotSummary(snapshotID string, startTime time.Time, s if stats.totalBlobsUploaded > 0 { v.UI.Detailf("Storage: %s compressed from %s (%.2fx ratio).", - v.UI.Size(totalBlobSizeCompressed), - v.UI.Size(totalBlobSizeUncompressed), + v.UI.Size(compressedSize), + v.UI.Size(uncompressedSize), compressionRatio) v.UI.Detailf("Upload: %d blobs, %s in %s (%s).", stats.totalBlobsUploaded, @@ -387,15 +426,19 @@ func (v *Vaultik) printSnapshotSummary(snapshotID string, startTime time.Time, s v.UI.Detailf("Snapshot create duration: %s.", v.UI.Duration(snapshotDuration)) } -// getSnapshotBlobSizes returns total compressed and uncompressed blob sizes for a snapshot -func (v *Vaultik) getSnapshotBlobSizes(snapshotID string) (compressed int64, uncompressed int64) { +// getSnapshotBlobSizes returns total compressed and uncompressed blob +// sizes for a snapshot. +func (v *Vaultik) getSnapshotBlobSizes(snapshotID string) (int64, int64) { + var compressed, uncompressed int64 + blobHashes, err := v.Repositories.Snapshots.GetBlobHashes(v.ctx, snapshotID) if err != nil { return 0, 0 } for _, hash := range blobHashes { - if blob, err := v.Repositories.Blobs.GetByHash(v.ctx, hash); err == nil && blob != nil { + blob, err := v.Repositories.Blobs.GetByHash(v.ctx, hash) + if err == nil && blob != nil { compressed += blob.CompressedSize uncompressed += blob.UncompressedSize } @@ -428,7 +471,7 @@ func (v *Vaultik) getSnapshotBlobSizes(snapshotID string) (compressed int64, unc func (v *Vaultik) ListSnapshots(jsonOutput bool) error { log.Info("Listing snapshots") - localSnaps, err := v.Repositories.Snapshots.ListRecent(v.ctx, 10000) + localSnaps, err := v.Repositories.Snapshots.ListRecent(v.ctx, listRecentLimit) if err != nil { return fmt.Errorf("listing local snapshots: %w", err) } @@ -453,7 +496,8 @@ func (v *Vaultik) ListSnapshots(jsonOutput bool) error { return encoder.Encode(snapshots) } - if err := v.printSnapshotTable(snapshots); err != nil { + err = v.printSnapshotTable(snapshots) + if err != nil { return err } @@ -461,11 +505,20 @@ func (v *Vaultik) ListSnapshots(jsonOutput bool) error { return nil } + v.reportRemoteDrift(localSnaps) + + return nil +} + +// reportRemoteDrift cross-references local snapshot records against the +// remote metadata keys and reports local-only records and unknown +// remote keys. Never fails: remote listing errors degrade to a warning. +func (v *Vaultik) reportRemoteDrift(localSnaps []*database.Snapshot) { remoteKeys, err := v.listAllRemoteSnapshotKeys() if err != nil { v.UI.Warningf("Could not list backup destination store: %v.", err) - return nil + return } localKeys := make(map[string]string, len(localSnaps)) @@ -499,7 +552,8 @@ func (v *Vaultik) ListSnapshots(jsonOutput bool) error { } if len(localOnly) > 0 { - v.UI.Warningf("%d local snapshot record(s) not found in backup destination store:", len(localOnly)) + v.UI.Warningf("%d local snapshot record(s) not found in backup "+ + "destination store:", len(localOnly)) for _, id := range localOnly { v.UI.Infof("%s", v.UI.Snapshot(id)) @@ -509,10 +563,9 @@ func (v *Vaultik) ListSnapshots(jsonOutput bool) error { } if remoteOnlyCount > 0 { - v.UI.Noticef("NOTE: %d remote snapshot(s) found in backup destination store but not in local database.", remoteOnlyCount) + v.UI.Noticef("NOTE: %d remote snapshot(s) found in backup destination "+ + "store but not in local database.", remoteOnlyCount) } - - return nil } // snapshotInfoFromLocal builds a SnapshotInfo row from a local snapshot @@ -521,14 +574,16 @@ func (v *Vaultik) ListSnapshots(jsonOutput bool) error { func (v *Vaultik) snapshotInfoFromLocal(ls *database.Snapshot) SnapshotInfo { idStr := ls.ID.String() - totalSize, err := v.Repositories.Snapshots.GetSnapshotTotalCompressedSize(v.ctx, idStr) + totalSize, err := v.Repositories.Snapshots.GetSnapshotTotalCompressedSize( + v.ctx, idStr) if err != nil { log.Warn("Failed to get total compressed size", "id", idStr, "error", err) totalSize = ls.BlobSize } - uncompressedSize, err := v.Repositories.Snapshots.GetSnapshotUncompressedChunkSize(v.ctx, idStr) + uncompressedSize, err := v.Repositories.Snapshots.GetSnapshotUncompressedChunkSize( + v.ctx, idStr) if err != nil { log.Warn("Failed to get uncompressed chunk size", "id", idStr, "error", err) } @@ -548,19 +603,25 @@ func (v *Vaultik) snapshotInfoFromLocal(ls *database.Snapshot) SnapshotInfo { } } +// tabPadding is the tabwriter cell padding for the snapshot table. +const tabPadding = 3 + // printSnapshotTable renders the snapshot list as a formatted table func (v *Vaultik) printSnapshotTable(snapshots []SnapshotInfo) error { - w := tabwriter.NewWriter(v.Stdout, 0, 0, 3, ' ', 0) + w := tabwriter.NewWriter(v.Stdout, 0, 0, tabPadding, ' ', 0) - if _, err := fmt.Fprintln(w, "CONFIGURED SNAPSHOTS:"); err != nil { + _, err := fmt.Fprintln(w, "CONFIGURED SNAPSHOTS:") + if err != nil { return err } - if _, err := fmt.Fprintln(w, "NAME\tPATHS"); err != nil { + _, err = fmt.Fprintln(w, "NAME\tPATHS") + if err != nil { return err } - if _, err := fmt.Fprintln(w, "────\t─────"); err != nil { + _, err = fmt.Fprintln(w, "────\t─────") + if err != nil { return err } @@ -568,24 +629,34 @@ func (v *Vaultik) printSnapshotTable(snapshots []SnapshotInfo) error { snap := v.Config.Snapshots[name] paths := strings.Join(snap.Paths, ", ") - if _, err := fmt.Fprintf(w, "%s\t%s\n", name, paths); err != nil { + + _, err = fmt.Fprintf(w, "%s\t%s\n", name, paths) + if err != nil { return err } } - if _, err := fmt.Fprintln(w); err != nil { + _, err = fmt.Fprintln(w) + if err != nil { return err } - if _, err := fmt.Fprintln(w, "REMOTE SNAPSHOTS:"); err != nil { + _, err = fmt.Fprintln(w, "REMOTE SNAPSHOTS:") + if err != nil { return err } - if _, err := fmt.Fprintln(w, "SNAPSHOT ID\tTIMESTAMP\tCOMPRESSED SIZE\tUNCOMPRESSED SIZE\tNEW CHUNK SIZE"); err != nil { + _, err = fmt.Fprintln(w, + "SNAPSHOT ID\tTIMESTAMP\tCOMPRESSED SIZE\t"+ + "UNCOMPRESSED SIZE\tNEW CHUNK SIZE") + if err != nil { return err } - if _, err := fmt.Fprintln(w, "───────────\t─────────\t───────────────\t─────────────────\t──────────────"); err != nil { + _, err = fmt.Fprintln(w, + "───────────\t─────────\t───────────────\t"+ + "─────────────────\t──────────────") + if err != nil { return err } @@ -599,12 +670,13 @@ func (v *Vaultik) printSnapshotTable(snapshots []SnapshotInfo) error { newChunks = formatBytes(snap.NewChunkSize) } - if _, err := fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n", + _, err = fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n", snap.ID, snap.Timestamp.Format("2006-01-02 15:04:05"), formatBytes(snap.CompressedSize), uncompressed, - newChunks); err != nil { + newChunks) + if err != nil { return err } } @@ -614,11 +686,15 @@ func (v *Vaultik) printSnapshotTable(snapshots []SnapshotInfo) error { // SnapshotPurgeOptions contains options for the snapshot purge command. type SnapshotPurgeOptions struct { - KeepLatest bool // Keep only the most recent snapshot per name - OlderThan string // Drop snapshots older than this duration (e.g. "30d", "6m", "1y") - Force bool // Skip confirmation prompt - Names []string // If non-empty, only operate on snapshots with one of these names - Quiet bool // Suppress informational output (used by --prune flag) + KeepLatest bool // Keep only the most recent snapshot per name + // OlderThan drops snapshots older than this duration (e.g. "30d", + // "6m", "1y"). + OlderThan string + Force bool // Skip confirmation prompt + // Names restricts the operation to snapshots with one of these + // names when non-empty. + Names []string + Quiet bool // Suppress informational output (used by --prune flag) } // PurgeSnapshotsWithOptions removes old snapshots based on criteria. @@ -626,16 +702,18 @@ type SnapshotPurgeOptions struct { // snapshot name, not the latest globally. This prevents `home` and `system` // snapshots from cannibalizing each other. func (v *Vaultik) PurgeSnapshotsWithOptions(opts *SnapshotPurgeOptions) error { - if err := v.EnsureStorageBinding(); err != nil { + err := v.EnsureStorageBinding() + if err != nil { return err } // Sync with remote first - if err := v.syncWithRemote(); err != nil { + err = v.syncWithRemote() + if err != nil { return fmt.Errorf("syncing with remote: %w", err) } // Get snapshots from local database - dbSnapshots, err := v.Repositories.Snapshots.ListRecent(v.ctx, 10000) + dbSnapshots, err := v.Repositories.Snapshots.ListRecent(v.ctx, listRecentLimit) if err != nil { return fmt.Errorf("listing snapshots: %w", err) } @@ -671,9 +749,32 @@ func (v *Vaultik) PurgeSnapshotsWithOptions(opts *SnapshotPurgeOptions) error { return snapshots[i].Timestamp.After(snapshots[j].Timestamp) }) + toDelete, err := selectSnapshotsToPurge(snapshots, opts) + if err != nil { + return err + } + + if len(toDelete) == 0 { + if !opts.Quiet { + v.printlnStdout("No snapshots to delete") + } + + return nil + } + + return v.confirmAndExecutePurge(toDelete, opts.Force, opts.Quiet) +} + +// selectSnapshotsToPurge applies the purge retention criteria to the +// newest-first sorted snapshot list and returns the deletion +// candidates. +func selectSnapshotsToPurge( + snapshots []SnapshotInfo, opts *SnapshotPurgeOptions, +) ([]SnapshotInfo, error) { var toDelete []SnapshotInfo - if opts.KeepLatest { + switch { + case opts.KeepLatest: // Keep the latest snapshot per snapshot name. Snapshots are sorted // newest-first, so the first occurrence of each name is kept. seen := make(map[string]bool) @@ -688,10 +789,10 @@ func (v *Vaultik) PurgeSnapshotsWithOptions(opts *SnapshotPurgeOptions) error { seen[name] = true } - } else if opts.OlderThan != "" { + case opts.OlderThan != "": duration, err := parseDuration(opts.OlderThan) if err != nil { - return fmt.Errorf("invalid duration: %w", err) + return nil, fmt.Errorf("invalid duration: %w", err) } cutoff := time.Now().UTC().Add(-duration) @@ -702,19 +803,14 @@ func (v *Vaultik) PurgeSnapshotsWithOptions(opts *SnapshotPurgeOptions) error { } } - if len(toDelete) == 0 { - if !opts.Quiet { - v.printlnStdout("No snapshots to delete") - } - - return nil - } - - return v.confirmAndExecutePurge(toDelete, opts.Force, opts.Quiet) + return toDelete, nil } -// confirmAndExecutePurge shows deletion candidates, confirms with user, and deletes snapshots -func (v *Vaultik) confirmAndExecutePurge(toDelete []SnapshotInfo, force, quiet bool) error { +// confirmAndExecutePurge shows deletion candidates, confirms with the +// user, and deletes snapshots. +func (v *Vaultik) confirmAndExecutePurge( + toDelete []SnapshotInfo, force, quiet bool, +) error { if !quiet { v.stdoutf("The following snapshots will be deleted:\n\n") @@ -731,11 +827,12 @@ func (v *Vaultik) confirmAndExecutePurge(toDelete []SnapshotInfo, force, quiet b v.stdoutf("\nDelete %d snapshot(s)? [y/N] ", len(toDelete)) var confirm string - if _, err := v.scanStdin(&confirm); err != nil { - // Treat EOF or error as "no" + + _, err := v.scanStdin(&confirm) + if err != nil { v.printlnStdout("Cancelled") - return nil + return nil //nolint:nilerr // treat EOF or read error as "no" } if strings.ToLower(confirm) != "y" { @@ -754,7 +851,8 @@ func (v *Vaultik) confirmAndExecutePurge(toDelete []SnapshotInfo, force, quiet b err := v.deleteSnapshotFromLocalDB(snapshotID) if err != nil { - log.Error("Failed to delete from local database", "snapshot_id", snapshotID, "error", err) + log.Error("Failed to delete from local database", + "snapshot_id", snapshotID, "error", err) } err = v.deleteRemoteSnapshotByKey(snapshot.RemoteSnapshotKey(snapshotID)) @@ -769,13 +867,15 @@ func (v *Vaultik) confirmAndExecutePurge(toDelete []SnapshotInfo, force, quiet b if v.SnapshotManager != nil { err := v.SnapshotManager.CleanupOrphanedData(v.ctx) if err != nil { - log.Warn("Failed to clean up orphaned local data after purge", "error", err) + log.Warn("Failed to clean up orphaned local data after purge", + "error", err) } } if !quiet { v.stdoutf("Deleted %d snapshot(s)\n", len(toDelete)) - v.printlnStdout("\nNote: Run 'vaultik prune' to clean up unreferenced remote blobs.") + v.printlnStdout( + "\nNote: Run 'vaultik prune' to clean up unreferenced remote blobs.") } return nil @@ -794,7 +894,9 @@ func (v *Vaultik) VerifySnapshot(snapshotID string, deep bool) error { // VerifySnapshotWithOptions checks snapshot integrity with full options. // Deep verification is delegated to RunDeepVerify so this function only // implements the shallow (existence-only) path. -func (v *Vaultik) VerifySnapshotWithOptions(snapshotID string, opts *VerifyOptions) error { +func (v *Vaultik) VerifySnapshotWithOptions( + snapshotID string, opts *VerifyOptions, +) error { if opts.Deep { return v.RunDeepVerify(snapshotID, opts) } @@ -811,7 +913,7 @@ func (v *Vaultik) VerifySnapshotWithOptions(snapshotID string, opts *VerifyOptio manifest, err := v.downloadManifestByKey(snapshot.RemoteSnapshotKey(snapshotID)) if err != nil { if opts.JSON { - result.Status = "failed" + result.Status = verifyStatusFailed result.ErrorMessage = fmt.Sprintf("downloading manifest: %v", err) return v.outputVerifyJSON(result) @@ -826,11 +928,13 @@ func (v *Vaultik) VerifySnapshotWithOptions(snapshotID string, opts *VerifyOptio if !opts.JSON { v.stdoutf("Snapshot information:\n") v.stdoutf(" Blob count: %d\n", manifest.BlobCount) - v.stdoutf(" Total size: %s\n", humanize.Bytes(uint64(manifest.TotalCompressedSize))) + v.stdoutf(" Total size: %s\n", ubytes(manifest.TotalCompressedSize)) if manifest.Timestamp != "" { - if t, err := time.Parse(time.RFC3339, manifest.Timestamp); err == nil { - v.stdoutf(" Created: %s\n", t.Format("2006-01-02 15:04:05 MST")) + t, terr := time.Parse(time.RFC3339, manifest.Timestamp) + if terr == nil { + v.stdoutf(" Created: %s\n", + t.Format("2006-01-02 15:04:05 MST")) } } @@ -840,16 +944,19 @@ func (v *Vaultik) VerifySnapshotWithOptions(snapshotID string, opts *VerifyOptio v.stdoutf("Checking blob existence...\n") } - result.Verified, result.Missing, result.MissingSize = v.verifyManifestBlobsExist(manifest, opts) + result.Verified, result.Missing, result.MissingSize = + v.verifyManifestBlobsExist(manifest, opts) return v.formatVerifyResult(result, manifest, opts) } -// printVerifyHeader prints the snapshot ID and parsed timestamp for verification output. -// Snapshot ID format: hostname[_name]_ +// printVerifyHeader prints the snapshot ID and parsed timestamp for +// verification output. Snapshot ID format: hostname[_name]_ func (v *Vaultik) printVerifyHeader(snapshotID string, opts *VerifyOptions) { var snapshotTime time.Time - if t, err := parseSnapshotTimestamp(snapshotID); err == nil { + + t, err := parseSnapshotTimestamp(snapshotID) + if err == nil { snapshotTime = t } @@ -857,23 +964,36 @@ func (v *Vaultik) printVerifyHeader(snapshotID string, opts *VerifyOptions) { v.stdoutf("Verifying snapshot %s\n", snapshotID) if !snapshotTime.IsZero() { - v.stdoutf("Snapshot time: %s\n", snapshotTime.Format("2006-01-02 15:04:05 MST")) + v.stdoutf("Snapshot time: %s\n", + snapshotTime.Format("2006-01-02 15:04:05 MST")) } v.printlnStdout() } } -// verifyManifestBlobsExist checks that each blob in the manifest exists in storage -func (v *Vaultik) verifyManifestBlobsExist(manifest *snapshot.Manifest, opts *VerifyOptions) (verified, missing int, missingSize int64) { - for _, blob := range manifest.Blobs { - blobPath := fmt.Sprintf("blobs/%s/%s/%s", blob.Hash[:2], blob.Hash[2:4], blob.Hash) +// verifyManifestBlobsExist checks that each blob in the manifest exists +// in storage, returning the verified count, missing count, and total +// missing bytes. +func (v *Vaultik) verifyManifestBlobsExist( + manifest *snapshot.Manifest, opts *VerifyOptions, +) (int, int, int64) { + var ( + verified, missing int + missingSize int64 + ) - // Shallow: just check existence (deep verification is handled by RunDeepVerify) + for _, blob := range manifest.Blobs { + blobPath := fmt.Sprintf("blobs/%s/%s/%s", + blob.Hash[:2], blob.Hash[2:4], blob.Hash) + + // Shallow: check existence only (deep verification is handled + // by RunDeepVerify). _, err := v.Storage.Stat(v.ctx, blobPath) if err != nil { if !opts.JSON { - v.stdoutf(" Missing: %s (%s)\n", blob.Hash, humanize.Bytes(uint64(blob.CompressedSize))) + v.stdoutf(" Missing: %s (%s)\n", + blob.Hash, ubytes(blob.CompressedSize)) } missing++ @@ -886,11 +1006,14 @@ func (v *Vaultik) verifyManifestBlobsExist(manifest *snapshot.Manifest, opts *Ve return verified, missing, missingSize } -// formatVerifyResult outputs the final verification results as JSON or human-readable text -func (v *Vaultik) formatVerifyResult(result *VerifyResult, manifest *snapshot.Manifest, opts *VerifyOptions) error { +// formatVerifyResult outputs the final verification results as JSON or +// human-readable text. +func (v *Vaultik) formatVerifyResult( + result *VerifyResult, manifest *snapshot.Manifest, opts *VerifyOptions, +) error { if opts.JSON { if result.Missing > 0 { - result.Status = "failed" + result.Status = verifyStatusFailed result.ErrorMessage = fmt.Sprintf("%d blobs are missing", result.Missing) } else { result.Status = "ok" @@ -901,10 +1024,11 @@ func (v *Vaultik) formatVerifyResult(result *VerifyResult, manifest *snapshot.Ma v.stdoutf("\nVerification complete:\n") v.stdoutf(" Verified: %d blobs (%s)\n", result.Verified, - humanize.Bytes(uint64(manifest.TotalCompressedSize-result.MissingSize))) + ubytes(manifest.TotalCompressedSize-result.MissingSize)) if result.Missing > 0 { - v.stdoutf(" Missing: %d blobs (%s)\n", result.Missing, humanize.Bytes(uint64(result.MissingSize))) + v.stdoutf(" Missing: %d blobs (%s)\n", + result.Missing, ubytes(result.MissingSize)) } else { v.stdoutf(" Missing: 0 blobs\n") } @@ -914,7 +1038,7 @@ func (v *Vaultik) formatVerifyResult(result *VerifyResult, manifest *snapshot.Ma if result.Missing > 0 { v.stdoutf("FAILED - %d blobs are missing\n", result.Missing) - return fmt.Errorf("%d blobs are missing", result.Missing) + return fmt.Errorf("%d %w", result.Missing, errBlobsMissing) } v.stdoutf("OK - All blobs verified\n") @@ -932,8 +1056,8 @@ func (v *Vaultik) outputVerifyJSON(result *VerifyResult) error { return fmt.Errorf("encoding JSON: %w", err) } - if result.Status == "failed" { - return fmt.Errorf("verification failed: %s", result.ErrorMessage) + if result.Status == verifyStatusFailed { + return fmt.Errorf("%w: %s", errSnapshotVerifyFailed, result.ErrorMessage) } return nil @@ -945,7 +1069,8 @@ func (v *Vaultik) outputVerifyJSON(result *VerifyResult) error { // human ID is hashed via RemoteSnapshotKey and compared against the // remote listing. func (v *Vaultik) CleanupLocalSnapshots() error { - if err := v.EnsureStorageBinding(); err != nil { + err := v.EnsureStorageBinding() + if err != nil { return err } @@ -959,7 +1084,7 @@ func (v *Vaultik) CleanupLocalSnapshots() error { remoteSet[k] = true } - localSnapshots, err := v.Repositories.Snapshots.ListRecent(v.ctx, 10000) + localSnapshots, err := v.Repositories.Snapshots.ListRecent(v.ctx, listRecentLimit) if err != nil { return fmt.Errorf("listing local snapshots: %w", err) } @@ -971,9 +1096,10 @@ func (v *Vaultik) CleanupLocalSnapshots() error { if !remoteSet[snapshot.RemoteSnapshotKey(id)] { v.stdoutf("Removing stale local record: %s\n", id) - err := v.deleteSnapshotFromLocalDB(id) + err = v.deleteSnapshotFromLocalDB(id) if err != nil { - log.Error("Failed to delete local snapshot", "snapshot_id", id, "error", err) + log.Error("Failed to delete local snapshot", + "snapshot_id", id, "error", err) continue } @@ -1028,7 +1154,8 @@ func (v *Vaultik) syncWithRemote() error { // Extract snapshot ID from paths like metadata/hostname-20240115-143052Z/ parts := strings.Split(object.Key, "/") - if len(parts) >= 2 && parts[0] == "metadata" && parts[1] != "" { + if len(parts) >= minSnapshotIDParts && + parts[0] == metadataDirName && parts[1] != "" { // Skip macOS resource fork files (._*) and other hidden files if strings.HasPrefix(parts[1], ".") { continue @@ -1041,7 +1168,7 @@ func (v *Vaultik) syncWithRemote() error { log.Debug("Found remote snapshots", "count", len(remoteSnapshots)) // Get all local snapshots (use a high limit to get all) - localSnapshots, err := v.Repositories.Snapshots.ListRecent(v.ctx, 10000) + localSnapshots, err := v.Repositories.Snapshots.ListRecent(v.ctx, listRecentLimit) if err != nil { return fmt.Errorf("listing local snapshots: %w", err) } @@ -1049,14 +1176,16 @@ func (v *Vaultik) syncWithRemote() error { // Remove local snapshots that don't exist remotely removedCount := 0 - for _, snapshot := range localSnapshots { - snapshotIDStr := snapshot.ID.String() + for _, snap := range localSnapshots { + snapshotIDStr := snap.ID.String() if !remoteSnapshots[snapshotIDStr] { - log.Info("Removing local snapshot not found in remote", "snapshot_id", snapshot.ID) + log.Info("Removing local snapshot not found in remote", + "snapshot_id", snap.ID) - err := v.deleteSnapshotFromLocalDB(snapshotIDStr) + err = v.deleteSnapshotFromLocalDB(snapshotIDStr) if err != nil { - log.Error("Failed to delete local snapshot", "snapshot_id", snapshot.ID, "error", err) + log.Error("Failed to delete local snapshot", + "snapshot_id", snap.ID, "error", err) } else { removedCount++ } @@ -1079,6 +1208,8 @@ type RemoveOptions struct { } // RemoveResult contains the result of a snapshot removal +// +//nolint:tagliatelle // snake_case is the established JSON output format type RemoveResult struct { SnapshotID string `json:"snapshot_id,omitempty"` SnapshotsRemoved []string `json:"snapshots_removed,omitempty"` @@ -1099,7 +1230,9 @@ const pruneCommandHint = "vaultik prune" // `vaultik prune` is for — the command prints the prune invocation to // run as a follow-up. When the remote is unreachable the command still // completes the local-DB removal and warns. -func (v *Vaultik) RemoveSnapshot(snapshotID string, opts *RemoveOptions) (*RemoveResult, error) { +func (v *Vaultik) RemoveSnapshot( + snapshotID string, opts *RemoveOptions, +) (*RemoveResult, error) { result := &RemoveResult{ SnapshotID: snapshotID, } @@ -1110,44 +1243,11 @@ func (v *Vaultik) RemoveSnapshot(snapshotID string, opts *RemoveOptions) (*Remov } if opts.DryRun { - result.DryRun = true - - if !opts.JSON { - v.stdoutf("Would remove snapshot: %s\n", snapshotID) - - if !opts.LocalOnly { - v.printlnStdout("Would also remove snapshot metadata from remote storage (blobs untouched)") - } - - v.printlnStdout("[Dry run - no changes made]") - } - - if opts.JSON { - return result, v.outputRemoveJSON(result) - } - - return result, nil + return v.removeSnapshotDryRun(result, snapshotID, opts) } - if !opts.Force && !opts.JSON { - if opts.LocalOnly { - v.stdoutf("Remove snapshot '%s' from local database (remote untouched)? [y/N] ", snapshotID) - } else { - v.stdoutf("Remove snapshot '%s' from local database AND its metadata from remote storage? [y/N] ", snapshotID) - } - - var confirm string - if _, err := v.scanStdin(&confirm); err != nil { - v.printlnStdout("Cancelled") - - return result, nil - } - - if strings.ToLower(confirm) != "y" { - v.printlnStdout("Cancelled") - - return result, nil - } + if !opts.Force && !opts.JSON && !v.confirmRemoveSnapshot(snapshotID, opts) { + return result, nil } log.Info("Removing snapshot from local database", "snapshot_id", snapshotID) @@ -1158,30 +1258,14 @@ func (v *Vaultik) RemoveSnapshot(snapshotID string, opts *RemoveOptions) (*Remov } if !opts.LocalOnly { - log.Info("Removing snapshot metadata from remote storage", "snapshot_id", snapshotID) - - remoteKey := snapshot.RemoteSnapshotKey(snapshotID) - - err := v.deleteRemoteSnapshotByKey(remoteKey) - if err != nil { - // Warn-and-proceed: the local-DB removal has already - // happened, so let the user know the remote half didn't - // finish and they can retry with `vaultik prune` once the - // destination store is reachable. - log.Warn("Could not remove snapshot metadata from remote storage", "error", err) - - if v.UI != nil { - v.UI.Warningf("Could not remove snapshot metadata from remote: %v. Run '%s' once the remote is reachable to finish cleanup.", err, pruneCommandHint) - } - } else { - result.RemoteRemoved = true - } + result.RemoteRemoved = v.removeSnapshotRemote(snapshotID) } if v.SnapshotManager != nil { - err := v.SnapshotManager.CleanupOrphanedData(v.ctx) + err = v.SnapshotManager.CleanupOrphanedData(v.ctx) if err != nil { - log.Warn("Failed to clean up orphaned local data after removal", "error", err) + log.Warn("Failed to clean up orphaned local data after removal", + "error", err) } } @@ -1193,19 +1277,103 @@ func (v *Vaultik) RemoveSnapshot(snapshotID string, opts *RemoveOptions) (*Remov if !opts.LocalOnly && result.RemoteRemoved { v.printlnStdout("Removed snapshot metadata from remote storage") - v.stdoutf("\nNote: The removed snapshot's blobs remain on the remote. Run '%s' to delete any blobs no longer referenced by any remaining remote snapshot.\n", pruneCommandHint) + v.stdoutf("\nNote: The removed snapshot's blobs remain on the remote. "+ + "Run '%s' to delete any blobs no longer referenced by any "+ + "remaining remote snapshot.\n", pruneCommandHint) } return result, nil } +// removeSnapshotDryRun reports what RemoveSnapshot would do without +// making changes. +func (v *Vaultik) removeSnapshotDryRun( + result *RemoveResult, snapshotID string, opts *RemoveOptions, +) (*RemoveResult, error) { + result.DryRun = true + + if opts.JSON { + return result, v.outputRemoveJSON(result) + } + + v.stdoutf("Would remove snapshot: %s\n", snapshotID) + + if !opts.LocalOnly { + v.printlnStdout("Would also remove snapshot metadata from " + + "remote storage (blobs untouched)") + } + + v.printlnStdout("[Dry run - no changes made]") + + return result, nil +} + +// confirmRemoveSnapshot prompts for confirmation before removing a +// snapshot. Returns false (treating EOF/read errors as "no") unless the +// user answers "y". +func (v *Vaultik) confirmRemoveSnapshot(snapshotID string, opts *RemoveOptions) bool { + if opts.LocalOnly { + v.stdoutf("Remove snapshot '%s' from local database "+ + "(remote untouched)? [y/N] ", snapshotID) + } else { + v.stdoutf("Remove snapshot '%s' from local database AND its metadata "+ + "from remote storage? [y/N] ", snapshotID) + } + + var confirm string + + _, err := v.scanStdin(&confirm) + if err != nil { + v.printlnStdout("Cancelled") + + return false + } + + if strings.ToLower(confirm) != "y" { + v.printlnStdout("Cancelled") + + return false + } + + return true +} + +// removeSnapshotRemote strips the snapshot's metadata from the +// destination store, warning and proceeding on failure: the local-DB +// removal has already happened, so the user is told the remote half +// didn't finish and can retry with `vaultik prune` once the destination +// store is reachable. Returns true when the remote removal succeeded. +func (v *Vaultik) removeSnapshotRemote(snapshotID string) bool { + log.Info("Removing snapshot metadata from remote storage", + "snapshot_id", snapshotID) + + remoteKey := snapshot.RemoteSnapshotKey(snapshotID) + + err := v.deleteRemoteSnapshotByKey(remoteKey) + if err != nil { + log.Warn("Could not remove snapshot metadata from remote storage", + "error", err) + + if v.UI != nil { + v.UI.Warningf("Could not remove snapshot metadata from remote: "+ + "%v. Run '%s' once the remote is reachable to finish cleanup.", + err, pruneCommandHint) + } + + return false + } + + return true +} + // RemoveAllSnapshots removes every snapshot known to the local // database from the local index, and (with --remote) every snapshot // metadata directory in remote storage. Both sides are processed so a // "remove --all" leaves nothing behind, even when the local DB and // remote storage have diverged. func (v *Vaultik) RemoveAllSnapshots(opts *RemoveOptions) (*RemoveResult, error) { - if err := v.EnsureStorageBinding(); err != nil { + err := v.EnsureStorageBinding() + if err != nil { return nil, err } @@ -1263,7 +1431,7 @@ func (v *Vaultik) localSnapshotIDs() ([]string, error) { return nil, nil } - snaps, err := v.Repositories.Snapshots.ListRecent(v.ctx, 100000) + snaps, err := v.Repositories.Snapshots.ListRecent(v.ctx, listRecentLimit) if err != nil { return nil, err } @@ -1298,13 +1466,15 @@ func (v *Vaultik) listAllRemoteSnapshotKeys() ([]string, error) { } parts := strings.Split(object.Key, "/") - if len(parts) >= 2 && parts[0] == "metadata" && parts[1] != "" { + if len(parts) >= minSnapshotIDParts && + parts[0] == metadataDirName && parts[1] != "" { // Skip macOS resource fork files (._*) and other hidden files if strings.HasPrefix(parts[1], ".") { continue } - if strings.HasSuffix(object.Key, "/") || strings.Contains(object.Key, "/manifest.json.zst") { + if strings.HasSuffix(object.Key, "/") || + strings.Contains(object.Key, "/manifest.json.zst") { key := parts[1] if !seen[key] { seen[key] = true @@ -1318,12 +1488,15 @@ func (v *Vaultik) listAllRemoteSnapshotKeys() ([]string, error) { } // handleRemoveAllDryRun handles the dry-run mode for removing all snapshots -func (v *Vaultik) handleRemoveAllDryRun(localSnaps, orphanRemoteKeys []string, opts *RemoveOptions) (*RemoveResult, error) { +func (v *Vaultik) handleRemoveAllDryRun( + localSnaps, orphanRemoteKeys []string, opts *RemoveOptions, +) (*RemoveResult, error) { result := &RemoveResult{DryRun: true} result.SnapshotsRemoved = append(result.SnapshotsRemoved, localSnaps...) if !opts.LocalOnly { - result.SnapshotsRemoved = append(result.SnapshotsRemoved, orphanRemoteKeys...) + result.SnapshotsRemoved = append( + result.SnapshotsRemoved, orphanRemoteKeys...) } if !opts.JSON { @@ -1335,13 +1508,15 @@ func (v *Vaultik) handleRemoveAllDryRun(localSnaps, orphanRemoteKeys []string, o if !opts.LocalOnly { if len(orphanRemoteKeys) > 0 { - v.stdoutf("Would also remove %d orphan remote snapshot key(s):\n", len(orphanRemoteKeys)) + v.stdoutf("Would also remove %d orphan remote snapshot key(s):\n", + len(orphanRemoteKeys)) for _, key := range orphanRemoteKeys { v.stdoutf(" %s\n", key) } } else { - v.printlnStdout("Would also remove snapshot metadata from remote storage (blobs untouched)") + v.printlnStdout("Would also remove snapshot metadata from " + + "remote storage (blobs untouched)") } } @@ -1359,58 +1534,30 @@ func (v *Vaultik) handleRemoveAllDryRun(localSnaps, orphanRemoteKeys []string, o // is set, every corresponding remote metadata directory plus any // orphan remote keys. Blobs are not touched; the printed prune-command // hint is the next step. -func (v *Vaultik) executeRemoveAll(localSnaps, orphanRemoteKeys []string, opts *RemoveOptions) (*RemoveResult, error) { +func (v *Vaultik) executeRemoveAll( + localSnaps, orphanRemoteKeys []string, opts *RemoveOptions, +) (*RemoveResult, error) { if !opts.Force { - return nil, errors.New("--all requires --force") + return nil, errRemoveAllNeedsForce } - log.Info("Removing all snapshots", "local_count", len(localSnaps), "orphan_remote_count", len(orphanRemoteKeys)) + log.Info("Removing all snapshots", + "local_count", len(localSnaps), + "orphan_remote_count", len(orphanRemoteKeys)) result := &RemoveResult{} - remoteErrors := 0 - for _, snapshotID := range localSnaps { - log.Info("Removing snapshot", "snapshot_id", snapshotID) - - err := v.deleteSnapshotFromLocalDB(snapshotID) - if err != nil { - log.Error("Failed to remove from local database", "snapshot_id", snapshotID, "error", err) - - continue - } - - if !opts.LocalOnly { - err := v.deleteRemoteSnapshotByKey(snapshot.RemoteSnapshotKey(snapshotID)) - if err != nil { - log.Warn("Failed to remove snapshot metadata from remote", "snapshot_id", snapshotID, "error", err) - - remoteErrors++ - } - } - - result.SnapshotsRemoved = append(result.SnapshotsRemoved, snapshotID) - } + remoteErrors := v.removeAllLocalSnapshots(localSnaps, opts, result) if !opts.LocalOnly { - for _, key := range orphanRemoteKeys { - log.Info("Removing orphan remote snapshot", "remote_key", key) - - err := v.deleteRemoteSnapshotByKey(key) - if err != nil { - log.Warn("Failed to remove orphan from remote", "remote_key", key, "error", err) - - remoteErrors++ - - continue - } - - result.SnapshotsRemoved = append(result.SnapshotsRemoved, key) - } + remoteErrors += v.removeAllOrphanRemotes(orphanRemoteKeys, result) if remoteErrors == 0 { result.RemoteRemoved = true } else if v.UI != nil { - v.UI.Warningf("Some remote metadata deletions failed. Run '%s' once the remote is healthy to clean up unreferenced blobs.", pruneCommandHint) + v.UI.Warningf("Some remote metadata deletions failed. Run '%s' "+ + "once the remote is healthy to clean up unreferenced blobs.", + pruneCommandHint) } } @@ -1429,12 +1576,112 @@ func (v *Vaultik) executeRemoveAll(localSnaps, orphanRemoteKeys []string, opts * if !opts.LocalOnly && result.RemoteRemoved { v.printlnStdout("Removed snapshot metadata from remote storage") - v.stdoutf("\nNote: Removed snapshots' blobs remain on the remote. Run '%s' to delete any blobs no longer referenced by any remaining remote snapshot.\n", pruneCommandHint) + v.stdoutf("\nNote: Removed snapshots' blobs remain on the remote. "+ + "Run '%s' to delete any blobs no longer referenced by any "+ + "remaining remote snapshot.\n", pruneCommandHint) } return result, nil } +// deleteIncompleteSnapshot deletes one incomplete snapshot and its +// association rows, reporting whether the snapshot row itself was +// deleted. +func (v *Vaultik) deleteIncompleteSnapshot(snapshotIDStr string, id any) bool { + log.Info("Deleting incomplete snapshot", "snapshot_id", id) + + // Delete related records first + err := v.Repositories.Snapshots.DeleteSnapshotFiles(v.ctx, snapshotIDStr) + if err != nil { + log.Error("Failed to delete snapshot files", + "snapshot_id", id, "error", err) + } + + err = v.Repositories.Snapshots.DeleteSnapshotBlobs(v.ctx, snapshotIDStr) + if err != nil { + log.Error("Failed to delete snapshot blobs", + "snapshot_id", id, "error", err) + } + + err = v.Repositories.Snapshots.DeleteSnapshotUploads(v.ctx, snapshotIDStr) + if err != nil { + log.Error("Failed to delete snapshot uploads", + "snapshot_id", id, "error", err) + } + + err = v.Repositories.Snapshots.Delete(v.ctx, snapshotIDStr) + if err != nil { + log.Error("Failed to delete snapshot", + "snapshot_id", id, "error", err) + + return false + } + + return true +} + +// removeAllLocalSnapshots removes each known local snapshot (and, unless +// --local-only, its remote metadata), returning the number of remote +// deletion failures. +func (v *Vaultik) removeAllLocalSnapshots( + localSnaps []string, opts *RemoveOptions, result *RemoveResult, +) int { + remoteErrors := 0 + + for _, snapshotID := range localSnaps { + log.Info("Removing snapshot", "snapshot_id", snapshotID) + + err := v.deleteSnapshotFromLocalDB(snapshotID) + if err != nil { + log.Error("Failed to remove from local database", + "snapshot_id", snapshotID, "error", err) + + continue + } + + if !opts.LocalOnly { + err = v.deleteRemoteSnapshotByKey( + snapshot.RemoteSnapshotKey(snapshotID)) + if err != nil { + log.Warn("Failed to remove snapshot metadata from remote", + "snapshot_id", snapshotID, "error", err) + + remoteErrors++ + } + } + + result.SnapshotsRemoved = append(result.SnapshotsRemoved, snapshotID) + } + + return remoteErrors +} + +// removeAllOrphanRemotes deletes remote snapshot metadata that has no +// local counterpart, returning the number of deletion failures. +func (v *Vaultik) removeAllOrphanRemotes( + orphanRemoteKeys []string, result *RemoveResult, +) int { + remoteErrors := 0 + + for _, key := range orphanRemoteKeys { + log.Info("Removing orphan remote snapshot", "remote_key", key) + + err := v.deleteRemoteSnapshotByKey(key) + if err != nil { + log.Warn("Failed to remove orphan from remote", + "remote_key", key, "error", err) + + remoteErrors++ + + continue + } + + result.SnapshotsRemoved = append(result.SnapshotsRemoved, key) + } + + return remoteErrors +} + // deleteSnapshotFromLocalDB removes a snapshot from the local database only func (v *Vaultik) deleteSnapshotFromLocalDB(snapshotID string) error { if v.Repositories == nil { @@ -1516,8 +1763,10 @@ type PruneResult struct { // and blobs from the local database. This ensures database consistency // before starting a new backup or on-demand via the prune command. func (v *Vaultik) PruneDatabase() (*PruneResult, error) { - log.Info("Pruning local database: removing incomplete snapshots and orphaned data") - v.UI.Beginf("Pruning local index database (removing incomplete snapshots and orphaned data).") + log.Info("Pruning local database: " + + "removing incomplete snapshots and orphaned data") + v.UI.Beginf("Pruning local index database " + + "(removing incomplete snapshots and orphaned data).") result := &PruneResult{} @@ -1530,29 +1779,8 @@ func (v *Vaultik) PruneDatabase() (*PruneResult, error) { return nil, fmt.Errorf("getting incomplete snapshots: %w", err) } - for _, snapshot := range incompleteSnapshots { - snapshotIDStr := snapshot.ID.String() - log.Info("Deleting incomplete snapshot", "snapshot_id", snapshot.ID) - // Delete related records first - err := v.Repositories.Snapshots.DeleteSnapshotFiles(v.ctx, snapshotIDStr) - if err != nil { - log.Error("Failed to delete snapshot files", "snapshot_id", snapshot.ID, "error", err) - } - - err = v.Repositories.Snapshots.DeleteSnapshotBlobs(v.ctx, snapshotIDStr) - if err != nil { - log.Error("Failed to delete snapshot blobs", "snapshot_id", snapshot.ID, "error", err) - } - - err = v.Repositories.Snapshots.DeleteSnapshotUploads(v.ctx, snapshotIDStr) - if err != nil { - log.Error("Failed to delete snapshot uploads", "snapshot_id", snapshot.ID, "error", err) - } - - err = v.Repositories.Snapshots.Delete(v.ctx, snapshotIDStr) - if err != nil { - log.Error("Failed to delete snapshot", "snapshot_id", snapshot.ID, "error", err) - } else { + for _, snap := range incompleteSnapshots { + if v.deleteIncompleteSnapshot(snap.ID.String(), snap.ID) { result.SnapshotsDeleted++ } } @@ -1563,7 +1791,8 @@ func (v *Vaultik) PruneDatabase() (*PruneResult, error) { blobCountBefore, _ := v.getTableCount("blobs") // Run the cleanup - if err := v.SnapshotManager.CleanupOrphanedData(v.ctx); err != nil { + err = v.SnapshotManager.CleanupOrphanedData(v.ctx) + if err != nil { return nil, fmt.Errorf("cleanup orphaned data: %w", err) } @@ -1586,26 +1815,32 @@ func (v *Vaultik) PruneDatabase() (*PruneResult, error) { snapshotCountAfter := snapshotCountBefore - result.SnapshotsDeleted v.UI.Completef("Pruned local index database.") - v.UI.Detailf("Incomplete snapshots: %d removed (%d remain).", result.SnapshotsDeleted, snapshotCountAfter) - v.UI.Detailf("Orphaned files: %d removed (%d remain).", result.FilesDeleted, fileCountAfter) - v.UI.Detailf("Orphaned chunks: %d removed (%d remain).", result.ChunksDeleted, chunkCountAfter) - v.UI.Detailf("Orphaned blobs: %d removed (%d remain).", result.BlobsDeleted, blobCountAfter) + v.UI.Detailf("Incomplete snapshots: %d removed (%d remain).", + result.SnapshotsDeleted, snapshotCountAfter) + v.UI.Detailf("Orphaned files: %d removed (%d remain).", + result.FilesDeleted, fileCountAfter) + v.UI.Detailf("Orphaned chunks: %d removed (%d remain).", + result.ChunksDeleted, chunkCountAfter) + v.UI.Detailf("Orphaned blobs: %d removed (%d remain).", + result.BlobsDeleted, blobCountAfter) return result, nil } -// validTableNameRe matches table names containing only lowercase alphanumeric characters and underscores. +// validTableNameRe matches table names containing only lowercase +// alphanumeric characters and underscores. var validTableNameRe = regexp.MustCompile(`^[a-z0-9_]+$`) -// getTableCount returns the count of rows in a table. -// The tableName is sanitized to only allow [a-z0-9_] characters to prevent SQL injection. +// getTableCount returns the count of rows in a table. The tableName is +// sanitized to only allow [a-z0-9_] characters to prevent SQL +// injection. func (v *Vaultik) getTableCount(tableName string) (int64, error) { if v.DB == nil { return 0, nil } if !validTableNameRe.MatchString(tableName) { - return 0, fmt.Errorf("invalid table name: %q", tableName) + return 0, fmt.Errorf("%w: %q", errInvalidTableName, tableName) } var count int64 diff --git a/internal/vaultik/snapshot_prune_test.go b/internal/vaultik/snapshot_prune_test.go index dbff412..8b76a93 100644 --- a/internal/vaultik/snapshot_prune_test.go +++ b/internal/vaultik/snapshot_prune_test.go @@ -1,20 +1,24 @@ -package vaultik +package vaultik_test import ( "testing" + + "sneak.berlin/go/vaultik/internal/vaultik" ) // TestSnapshotCreateOptions_PruneFlag verifies the Prune field exists on // SnapshotCreateOptions and can be set. func TestSnapshotCreateOptions_PruneFlag(t *testing.T) { - opts := &SnapshotCreateOptions{ + t.Parallel() + + opts := &vaultik.SnapshotCreateOptions{ Prune: true, } if !opts.Prune { t.Error("Expected Prune to be true") } - opts2 := &SnapshotCreateOptions{ + opts2 := &vaultik.SnapshotCreateOptions{ Prune: false, } if opts2.Prune { diff --git a/internal/vaultik/storage_bind.go b/internal/vaultik/storage_bind.go index 27f79c2..2570d48 100644 --- a/internal/vaultik/storage_bind.go +++ b/internal/vaultik/storage_bind.go @@ -1,12 +1,18 @@ package vaultik import ( + "errors" "fmt" "sneak.berlin/go/vaultik/internal/database" "sneak.berlin/go/vaultik/internal/log" ) +// errStorageBindingMismatch is returned when the local index database is +// bound to a different destination than the configured storage_url. +var errStorageBindingMismatch = errors.New( + "local index is bound to a different backup destination") + // EnsureStorageBinding guarantees that the local index database is // bound to the currently-configured storage destination. Every mutating // command must call this before touching either the local index or the @@ -54,7 +60,8 @@ func (v *Vaultik) EnsureStorageBinding() error { } if stored == "" { - err := v.Repositories.LocalMeta.Set(v.ctx, database.LocalMetaKeyStorageURL, configured) + err = v.Repositories.LocalMeta.Set( + v.ctx, database.LocalMetaKeyStorageURL, configured) if err != nil { return fmt.Errorf("recording local storage binding: %w", err) } @@ -68,18 +75,19 @@ func (v *Vaultik) EnsureStorageBinding() error { return nil } - return fmt.Errorf("%s", buildBindingMismatchMessage(stored, configured)) + return fmt.Errorf("%w\n%s", + errStorageBindingMismatch, buildBindingMismatchMessage(stored, configured)) } // buildBindingMismatchMessage assembles the multi-line explanation // shown when the local index is bound to a different destination than -// the currently-configured one. Kept as a separate function so the -// lint-flagged multi-line format string is expressed as a plain string -// literal rather than a fmt.Errorf argument (staticcheck ST1005 -// disallows trailing punctuation on error format strings). +// the currently-configured one (the first line lives in the +// errStorageBindingMismatch sentinel). Kept as a separate function so +// the multi-line text is expressed as a plain string literal rather +// than a fmt.Errorf argument (staticcheck ST1005 disallows trailing +// punctuation on error format strings). func buildBindingMismatchMessage(stored, configured string) string { - return "local index is bound to a different backup destination\n" + - " local index bound to: " + stored + "\n" + + return " local index bound to: " + stored + "\n" + " currently configured: " + configured + "\n" + "\n" + "The local index database tracks which chunks and blobs already exist at the\n" + diff --git a/internal/vaultik/storage_bind_test.go b/internal/vaultik/storage_bind_test.go index 087031b..a2f8272 100644 --- a/internal/vaultik/storage_bind_test.go +++ b/internal/vaultik/storage_bind_test.go @@ -15,7 +15,9 @@ import ( // buildBindTestVaultik returns a minimal Vaultik wired with a real // in-memory DB and a config carrying the given StorageURL. Enough to // exercise EnsureStorageBinding without spinning up storage or fx. -func buildBindTestVaultik(t *testing.T, storageURL string) (*vaultik.TestVaultik, *database.Repositories) { +func buildBindTestVaultik( + t *testing.T, storageURL string, +) (*vaultik.TestVaultik, *database.Repositories) { t.Helper() db, err := database.NewTestDB() @@ -35,6 +37,7 @@ func buildBindTestVaultik(t *testing.T, storageURL string) (*vaultik.TestVaultik func TestEnsureStorageBinding_FreshDBRecordsURL(t *testing.T) { log.Initialize(log.Config{}) + t.Parallel() tv, repos := buildBindTestVaultik(t, "file:///mnt/backups/new") @@ -42,11 +45,13 @@ func TestEnsureStorageBinding_FreshDBRecordsURL(t *testing.T) { got, err := repos.LocalMeta.Get(context.Background(), database.LocalMetaKeyStorageURL) require.NoError(t, err) - assert.Equal(t, "file:///mnt/backups/new", got, "first call must record the configured URL") + assert.Equal(t, "file:///mnt/backups/new", got, + "first call must record the configured URL") } func TestEnsureStorageBinding_MatchingURLPasses(t *testing.T) { log.Initialize(log.Config{}) + t.Parallel() tv, repos := buildBindTestVaultik(t, "s3://bucket/prefix") @@ -58,6 +63,7 @@ func TestEnsureStorageBinding_MatchingURLPasses(t *testing.T) { func TestEnsureStorageBinding_MismatchRefuses(t *testing.T) { log.Initialize(log.Config{}) + t.Parallel() tv, repos := buildBindTestVaultik(t, "file:///mnt/backups/new") diff --git a/internal/vaultik/vaultik.go b/internal/vaultik/vaultik.go index 7679229..8ac75d0 100644 --- a/internal/vaultik/vaultik.go +++ b/internal/vaultik/vaultik.go @@ -1,3 +1,5 @@ +// Package vaultik implements the core backup, restore, verify, prune, +// and snapshot-management operations behind the vaultik CLI. package vaultik import ( @@ -19,6 +21,12 @@ import ( "sneak.berlin/go/vaultik/internal/ui" ) +// Sentinel errors for misconfigured encryption settings. +var ( + errNoAgeRecipients = errors.New("no age recipients configured") + errNoAgeSecretKey = errors.New("no age secret key configured") +) + // Vaultik contains all dependencies needed for vaultik operations type Vaultik struct { Globals *globals.Globals @@ -32,7 +40,7 @@ type Vaultik struct { Fs afero.Fs // Context management - ctx context.Context + ctx context.Context //nolint:containedctx // ctx bound at construction by design cancel context.CancelFunc // IO @@ -54,8 +62,8 @@ type Vaultik struct { restoreCacheObserver func(*blobDiskCache) } -// VaultikParams contains all parameters for New that can be provided by fx -type VaultikParams struct { +// Params contains all parameters for New that can be provided by fx +type Params struct { fx.In Globals *globals.Globals @@ -71,7 +79,7 @@ type VaultikParams struct { // New creates a new Vaultik instance with proper context management // It automatically includes crypto capabilities if age_secret_key is configured -func New(params VaultikParams) *Vaultik { +func New(params Params) *Vaultik { ctx, cancel := context.WithCancel(context.Background()) // Use provided filesystem or default to OS filesystem @@ -126,7 +134,7 @@ func (v *Vaultik) CanDecrypt() bool { // Returns an error if no recipients are configured func (v *Vaultik) GetEncryptor() (*crypto.Encryptor, error) { if len(v.Config.AgeRecipients) == 0 { - return nil, errors.New("no age recipients configured") + return nil, errNoAgeRecipients } return crypto.NewEncryptor(v.Config.AgeRecipients) @@ -136,13 +144,15 @@ func (v *Vaultik) GetEncryptor() (*crypto.Encryptor, error) { // Returns an error if no secret key is configured func (v *Vaultik) GetDecryptor() (*crypto.Decryptor, error) { if v.Config.AgeSecretKey == "" { - return nil, errors.New("no age secret key configured") + return nil, errNoAgeSecretKey } return crypto.NewDecryptor(v.Config.AgeSecretKey) } // GetFilesystem returns the filesystem instance used by Vaultik +// +//nolint:ireturn // afero.Fs is the filesystem abstraction by design func (v *Vaultik) GetFilesystem() afero.Fs { return v.Fs } diff --git a/internal/vaultik/verify.go b/internal/vaultik/verify.go index 61d1089..f5a7365 100644 --- a/internal/vaultik/verify.go +++ b/internal/vaultik/verify.go @@ -4,19 +4,37 @@ import ( "crypto/sha256" "database/sql" "encoding/hex" + "errors" "fmt" "hash" "io" "os" "time" - "github.com/dustin/go-humanize" "github.com/klauspost/compress/zstd" + + // Blank import registers the pure-Go sqlite driver for database/sql. _ "modernc.org/sqlite" "sneak.berlin/go/vaultik/internal/log" "sneak.berlin/go/vaultik/internal/snapshot" ) +// Sentinel errors for snapshot verification failures. +var ( + errVerificationFailed = errors.New("verification failed") + errSecretKeyRequired = errors.New( + "VAULTIK_AGE_SECRET_KEY not set; required for deep verification") + errChunksOutOfOrder = errors.New("chunks out of order") + errChunkHashMismatch = errors.New("chunk hash mismatch") + errTrailingBlobData = errors.New( + "blob has unexpected trailing bytes not covered by chunk list") + errManifestExtraBlob = errors.New("manifest contains blob not in database") + errBlobSizeMismatch = errors.New("blob size mismatch") +) + +// verifyStatusFailed is the JSON status value for a failed verification. +const verifyStatusFailed = "failed" + // VerifyOptions contains options for the verify command type VerifyOptions struct { Deep bool @@ -24,6 +42,8 @@ type VerifyOptions struct { } // VerifyResult contains the result of a snapshot verification +// +//nolint:tagliatelle // snake_case is the established JSON output format type VerifyResult struct { SnapshotID string `json:"snapshot_id"` Status string `json:"status"` // "ok" or "failed" @@ -37,8 +57,10 @@ type VerifyResult struct { } // deepVerifyFailure records a failure in the result and returns it appropriately -func (v *Vaultik) deepVerifyFailure(result *VerifyResult, opts *VerifyOptions, msg string, err error) error { - result.Status = "failed" +func (v *Vaultik) deepVerifyFailure( + result *VerifyResult, opts *VerifyOptions, msg string, err error, +) error { + result.Status = verifyStatusFailed result.ErrorMessage = msg if opts.JSON { @@ -49,7 +71,7 @@ func (v *Vaultik) deepVerifyFailure(result *VerifyResult, opts *VerifyOptions, m return err } - return fmt.Errorf("%s", msg) + return fmt.Errorf("%w: %s", errVerificationFailed, msg) } // RunDeepVerify executes deep verification operation @@ -60,9 +82,8 @@ func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error { } if !v.CanDecrypt() { - msg := "VAULTIK_AGE_SECRET_KEY not set; required for deep verification" - - return v.deepVerifyFailure(result, opts, msg, fmt.Errorf("%s", msg)) + return v.deepVerifyFailure(result, opts, + errSecretKeyRequired.Error(), errSecretKeyRequired) } log.Info("Starting snapshot verification", "snapshot_id", snapshotID, "mode", "deep") @@ -90,7 +111,8 @@ func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error { result.TotalSize = totalSize - if err := v.runVerificationSteps(manifest, dbBlobs, tempDB, opts, result, totalSize); err != nil { + err = v.runVerificationSteps(manifest, dbBlobs, tempDB, opts, result, totalSize) + if err != nil { return err } @@ -106,13 +128,15 @@ func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error { v.stdoutf("\n✓ Verification completed successfully\n") v.stdoutf(" Snapshot: %s\n", snapshotID) v.stdoutf(" Blobs verified: %d\n", len(dbBlobs)) - v.stdoutf(" Total size: %s\n", humanize.Bytes(uint64(totalSize))) + v.stdoutf(" Total size: %s\n", ubytes(totalSize)) return nil } // loadVerificationData downloads manifest, database, and blob list for verification -func (v *Vaultik) loadVerificationData(snapshotID string, opts *VerifyOptions, result *VerifyResult) (*snapshot.Manifest, *tempDB, []snapshot.BlobInfo, error) { +func (v *Vaultik) loadVerificationData( + snapshotID string, opts *VerifyOptions, result *VerifyResult, +) (*snapshot.Manifest, *tempDB, []snapshot.BlobInfo, error) { // All remote paths use the hashed key derived from the human ID. remoteKey := snapshot.RemoteSnapshotKey(snapshotID) @@ -142,10 +166,11 @@ func (v *Vaultik) loadVerificationData(snapshotID string, opts *VerifyOptions, r log.Info("Manifest loaded", "manifest_blob_count", manifest.BlobCount, - "manifest_total_size", humanize.Bytes(uint64(manifest.TotalCompressedSize))) + "manifest_total_size", ubytes(manifest.TotalCompressedSize)) if !opts.JSON { - v.stdoutf("Manifest loaded: %d blobs (%s)\n", manifest.BlobCount, humanize.Bytes(uint64(manifest.TotalCompressedSize))) + v.stdoutf("Manifest loaded: %d blobs (%s)\n", + manifest.BlobCount, ubytes(manifest.TotalCompressedSize)) v.stdoutf("Downloading and decrypting database...\n") } @@ -162,7 +187,7 @@ func (v *Vaultik) loadVerificationData(snapshotID string, opts *VerifyOptions, r defer func() { _ = dbReader.Close() }() - tdb, err := v.decryptAndLoadDatabase(dbReader, v.Config.AgeSecretKey) + tdb, err := v.decryptAndLoadDatabase(dbReader) if err != nil { return nil, nil, nil, v.deepVerifyFailure(result, opts, fmt.Sprintf("failed to decrypt database: %v", err), @@ -185,17 +210,26 @@ func (v *Vaultik) loadVerificationData(snapshotID string, opts *VerifyOptions, r log.Info("Database loaded", "db_blob_count", len(dbBlobs), - "db_total_size", humanize.Bytes(uint64(dbTotalSize))) + "db_total_size", ubytes(dbTotalSize)) if !opts.JSON { - v.stdoutf("Database loaded: %d blobs (%s)\n", len(dbBlobs), humanize.Bytes(uint64(dbTotalSize))) + v.stdoutf("Database loaded: %d blobs (%s)\n", + len(dbBlobs), ubytes(dbTotalSize)) } return manifest, tdb, dbBlobs, nil } -// runVerificationSteps executes manifest verification, blob existence check, and deep content verification -func (v *Vaultik) runVerificationSteps(manifest *snapshot.Manifest, dbBlobs []snapshot.BlobInfo, tdb *tempDB, opts *VerifyOptions, result *VerifyResult, totalSize int64) error { +// runVerificationSteps executes manifest verification, blob existence +// check, and deep content verification. +func (v *Vaultik) runVerificationSteps( + manifest *snapshot.Manifest, + dbBlobs []snapshot.BlobInfo, + tdb *tempDB, + opts *VerifyOptions, + result *VerifyResult, + totalSize int64, +) error { if !opts.JSON { v.stdoutf("Verifying manifest against database...\n") } @@ -217,7 +251,8 @@ func (v *Vaultik) runVerificationSteps(manifest *snapshot.Manifest, dbBlobs []sn if !opts.JSON { v.stdoutf("All blobs exist.\n") - v.stdoutf("Downloading and verifying blob contents (%d blobs, %s)...\n", len(dbBlobs), humanize.Bytes(uint64(totalSize))) + v.stdoutf("Downloading and verifying blob contents (%d blobs, %s)...\n", + len(dbBlobs), ubytes(totalSize)) } err = v.performDeepVerificationFromDB(dbBlobs, tdb.DB, opts) @@ -242,8 +277,9 @@ func (t *tempDB) Close() error { return err } -// decryptAndLoadDatabase decrypts and loads the binary SQLite database from the encrypted stream -func (v *Vaultik) decryptAndLoadDatabase(reader io.ReadCloser, secretKey string) (*tempDB, error) { +// decryptAndLoadDatabase decrypts and loads the binary SQLite database +// from the encrypted stream. +func (v *Vaultik) decryptAndLoadDatabase(reader io.ReadCloser) (*tempDB, error) { // Get decryptor decryptor, err := v.GetDecryptor() if err != nil { @@ -284,7 +320,7 @@ func (v *Vaultik) decryptAndLoadDatabase(reader io.ReadCloser, secretKey string) _ = tempFile.Close() - log.Info("Database decompressed", "size", humanize.Bytes(uint64(written))) + log.Info("Database decompressed", "size", ubytes(written)) // Open the database db, err := sql.Open("sqlite", tempPath) @@ -337,22 +373,25 @@ func (v *Vaultik) verifyBlob(blobInfo snapshot.BlobInfo, db *sql.DB) error { return err } - if err := v.verifyBlobFinalIntegrity(decompressor, blobHasher, blobInfo.Hash); err != nil { + err = v.verifyBlobFinalIntegrity(decompressor, blobHasher, blobInfo.Hash) + if err != nil { return err } log.Info("Blob verified", "hash", blobInfo.Hash[:16]+"...", "chunks", chunkCount, - "size", humanize.Bytes(uint64(blobInfo.CompressedSize)), + "size", ubytes(blobInfo.CompressedSize), ) return nil } -// verifyBlobChunks queries blob chunks from the database and verifies each chunk's hash -// against the decompressed blob stream -func (v *Vaultik) verifyBlobChunks(db *sql.DB, blobHash string, decompressor io.Reader) (int, error) { +// verifyBlobChunks queries blob chunks from the database and verifies +// each chunk's hash against the decompressed blob stream. +func (v *Vaultik) verifyBlobChunks( + db *sql.DB, blobHash string, decompressor io.Reader, +) (int, error) { query := ` SELECT bc.chunk_hash, bc.offset, bc.length FROM blob_chunks bc @@ -387,7 +426,8 @@ func (v *Vaultik) verifyBlobChunks(db *sql.DB, blobHash string, decompressor io. // Verify chunk ordering if offset <= lastOffset { - return 0, fmt.Errorf("chunks out of order: offset %d after %d", offset, lastOffset) + return 0, fmt.Errorf("%w: offset %d after %d", + errChunksOutOfOrder, offset, lastOffset) } lastOffset = offset @@ -396,7 +436,9 @@ func (v *Vaultik) verifyBlobChunks(db *sql.DB, blobHash string, decompressor io. if offset > totalRead { // Skip to the correct offset skipBytes := offset - totalRead - if _, err := io.CopyN(io.Discard, decompressor, skipBytes); err != nil { + + _, err = io.CopyN(io.Discard, decompressor, skipBytes) + if err != nil { return 0, fmt.Errorf("failed to skip to offset %d: %w", offset, err) } @@ -405,7 +447,9 @@ func (v *Vaultik) verifyBlobChunks(db *sql.DB, blobHash string, decompressor io. // Read chunk data chunkData := make([]byte, length) - if _, err := io.ReadFull(decompressor, chunkData); err != nil { + + _, err = io.ReadFull(decompressor, chunkData) + if err != nil { return 0, fmt.Errorf("failed to read chunk at offset %d: %w", offset, err) } @@ -417,45 +461,52 @@ func (v *Vaultik) verifyBlobChunks(db *sql.DB, blobHash string, decompressor io. calculatedHash := hex.EncodeToString(hasher.Sum(nil)) if calculatedHash != chunkHash { - return 0, fmt.Errorf("chunk hash mismatch at offset %d: calculated %s, expected %s", - offset, calculatedHash, chunkHash) + return 0, fmt.Errorf("%w at offset %d: calculated %s, expected %s", + errChunkHashMismatch, offset, calculatedHash, chunkHash) } chunkCount++ } - if err := rows.Err(); err != nil { + err = rows.Err() + if err != nil { return 0, fmt.Errorf("error iterating blob chunks: %w", err) } return chunkCount, nil } -// verifyBlobFinalIntegrity checks that no trailing data exists in the decompressed stream -// and that the encrypted blob hash matches the expected value -func (v *Vaultik) verifyBlobFinalIntegrity(decompressor io.Reader, blobHasher hash.Hash, expectedHash string) error { - // Verify no remaining data in blob - if chunk list is accurate, blob should be fully consumed +// verifyBlobFinalIntegrity checks that no trailing data exists in the +// decompressed stream and that the encrypted blob hash matches the +// expected value. +func (v *Vaultik) verifyBlobFinalIntegrity( + decompressor io.Reader, blobHasher hash.Hash, expectedHash string, +) error { + // Verify no remaining data in blob - if the chunk list is accurate, + // the blob should be fully consumed. remaining, err := io.Copy(io.Discard, decompressor) if err != nil { return fmt.Errorf("failed to check for remaining blob data: %w", err) } if remaining > 0 { - return fmt.Errorf("blob has %d unexpected trailing bytes not covered by chunk list", remaining) + return fmt.Errorf("%w: %d bytes", errTrailingBlobData, remaining) } // Verify blob hash matches the encrypted data we downloaded calculatedBlobHash := hex.EncodeToString(blobHasher.Sum(nil)) if calculatedBlobHash != expectedHash { - return fmt.Errorf("blob hash mismatch: calculated %s, expected %s", - calculatedBlobHash, expectedHash) + return fmt.Errorf("%w: calculated %s, expected %s", + errBlobHashMismatch, calculatedBlobHash, expectedHash) } return nil } // getBlobsFromDatabase gets all blobs for the snapshot from the database -func (v *Vaultik) getBlobsFromDatabase(snapshotID string, db *sql.DB) ([]snapshot.BlobInfo, error) { +func (v *Vaultik) getBlobsFromDatabase( + snapshotID string, db *sql.DB, +) ([]snapshot.BlobInfo, error) { query := ` SELECT b.blob_hash, b.compressed_size FROM snapshot_blobs sb @@ -490,15 +541,19 @@ func (v *Vaultik) getBlobsFromDatabase(snapshotID string, db *sql.DB) ([]snapsho }) } - if err := rows.Err(); err != nil { + err = rows.Err() + if err != nil { return nil, fmt.Errorf("error iterating blobs: %w", err) } return blobs, nil } -// verifyManifestAgainstDatabase verifies the manifest matches the authoritative database -func (v *Vaultik) verifyManifestAgainstDatabase(manifest *snapshot.Manifest, dbBlobs []snapshot.BlobInfo) error { +// verifyManifestAgainstDatabase verifies the manifest matches the +// authoritative database. +func (v *Vaultik) verifyManifestAgainstDatabase( + manifest *snapshot.Manifest, dbBlobs []snapshot.BlobInfo, +) error { log.Info("Verifying manifest against database") // Build map of database blobs @@ -526,12 +581,13 @@ func (v *Vaultik) verifyManifestAgainstDatabase(manifest *snapshot.Manifest, dbB for hash, manifestSize := range manifestBlobMap { dbSize, exists := dbBlobMap[hash] if !exists { - return fmt.Errorf("manifest contains blob %s not in database", hash) + return fmt.Errorf("%w: %s", errManifestExtraBlob, hash) } if dbSize != manifestSize { - return fmt.Errorf("blob %s size mismatch: database has %d bytes, manifest has %d bytes", - hash, dbSize, manifestSize) + return fmt.Errorf( + "%w: blob %s: database has %d bytes, manifest has %d bytes", + errBlobSizeMismatch, hash, dbSize, manifestSize) } } @@ -559,16 +615,18 @@ func (v *Vaultik) verifyBlobExistenceFromDB(blobs []snapshot.BlobInfo) error { // Verify size matches if stat.Size != blob.CompressedSize { - return fmt.Errorf("blob %s size mismatch: S3 has %d bytes, database has %d bytes", - blob.Hash, stat.Size, blob.CompressedSize) + return fmt.Errorf( + "%w: blob %s: S3 has %d bytes, database has %d bytes", + errBlobSizeMismatch, blob.Hash, stat.Size, blob.CompressedSize) } // Progress update every 100 blobs - if (i+1)%100 == 0 || i == len(blobs)-1 { + if (i+1)%progressLogEvery == 0 || i == len(blobs)-1 { log.Info("Blob existence check progress", "checked", i+1, "total", len(blobs), - "percent", fmt.Sprintf("%.1f%%", float64(i+1)/float64(len(blobs))*100), + "percent", fmt.Sprintf("%.1f%%", + float64(i+1)/float64(len(blobs))*percentScale), ) } } @@ -578,8 +636,11 @@ func (v *Vaultik) verifyBlobExistenceFromDB(blobs []snapshot.BlobInfo) error { return nil } -// performDeepVerificationFromDB downloads and verifies the content of each blob using database as source -func (v *Vaultik) performDeepVerificationFromDB(blobs []snapshot.BlobInfo, db *sql.DB, opts *VerifyOptions) error { +// performDeepVerificationFromDB downloads and verifies the content of +// each blob using the database as source. +func (v *Vaultik) performDeepVerificationFromDB( + blobs []snapshot.BlobInfo, db *sql.DB, opts *VerifyOptions, +) error { // Calculate total bytes for ETA var totalBytesExpected int64 for _, b := range blobs { @@ -588,7 +649,7 @@ func (v *Vaultik) performDeepVerificationFromDB(blobs []snapshot.BlobInfo, db *s log.Info("Starting deep verification - downloading and verifying all blobs", "blob_count", len(blobs), - "total_size", humanize.Bytes(uint64(totalBytesExpected)), + "total_size", ubytes(totalBytesExpected), ) startTime := time.Now() @@ -622,9 +683,9 @@ func (v *Vaultik) performDeepVerificationFromDB(blobs []snapshot.BlobInfo, db *s "blobs_total", len(blobs), "blobs_remaining", remaining, "bytes_done", bytesProcessed, - "bytes_done_human", humanize.Bytes(uint64(bytesProcessed)), + "bytes_done_human", ubytes(bytesProcessed), "bytes_total", totalBytesExpected, - "bytes_total_human", humanize.Bytes(uint64(totalBytesExpected)), + "bytes_total_human", ubytes(totalBytesExpected), "elapsed", elapsed.Round(time.Second), "eta", eta.Round(time.Second), ) @@ -632,8 +693,8 @@ func (v *Vaultik) performDeepVerificationFromDB(blobs []snapshot.BlobInfo, db *s if !opts.JSON { v.stdoutf(" Verified %d/%d blobs (%d remaining) - %s/%s - elapsed %s, eta %s\n", i+1, len(blobs), remaining, - humanize.Bytes(uint64(bytesProcessed)), - humanize.Bytes(uint64(totalBytesExpected)), + ubytes(bytesProcessed), + ubytes(totalBytesExpected), elapsed.Round(time.Second), eta.Round(time.Second)) } @@ -643,7 +704,7 @@ func (v *Vaultik) performDeepVerificationFromDB(blobs []snapshot.BlobInfo, db *s log.Info("✓ Deep verification completed successfully", "blobs_verified", len(blobs), "total_bytes", bytesProcessed, - "total_bytes_human", humanize.Bytes(uint64(bytesProcessed)), + "total_bytes_human", ubytes(bytesProcessed), "duration", totalElapsed.Round(time.Second), ) diff --git a/internal/vaultik/verify_test.go b/internal/vaultik/verify_test.go index c1fc1c3..75aab5f 100644 --- a/internal/vaultik/verify_test.go +++ b/internal/vaultik/verify_test.go @@ -18,6 +18,8 @@ import ( // bytes when streaming through age decryption and zstd decompression. // This validates the verification path: hash encrypted blob -> decrypt -> decompress. func TestTeeReaderWithDecryption(t *testing.T) { + t.Parallel() + // Test data - use random data that doesn't compress well (5MB) testData := make([]byte, 5*1024*1024) _, err := rand.Read(testData) @@ -26,7 +28,8 @@ func TestTeeReaderWithDecryption(t *testing.T) { // Compress the data var compressedBuf bytes.Buffer - compressor, err := zstd.NewWriter(&compressedBuf, zstd.WithEncoderLevel(zstd.SpeedDefault)) + compressor, err := zstd.NewWriter(&compressedBuf, + zstd.WithEncoderLevel(zstd.SpeedDefault)) require.NoError(t, err) _, err = compressor.Write(testData) require.NoError(t, err) @@ -34,8 +37,10 @@ func TestTeeReaderWithDecryption(t *testing.T) { require.NoError(t, err) // Encrypt the compressed data - testRecipient := "age1cplgrwj77ta54dnmydvvmzn64ltk83ankxl5sww04mrtmu62kv3s89gmvv" - testSecretKey := "AGE-SECRET-KEY-1C77PYNTHXSHNNC6EYR2W52UWYXACXA5JT00J9CCW9986M3XY87PSGP89AQ" + testRecipient := "age1cplgrwj77ta54dnmydvvmzn64ltk83ankxl5sww04mrt" + + "mu62kv3s89gmvv" + testSecretKey := "AGE-SECRET-KEY-1C77PYNTHXSHNNC6EYR2W52UWYXACXA5J" + + "T00J9CCW9986M3XY87PSGP89AQ" encryptor, err := crypto.NewEncryptor([]string{testRecipient}) require.NoError(t, err) -- 2.49.1