diff --git a/.golangci.yml b/.golangci.yml index 34a8e31..26b1610 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,5 +1,9 @@ version: "2" +# Config schema uses the golangci-lint v2 layout (settings live under +# linters.settings, not top-level linters-settings) so that the +# thresholds below are actually applied by golangci-lint >= v2. + run: timeout: 5m modules-download-mode: readonly @@ -14,19 +18,17 @@ linters: - wsl # Deprecated, replaced by wsl_v5 - wrapcheck # Too verbose for internal packages - varnamelen # Short names like db, id are idiomatic Go - -linters-settings: - lll: - line-length: 88 - funlen: - lines: 80 - statements: 50 - cyclop: - max-complexity: 15 - dupl: - threshold: 100 + settings: + lll: + line-length: 88 + funlen: + lines: 80 + statements: 50 + cyclop: + max-complexity: 15 + dupl: + threshold: 100 issues: - exclude-use-default: false max-issues-per-linter: 0 max-same-issues: 0 diff --git a/Dockerfile b/Dockerfile index 72ea9b2..cebd9d9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ # Lint stage -# golangci/golangci-lint:v2.11.3-alpine, 2026-03-17 -FROM golangci/golangci-lint:v2.11.3-alpine@sha256:b1c3de5862ad0a95b4e45a993b0f00415835d687e4f12c845c7493b86c13414e AS lint +# golangci/golangci-lint:v2.12.2-alpine, 2026-08-07 +FROM golangci/golangci-lint:v2.12.2-alpine@sha256:91b27804074a0bacea298707f016911e60cf0cdbc6c7bf5ccacb5f0606d18d60 AS lint RUN apk add --no-cache make build-base diff --git a/Makefile b/Makefile index 2431df1..626882e 100644 --- a/Makefile +++ b/Makefile @@ -59,7 +59,7 @@ clean: # Install dependencies. deps: go mod download - go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest + go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.12.2 # Run tests with coverage. test-coverage: diff --git a/TODO.md b/TODO.md index 64d9501..17d8258 100644 --- a/TODO.md +++ b/TODO.md @@ -14,18 +14,16 @@ pre-1.0 # Next Step -Continue the lint remediation (issue #61): 1,077 findings remain after -the mechanical chunk. Next chunk candidates: `paralleltest` (137), -`funcorder` (68), `testpackage` (34), `lll` (79) — still mostly -mechanical — before the judgment-heavy linters (`revive` 142, `err113` -96, `mnd` 93, `gosec` 78, `goconst` 55, `cyclop` 52). +Reconcile the uncommitted ARCHITECTURE.md edits on main: finish and +commit, or revert. # Completed Steps -- 2026-08-07: Lint remediation chunk 1 (issue #61): `wsl_v5` (1050), - `nlreturn` (378), and `noinlineerr` (373) all fixed to zero via a new - `make lint-fix` autofix entrypoint plus hand-fixes; total findings - 2,990 → 1,077. Full test suite green. +- 2026-08-07: Updated golangci-lint to v2.12.2 everywhere it is pinned + (`Dockerfile` lint stage, `Makefile` deps target), replaced + `.golangci.yml` with the canonical config (v2 schema, `default: all`), + and remediated all lint findings it surfaced (issue #61): + behavior-preserving fixes across every package, `make check` green. - 2026-08-07: Added the standard `.golangci.yml` and `.editorconfig` (issue #59); lint findings under the new config are tracked in issue #61. `script/bootstrap` now installs sqlite3 (needed by tests). @@ -51,8 +49,6 @@ mechanical — before the judgment-heavy linters (`revive` 142, `err113` # Future Steps -- Reconcile the uncommitted ARCHITECTURE.md edits on main: finish and - commit, or revert. - Review stale local branches (add-godoc-to-cli-package, feature/pluggable-storage-backend) and merge or delete them. - Define remaining scope for a first tagged release and cut v0.1.0. diff --git a/cmd/vaultik/main.go b/cmd/vaultik/main.go index 8cfc6dd..c9cbacc 100644 --- a/cmd/vaultik/main.go +++ b/cmd/vaultik/main.go @@ -1,3 +1,4 @@ +// Package main is the vaultik command-line entry point. package main import ( @@ -11,7 +12,7 @@ import ( func main() { // CPU profiling: set VAULTIK_CPUPROFILE=/path/to/cpu.prof if cpuProfile := os.Getenv("VAULTIK_CPUPROFILE"); cpuProfile != "" { - f, err := os.Create(cpuProfile) + f, err := os.Create(cpuProfile) //nolint:gosec // G304: operator-set path if err != nil { panic("could not create CPU profile: " + err.Error()) } @@ -28,7 +29,7 @@ func main() { // Memory profiling: set VAULTIK_MEMPROFILE=/path/to/mem.prof if memProfile := os.Getenv("VAULTIK_MEMPROFILE"); memProfile != "" { defer func() { - f, err := os.Create(memProfile) + f, err := os.Create(memProfile) //nolint:gosec // G304: operator-set path if err != nil { panic("could not create memory profile: " + err.Error()) } @@ -43,5 +44,5 @@ func main() { }() } - cli.CLIEntry() + cli.Entry() } diff --git a/internal/blob/errors.go b/internal/blob/errors.go index ceaa0ad..7817486 100644 --- a/internal/blob/errors.go +++ b/internal/blob/errors.go @@ -2,5 +2,18 @@ package blob import "errors" -// ErrBlobSizeLimitExceeded is returned when adding a chunk would exceed the blob size limit +// ErrBlobSizeLimitExceeded is returned when adding a chunk would exceed +// the blob size limit. var ErrBlobSizeLimitExceeded = errors.New("adding chunk would exceed blob size limit") + +// ErrNoRecipients is returned when a Packer is created without any age +// recipients; blobs must always be encrypted. +var ErrNoRecipients = errors.New("recipients are required - blobs must be encrypted") + +// ErrInvalidMaxBlobSize is returned when the configured maximum blob size +// is zero or negative. +var ErrInvalidMaxBlobSize = errors.New("max blob size must be positive") + +// ErrNoFilesystem is returned when a Packer is created without a filesystem +// for temporary files. +var ErrNoFilesystem = errors.New("filesystem is required") diff --git a/internal/blob/packer.go b/internal/blob/packer.go index c31c28e..cfbb081 100644 --- a/internal/blob/packer.go +++ b/internal/blob/packer.go @@ -32,21 +32,28 @@ import ( "sneak.berlin/go/vaultik/internal/types" ) -// BlobHandler is a callback function invoked when a blob is finalized and ready for upload. -// The handler receives a BlobWithReader containing the blob metadata and a reader for -// the compressed and encrypted blob content. The handler is responsible for uploading -// the blob to storage and cleaning up any temporary files. -type BlobHandler func(blob *BlobWithReader) error +// Handler is a callback function invoked when a blob is finalized and +// ready for upload. The handler receives a WithReader containing the +// blob metadata and a reader for the compressed and encrypted blob content. +// The handler is responsible for uploading the blob to storage and cleaning +// up any temporary files. +type Handler func(blob *WithReader) error // PackerConfig holds configuration for creating a Packer. // All fields except BlobHandler are required. type PackerConfig struct { - MaxBlobSize int64 // Maximum size of a blob before forcing finalization - CompressionLevel int // Zstd compression level (1-19, higher = better compression) - Recipients []string // Age recipients for encryption - Repositories *database.Repositories // Database repositories for tracking blob metadata - BlobHandler BlobHandler // Optional callback when blob is ready for upload - Fs afero.Fs // Filesystem for temporary files + // MaxBlobSize is the maximum size of a blob before forcing finalization. + MaxBlobSize int64 + // CompressionLevel is the zstd level (1-19, higher = better compression). + CompressionLevel int + // Recipients holds the age recipients for encryption. + Recipients []string + // Repositories provides database access for tracking blob metadata. + Repositories *database.Repositories + // BlobHandler is an optional callback when a blob is ready for upload. + BlobHandler Handler + // Fs is the filesystem used for temporary files. + Fs afero.Fs } // PendingChunk represents a chunk waiting to be inserted into the database. @@ -62,7 +69,7 @@ type Packer struct { maxBlobSize int64 compressionLevel int recipients []string // Age recipients for encryption - blobHandler BlobHandler // Called when blob is ready + blobHandler Handler // Called when blob is ready repos *database.Repositories // For creating blob records fs afero.Fs // Filesystem for temporary files @@ -109,21 +116,21 @@ type FinishedBlob struct { ID string Hash string Data []byte // Compressed data - Chunks []*BlobChunkRef + Chunks []*ChunkPosition CreatedTS time.Time Uncompressed int64 Compressed int64 } -// BlobChunkRef represents a chunk's position within a blob -type BlobChunkRef struct { +// ChunkPosition represents a chunk's position within a blob +type ChunkPosition struct { ChunkHash string Offset int64 Length int64 } -// BlobWithReader wraps a FinishedBlob with its data reader -type BlobWithReader struct { +// WithReader wraps a FinishedBlob with its data reader +type WithReader struct { *FinishedBlob Reader io.ReadSeeker @@ -136,15 +143,15 @@ type BlobWithReader struct { // Returns an error if required configuration fields are missing or invalid. func NewPacker(cfg PackerConfig) (*Packer, error) { if len(cfg.Recipients) == 0 { - return nil, errors.New("recipients are required - blobs must be encrypted") + return nil, ErrNoRecipients } if cfg.MaxBlobSize <= 0 { - return nil, errors.New("max blob size must be positive") + return nil, ErrInvalidMaxBlobSize } if cfg.Fs == nil { - return nil, errors.New("filesystem is required") + return nil, ErrNoFilesystem } return &Packer{ @@ -162,7 +169,7 @@ func NewPacker(cfg PackerConfig) (*Packer, error) { // The handler is responsible for uploading the blob to storage. // If no handler is set, finalized blobs are stored in memory and can be // retrieved with GetFinishedBlobs(). -func (p *Packer) SetBlobHandler(handler BlobHandler) { +func (p *Packer) SetBlobHandler(handler Handler) { p.mu.Lock() defer p.mu.Unlock() @@ -184,13 +191,13 @@ func (p *Packer) AddPendingChunk(hash string, size int64) { // In this case, the caller should finalize the current blob and retry. // The chunk data is written immediately and can be garbage collected after this call. // Thread-safe. -func (p *Packer) AddChunk(chunk *ChunkRef) error { +func (p *Packer) AddChunk(ctx context.Context, chunk *ChunkRef) error { p.mu.Lock() defer p.mu.Unlock() // Initialize new blob if needed if p.currentBlob == nil { - err := p.startNewBlob() + err := p.startNewBlob(ctx) if err != nil { return fmt.Errorf("starting new blob: %w", err) } @@ -222,12 +229,12 @@ func (p *Packer) AddChunk(chunk *ChunkRef) error { // This should be called after all chunks have been added to ensure no data is lost. // If a BlobHandler is set, it will be called with the finalized blob. // Thread-safe. -func (p *Packer) Flush() error { +func (p *Packer) Flush(ctx context.Context) error { p.mu.Lock() defer p.mu.Unlock() if p.currentBlob != nil && len(p.currentBlob.chunks) > 0 { - err := p.finalizeCurrentBlob() + err := p.finalizeCurrentBlob(ctx) if err != nil { return fmt.Errorf("finalizing blob: %w", err) } @@ -242,7 +249,7 @@ func (p *Packer) Flush() error { // BlobHandler (if set) or stored internally. // Caller must handle retrying any chunk that triggered size limit exceeded. // Not thread-safe - caller must hold the lock. -func (p *Packer) FinalizeBlob() error { +func (p *Packer) FinalizeBlob(ctx context.Context) error { p.mu.Lock() defer p.mu.Unlock() @@ -250,7 +257,7 @@ func (p *Packer) FinalizeBlob() error { return nil } - return p.finalizeCurrentBlob() + return p.finalizeCurrentBlob(ctx) } // GetFinishedBlobs returns all completed blobs and clears the internal list. @@ -267,8 +274,33 @@ func (p *Packer) GetFinishedBlobs() []*FinishedBlob { return blobs } +// PackChunks is a convenience method to pack multiple chunks at once. +func (p *Packer) PackChunks(ctx context.Context, chunks []*ChunkRef) error { + for _, chunk := range chunks { + err := p.AddChunk(ctx, chunk) + if errors.Is(err, ErrBlobSizeLimitExceeded) { + // Finalize current blob and retry + err = p.FinalizeBlob(ctx) + if err != nil { + return fmt.Errorf("finalizing blob before retry: %w", err) + } + + // Retry the chunk + err = p.AddChunk(ctx, chunk) + if err != nil { + return fmt.Errorf( + "adding chunk %s after finalize: %w", chunk.Hash, err) + } + } else if err != nil { + return fmt.Errorf("adding chunk %s: %w", chunk.Hash, err) + } + } + + return p.Flush(ctx) +} + // startNewBlob initializes a new blob (must be called with lock held) -func (p *Packer) startNewBlob() error { +func (p *Packer) startNewBlob(ctx context.Context) error { // Generate UUID for the blob blobID := uuid.New().String() @@ -280,8 +312,9 @@ func (p *Packer) startNewBlob() error { } blob := &database.Blob{ - ID: blobIDTyped, - Hash: types.BlobHash("temp-placeholder-" + blobID), // Temporary placeholder until finalized + ID: blobIDTyped, + // Temporary placeholder hash until finalized. + Hash: types.BlobHash("temp-placeholder-" + blobID), CreatedTS: time.Now().UTC(), FinishedTS: nil, UncompressedSize: 0, @@ -289,9 +322,11 @@ func (p *Packer) startNewBlob() error { UploadedTS: nil, } - err = p.repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error { - return p.repos.Blobs.Create(ctx, tx, blob) - }) + err = p.repos.WithTx( + ctx, + func(txCtx context.Context, tx *sql.Tx) error { + return p.repos.Blobs.Create(txCtx, tx, blob) + }) if err != nil { return fmt.Errorf("creating blob record: %w", err) } @@ -322,16 +357,19 @@ func (p *Packer) startNewBlob() error { size: 0, } - log.Debug("Created new blob container", "blob_id", blobID, "temp_file", tempFile.Name()) + log.Debug("Created new blob container", + "blob_id", blobID, "temp_file", tempFile.Name()) return nil } -// addChunkToCurrentBlob adds a chunk to the current blob (must be called with lock held) +// addChunkToCurrentBlob adds a chunk to the current blob (must be called +// with lock held). func (p *Packer) addChunkToCurrentBlob(chunk *ChunkRef) error { // Skip if chunk already in current blob if p.currentBlob.chunkSet[chunk.Hash] { - log.Debug("Skipping duplicate chunk already in current blob", "chunk_hash", chunk.Hash) + log.Debug("Skipping duplicate chunk already in current blob", + "chunk_hash", chunk.Hash) return nil } @@ -374,7 +412,7 @@ func (p *Packer) addChunkToCurrentBlob(chunk *ChunkRef) error { } // finalizeCurrentBlob completes the current blob (must be called with lock held) -func (p *Packer) finalizeCurrentBlob() error { +func (p *Packer) finalizeCurrentBlob(ctx context.Context) error { if p.currentBlob == nil { return nil } @@ -389,7 +427,7 @@ func (p *Packer) finalizeCurrentBlob() error { chunksToInsert := p.pendingChunks p.pendingChunks = nil - err = p.commitBlobToDatabase(blobHash, finalSize, chunksToInsert) + err = p.commitBlobToDatabase(ctx, blobHash, finalSize, chunksToInsert) if err != nil { return err } @@ -418,7 +456,8 @@ func (p *Packer) finalizeCurrentBlob() error { return p.deliverFinishedBlob(finished, insertedChunkHashes) } -// closeBlobWriter closes the writer, syncs to disk, and returns the blob hash and final size +// closeBlobWriter closes the writer, syncs to disk, and returns the blob +// hash and final size. func (p *Packer) closeBlobWriter() (string, int64, error) { err := p.currentBlob.writer.Close() if err != nil { @@ -453,11 +492,11 @@ func (p *Packer) closeBlobWriter() (string, int64, error) { return hex.EncodeToString(finalHash), finalSize, nil } -// buildChunkRefs creates BlobChunkRef entries from the current blob's chunks -func (p *Packer) buildChunkRefs() []*BlobChunkRef { - refs := make([]*BlobChunkRef, 0, len(p.currentBlob.chunks)) +// buildChunkRefs creates ChunkPosition entries from the current blob's chunks +func (p *Packer) buildChunkRefs() []*ChunkPosition { + refs := make([]*ChunkPosition, 0, len(p.currentBlob.chunks)) for _, chunk := range p.currentBlob.chunks { - refs = append(refs, &BlobChunkRef{ + refs = append(refs, &ChunkPosition{ ChunkHash: chunk.Hash, Offset: chunk.Offset, Length: chunk.Size, }) } @@ -466,7 +505,10 @@ func (p *Packer) buildChunkRefs() []*BlobChunkRef { } // commitBlobToDatabase inserts pending chunks, blob_chunks, and updates the blob record -func (p *Packer) commitBlobToDatabase(blobHash string, finalSize int64, chunksToInsert []PendingChunk) error { +func (p *Packer) commitBlobToDatabase( + ctx context.Context, + blobHash string, finalSize int64, chunksToInsert []PendingChunk, +) error { if p.repos == nil { return nil } @@ -478,30 +520,12 @@ func (p *Packer) commitBlobToDatabase(blobHash string, finalSize int64, chunksTo return fmt.Errorf("parsing blob ID: %w", parseErr) } - err := p.repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error { - for _, chunk := range chunksToInsert { - dbChunk := &database.Chunk{ChunkHash: types.ChunkHash(chunk.Hash), Size: chunk.Size} - - err := p.repos.Chunks.Create(ctx, tx, dbChunk) - if err != nil { - return fmt.Errorf("creating chunk: %w", err) - } - } - - for _, chunk := range p.currentBlob.chunks { - blobChunk := &database.BlobChunk{ - BlobID: blobIDTyped, ChunkHash: types.ChunkHash(chunk.Hash), - Offset: chunk.Offset, Length: chunk.Size, - } - - err := p.repos.BlobChunks.Create(ctx, tx, blobChunk) - if err != nil { - return fmt.Errorf("creating blob_chunk: %w", err) - } - } - - return p.repos.Blobs.UpdateFinished(ctx, tx, p.currentBlob.id, blobHash, p.currentBlob.size, finalSize) - }) + err := p.repos.WithTx( + ctx, + func(txCtx context.Context, tx *sql.Tx) error { + return p.insertBlobRecords(txCtx, tx, blobIDTyped, blobHash, + finalSize, chunksToInsert) + }) if err != nil { p.cleanupTempFile() @@ -509,13 +533,53 @@ func (p *Packer) commitBlobToDatabase(blobHash string, finalSize int64, chunksTo } log.Debug("Committed blob transaction", - "chunks_inserted", len(chunksToInsert), "blob_chunks_inserted", len(p.currentBlob.chunks)) + "chunks_inserted", len(chunksToInsert), + "blob_chunks_inserted", len(p.currentBlob.chunks)) return nil } +// insertBlobRecords inserts pending chunks and blob_chunk rows, then marks +// the blob finished, all within the supplied transaction. +func (p *Packer) insertBlobRecords( + ctx context.Context, + tx *sql.Tx, + blobIDTyped types.BlobID, + blobHash string, + finalSize int64, + chunksToInsert []PendingChunk, +) error { + for _, chunk := range chunksToInsert { + dbChunk := &database.Chunk{ + ChunkHash: types.ChunkHash(chunk.Hash), Size: chunk.Size, + } + + err := p.repos.Chunks.Create(ctx, tx, dbChunk) + if err != nil { + return fmt.Errorf("creating chunk: %w", err) + } + } + + for _, chunk := range p.currentBlob.chunks { + blobChunk := &database.BlobChunk{ + BlobID: blobIDTyped, ChunkHash: types.ChunkHash(chunk.Hash), + Offset: chunk.Offset, Length: chunk.Size, + } + + err := p.repos.BlobChunks.Create(ctx, tx, blobChunk) + if err != nil { + return fmt.Errorf("creating blob_chunk: %w", err) + } + } + + return p.repos.Blobs.UpdateFinished(ctx, tx, p.currentBlob.id, blobHash, + p.currentBlob.size, finalSize) +} + // deliverFinishedBlob passes the blob to the handler or stores it internally -func (p *Packer) deliverFinishedBlob(finished *FinishedBlob, insertedChunkHashes []string) error { +func (p *Packer) deliverFinishedBlob( + finished *FinishedBlob, insertedChunkHashes []string, +) error { if p.blobHandler != nil { _, err := p.currentBlob.tempFile.Seek(0, io.SeekStart) if err != nil { @@ -524,7 +588,7 @@ func (p *Packer) deliverFinishedBlob(finished *FinishedBlob, insertedChunkHashes return fmt.Errorf("seeking for handler: %w", err) } - blobWithReader := &BlobWithReader{ + blobWithReader := &WithReader{ FinishedBlob: finished, Reader: p.currentBlob.tempFile, TempFile: p.currentBlob.tempFile, @@ -576,26 +640,3 @@ func (p *Packer) cleanupTempFile() { _ = p.fs.Remove(name) } } - -// PackChunks is a convenience method to pack multiple chunks at once -func (p *Packer) PackChunks(chunks []*ChunkRef) error { - for _, chunk := range chunks { - err := p.AddChunk(chunk) - if errors.Is(err, ErrBlobSizeLimitExceeded) { - // Finalize current blob and retry - err := p.FinalizeBlob() - if err != nil { - return fmt.Errorf("finalizing blob before retry: %w", err) - } - // Retry the chunk - err = p.AddChunk(chunk) - if err != nil { - return fmt.Errorf("adding chunk %s after finalize: %w", chunk.Hash, err) - } - } else if err != nil { - return fmt.Errorf("adding chunk %s: %w", chunk.Hash, err) - } - } - - return p.Flush() -} diff --git a/internal/blob/packer_test.go b/internal/blob/packer_test.go index 0cba6e4..09994fa 100644 --- a/internal/blob/packer_test.go +++ b/internal/blob/packer_test.go @@ -1,4 +1,4 @@ -package blob +package blob_test import ( "bytes" @@ -13,6 +13,7 @@ import ( "filippo.io/age" "github.com/klauspost/compress/zstd" "github.com/spf13/afero" + "sneak.berlin/go/vaultik/internal/blob" "sneak.berlin/go/vaultik/internal/database" "sneak.berlin/go/vaultik/internal/log" "sneak.berlin/go/vaultik/internal/types" @@ -20,399 +21,298 @@ import ( const ( // Test key from test/insecure-integration-test.key - testPrivateKey = "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5" - testPublicKey = "age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05gl0sjq9q9wjg" + testPrivateKey = "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7A" + + "PHXA2QS2NJA5" + testPublicKey = "age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05gl0sjq9q9wjg" + + defaultMaxBlobSize = 10 * 1024 * 1024 // 10MB + testChunkSize = 1000 + testChunkCount = 10 ) -func TestPacker(t *testing.T) { - // Initialize logger for tests - log.Initialize(log.Config{}) +// parseTestIdentity parses the fixed test age identity. +func parseTestIdentity(t *testing.T) *age.X25519Identity { + t.Helper() - // Parse test identity identity, err := age.ParseX25519Identity(testPrivateKey) if err != nil { t.Fatalf("failed to parse test identity: %v", err) } - t.Run("single chunk creates single blob", func(t *testing.T) { - // Create test database - db, err := database.NewTestDB() - if err != nil { - t.Fatalf("failed to create test db: %v", err) - } - defer func() { _ = db.Close() }() - - repos := database.NewRepositories(db) - - cfg := PackerConfig{ - MaxBlobSize: 10 * 1024 * 1024, // 10MB - CompressionLevel: 3, - Recipients: []string{testPublicKey}, - Repositories: repos, - Fs: afero.NewMemMapFs(), - } - - packer, err := NewPacker(cfg) - if err != nil { - t.Fatalf("failed to create packer: %v", err) - } - - // Create a chunk - data := []byte("Hello, World!") - hash := sha256.Sum256(data) - hashStr := hex.EncodeToString(hash[:]) - - // Create chunk in database first - dbChunk := &database.Chunk{ - ChunkHash: types.ChunkHash(hashStr), - Size: int64(len(data)), - } - - err = repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error { - return repos.Chunks.Create(ctx, tx, dbChunk) - }) - if err != nil { - t.Fatalf("failed to create chunk in db: %v", err) - } - - chunk := &ChunkRef{ - Hash: hashStr, - Data: data, - } - - // Add chunk - err = packer.AddChunk(chunk) - if err != nil { - t.Fatalf("failed to add chunk: %v", err) - } - - // Flush - err = packer.Flush() - if err != nil { - t.Fatalf("failed to flush: %v", err) - } - - // Get finished blobs - blobs := packer.GetFinishedBlobs() - if len(blobs) != 1 { - t.Fatalf("expected 1 blob, got %d", len(blobs)) - } - - blob := blobs[0] - if len(blob.Chunks) != 1 { - t.Errorf("expected 1 chunk in blob, got %d", len(blob.Chunks)) - } - - // Note: Very small data may not compress well - t.Logf("Compression: %d -> %d bytes", blob.Uncompressed, blob.Compressed) - - // Decrypt the blob data - decrypted, err := age.Decrypt(bytes.NewReader(blob.Data), identity) - if err != nil { - t.Fatalf("failed to decrypt blob: %v", err) - } - - // Decompress the decrypted data - reader, err := zstd.NewReader(decrypted) - if err != nil { - t.Fatalf("failed to create decompressor: %v", err) - } - defer reader.Close() - - var decompressed bytes.Buffer - - _, err = io.Copy(&decompressed, reader) - if err != nil { - t.Fatalf("failed to decompress: %v", err) - } - - if !bytes.Equal(decompressed.Bytes(), data) { - t.Error("decompressed data doesn't match original") - } - }) - - t.Run("multiple chunks packed together", func(t *testing.T) { - // Create test database - db, err := database.NewTestDB() - if err != nil { - t.Fatalf("failed to create test db: %v", err) - } - defer func() { _ = db.Close() }() - - repos := database.NewRepositories(db) - - cfg := PackerConfig{ - MaxBlobSize: 10 * 1024 * 1024, // 10MB - CompressionLevel: 3, - Recipients: []string{testPublicKey}, - Repositories: repos, - Fs: afero.NewMemMapFs(), - } - - packer, err := NewPacker(cfg) - if err != nil { - t.Fatalf("failed to create packer: %v", err) - } - - // Create multiple small chunks - chunks := make([]*ChunkRef, 10) - - for i := range 10 { - data := bytes.Repeat([]byte{byte(i)}, 1000) - hash := sha256.Sum256(data) - hashStr := hex.EncodeToString(hash[:]) - - // Create chunk in database first - dbChunk := &database.Chunk{ - ChunkHash: types.ChunkHash(hashStr), - Size: int64(len(data)), - } - - err = repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error { - return repos.Chunks.Create(ctx, tx, dbChunk) - }) - if err != nil { - t.Fatalf("failed to create chunk in db: %v", err) - } - - chunks[i] = &ChunkRef{ - Hash: hashStr, - Data: data, - } - } - - // Add all chunks - for _, chunk := range chunks { - err := packer.AddChunk(chunk) - if err != nil { - t.Fatalf("failed to add chunk: %v", err) - } - } - - // Flush - err = packer.Flush() - if err != nil { - t.Fatalf("failed to flush: %v", err) - } - - // Should have one blob with all chunks - blobs := packer.GetFinishedBlobs() - if len(blobs) != 1 { - t.Fatalf("expected 1 blob, got %d", len(blobs)) - } - - if len(blobs[0].Chunks) != 10 { - t.Errorf("expected 10 chunks in blob, got %d", len(blobs[0].Chunks)) - } - - // Verify offsets are correct - expectedOffset := int64(0) - for i, chunkRef := range blobs[0].Chunks { - if chunkRef.Offset != expectedOffset { - t.Errorf("chunk %d: expected offset %d, got %d", i, expectedOffset, chunkRef.Offset) - } - - if chunkRef.Length != 1000 { - t.Errorf("chunk %d: expected length 1000, got %d", i, chunkRef.Length) - } - - expectedOffset += chunkRef.Length - } - }) - - t.Run("blob size limit enforced", func(t *testing.T) { - // Create test database - db, err := database.NewTestDB() - if err != nil { - t.Fatalf("failed to create test db: %v", err) - } - defer func() { _ = db.Close() }() - - repos := database.NewRepositories(db) - - // Small blob size limit to force multiple blobs - cfg := PackerConfig{ - MaxBlobSize: 5000, // 5KB max - CompressionLevel: 3, - Recipients: []string{testPublicKey}, - Repositories: repos, - Fs: afero.NewMemMapFs(), - } - - packer, err := NewPacker(cfg) - if err != nil { - t.Fatalf("failed to create packer: %v", err) - } - - // Create chunks that will exceed the limit - chunks := make([]*ChunkRef, 10) - - for i := range 10 { - data := bytes.Repeat([]byte{byte(i)}, 1000) // 1KB each - hash := sha256.Sum256(data) - hashStr := hex.EncodeToString(hash[:]) - - // Create chunk in database first - dbChunk := &database.Chunk{ - ChunkHash: types.ChunkHash(hashStr), - Size: int64(len(data)), - } - - err = repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error { - return repos.Chunks.Create(ctx, tx, dbChunk) - }) - if err != nil { - t.Fatalf("failed to create chunk in db: %v", err) - } - - chunks[i] = &ChunkRef{ - Hash: hashStr, - Data: data, - } - } - - blobCount := 0 - - // Add chunks and handle size limit errors - for _, chunk := range chunks { - err := packer.AddChunk(chunk) - if errors.Is(err, ErrBlobSizeLimitExceeded) { - // Finalize current blob - err := packer.FinalizeBlob() - if err != nil { - t.Fatalf("failed to finalize blob: %v", err) - } - - blobCount++ - // Retry adding the chunk - err = packer.AddChunk(chunk) - if err != nil { - t.Fatalf("failed to add chunk after finalize: %v", err) - } - } else if err != nil { - t.Fatalf("failed to add chunk: %v", err) - } - } - - // Flush remaining - err = packer.Flush() - if err != nil { - t.Fatalf("failed to flush: %v", err) - } - - // Get all blobs - blobs := packer.GetFinishedBlobs() - totalBlobs := blobCount + len(blobs) - - // Should have multiple blobs due to size limit - if totalBlobs < 2 { - t.Errorf("expected multiple blobs due to size limit, got %d", totalBlobs) - } - - // Verify each blob respects size limit (approximately) - for _, blob := range blobs { - if blob.Compressed > 6000 { // Allow some overhead - t.Errorf("blob size %d exceeds limit", blob.Compressed) - } - } - }) - - t.Run("with encryption", func(t *testing.T) { - // Create test database - db, err := database.NewTestDB() - if err != nil { - t.Fatalf("failed to create test db: %v", err) - } - defer func() { _ = db.Close() }() - - repos := database.NewRepositories(db) - - // Generate test identity (using the one from parent test) - cfg := PackerConfig{ - MaxBlobSize: 10 * 1024 * 1024, // 10MB - CompressionLevel: 3, - Recipients: []string{testPublicKey}, - Repositories: repos, - Fs: afero.NewMemMapFs(), - } - - packer, err := NewPacker(cfg) - if err != nil { - t.Fatalf("failed to create packer: %v", err) - } - - // Create test data - data := bytes.Repeat([]byte("Test data for encryption!"), 100) - hash := sha256.Sum256(data) - hashStr := hex.EncodeToString(hash[:]) - - // Create chunk in database first - dbChunk := &database.Chunk{ - ChunkHash: types.ChunkHash(hashStr), - Size: int64(len(data)), - } - - err = repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error { - return repos.Chunks.Create(ctx, tx, dbChunk) - }) - if err != nil { - t.Fatalf("failed to create chunk in db: %v", err) - } - - chunk := &ChunkRef{ - Hash: hashStr, - Data: data, - } - - // Add chunk and flush - err = packer.AddChunk(chunk) - if err != nil { - t.Fatalf("failed to add chunk: %v", err) - } - - err = packer.Flush() - if err != nil { - t.Fatalf("failed to flush: %v", err) - } - - // Get blob - blobs := packer.GetFinishedBlobs() - if len(blobs) != 1 { - t.Fatalf("expected 1 blob, got %d", len(blobs)) - } - - blob := blobs[0] - - // Decrypt the blob - decrypted, err := age.Decrypt(bytes.NewReader(blob.Data), identity) - if err != nil { - t.Fatalf("failed to decrypt blob: %v", err) - } - - var decryptedData bytes.Buffer - - _, err = decryptedData.ReadFrom(decrypted) - if err != nil { - t.Fatalf("failed to read decrypted data: %v", err) - } - - // Decompress - reader, err := zstd.NewReader(&decryptedData) - if err != nil { - t.Fatalf("failed to create decompressor: %v", err) - } - defer reader.Close() - - var decompressed bytes.Buffer - - _, err = decompressed.ReadFrom(reader) - if err != nil { - t.Fatalf("failed to decompress: %v", err) - } - - // Verify data - if !bytes.Equal(decompressed.Bytes(), data) { - t.Error("decrypted and decompressed data doesn't match original") - } - }) + return identity +} + +// newTestPacker creates a test database and a Packer backed by it. +func newTestPacker( + t *testing.T, maxBlobSize int64, +) (*database.Repositories, *blob.Packer) { + t.Helper() + + db, err := database.NewTestDB() + if err != nil { + t.Fatalf("failed to create test db: %v", err) + } + + t.Cleanup(func() { _ = db.Close() }) + + repos := database.NewRepositories(db) + + packer, err := blob.NewPacker(blob.PackerConfig{ + MaxBlobSize: maxBlobSize, + CompressionLevel: 3, + Recipients: []string{testPublicKey}, + Repositories: repos, + Fs: afero.NewMemMapFs(), + }) + if err != nil { + t.Fatalf("failed to create packer: %v", err) + } + + return repos, packer +} + +// makeChunk creates a ChunkRef for data and registers the chunk in the +// database. +func makeChunk( + t *testing.T, repos *database.Repositories, data []byte, +) *blob.ChunkRef { + t.Helper() + + hash := sha256.Sum256(data) + hashStr := hex.EncodeToString(hash[:]) + + dbChunk := &database.Chunk{ + ChunkHash: types.ChunkHash(hashStr), + Size: int64(len(data)), + } + + err := repos.WithTx( + context.Background(), + func(ctx context.Context, tx *sql.Tx) error { + return repos.Chunks.Create(ctx, tx, dbChunk) + }) + if err != nil { + t.Fatalf("failed to create chunk in db: %v", err) + } + + return &blob.ChunkRef{ + Hash: hashStr, + Data: data, + } +} + +// decryptAndDecompress reverses the blob pipeline: age decrypt, then zstd +// decompress. +func decryptAndDecompress( + t *testing.T, blobData []byte, identity *age.X25519Identity, +) []byte { + t.Helper() + + decrypted, err := age.Decrypt(bytes.NewReader(blobData), identity) + if err != nil { + t.Fatalf("failed to decrypt blob: %v", err) + } + + reader, err := zstd.NewReader(decrypted) + if err != nil { + t.Fatalf("failed to create decompressor: %v", err) + } + defer reader.Close() + + var decompressed bytes.Buffer + + _, err = io.Copy(&decompressed, reader) + if err != nil { + t.Fatalf("failed to decompress: %v", err) + } + + return decompressed.Bytes() +} + +func TestPackerSingleChunk(t *testing.T) { + log.Initialize(log.Config{}) + t.Parallel() + + identity := parseTestIdentity(t) + repos, packer := newTestPacker(t, defaultMaxBlobSize) + ctx := context.Background() + + data := []byte("Hello, World!") + chunk := makeChunk(t, repos, data) + + err := packer.AddChunk(ctx, chunk) + if err != nil { + t.Fatalf("failed to add chunk: %v", err) + } + + err = packer.Flush(ctx) + if err != nil { + t.Fatalf("failed to flush: %v", err) + } + + blobs := packer.GetFinishedBlobs() + if len(blobs) != 1 { + t.Fatalf("expected 1 blob, got %d", len(blobs)) + } + + finished := blobs[0] + if len(finished.Chunks) != 1 { + t.Errorf("expected 1 chunk in blob, got %d", len(finished.Chunks)) + } + + // Note: Very small data may not compress well + t.Logf("Compression: %d -> %d bytes", + finished.Uncompressed, finished.Compressed) + + decompressed := decryptAndDecompress(t, finished.Data, identity) + if !bytes.Equal(decompressed, data) { + t.Error("decompressed data doesn't match original") + } +} + +func TestPackerMultipleChunks(t *testing.T) { + log.Initialize(log.Config{}) + t.Parallel() + + repos, packer := newTestPacker(t, defaultMaxBlobSize) + ctx := context.Background() + + chunks := make([]*blob.ChunkRef, testChunkCount) + for i := range testChunkCount { + data := bytes.Repeat([]byte{byte(i)}, testChunkSize) + chunks[i] = makeChunk(t, repos, data) + } + + for _, chunk := range chunks { + err := packer.AddChunk(ctx, chunk) + if err != nil { + t.Fatalf("failed to add chunk: %v", err) + } + } + + err := packer.Flush(ctx) + if err != nil { + t.Fatalf("failed to flush: %v", err) + } + + blobs := packer.GetFinishedBlobs() + if len(blobs) != 1 { + t.Fatalf("expected 1 blob, got %d", len(blobs)) + } + + if len(blobs[0].Chunks) != testChunkCount { + t.Errorf("expected %d chunks in blob, got %d", + testChunkCount, len(blobs[0].Chunks)) + } + + // Verify offsets are correct + expectedOffset := int64(0) + + for i, chunkRef := range blobs[0].Chunks { + if chunkRef.Offset != expectedOffset { + t.Errorf("chunk %d: expected offset %d, got %d", + i, expectedOffset, chunkRef.Offset) + } + + if chunkRef.Length != testChunkSize { + t.Errorf("chunk %d: expected length %d, got %d", + i, testChunkSize, chunkRef.Length) + } + + expectedOffset += chunkRef.Length + } +} + +func TestPackerSizeLimit(t *testing.T) { + log.Initialize(log.Config{}) + t.Parallel() + + const ( + maxBlobSize = 5000 // 5KB max, forces multiple blobs + maxBlobawoOverhead = 6000 // allow some overhead over the limit + ) + + repos, packer := newTestPacker(t, maxBlobSize) + ctx := context.Background() + + chunks := make([]*blob.ChunkRef, testChunkCount) + for i := range testChunkCount { + data := bytes.Repeat([]byte{byte(i)}, testChunkSize) // 1KB each + chunks[i] = makeChunk(t, repos, data) + } + + blobCount := 0 + + // Add chunks and handle size limit errors + for _, chunk := range chunks { + err := packer.AddChunk(ctx, chunk) + if errors.Is(err, blob.ErrBlobSizeLimitExceeded) { + // Finalize current blob + err = packer.FinalizeBlob(ctx) + if err != nil { + t.Fatalf("failed to finalize blob: %v", err) + } + + blobCount++ + + // Retry adding the chunk + err = packer.AddChunk(ctx, chunk) + if err != nil { + t.Fatalf("failed to add chunk after finalize: %v", err) + } + } else if err != nil { + t.Fatalf("failed to add chunk: %v", err) + } + } + + err := packer.Flush(ctx) + if err != nil { + t.Fatalf("failed to flush: %v", err) + } + + blobs := packer.GetFinishedBlobs() + + totalBlobs := blobCount + len(blobs) + if totalBlobs < 2 { + t.Errorf("expected multiple blobs due to size limit, got %d", totalBlobs) + } + + // Verify each blob respects size limit (approximately) + for _, finished := range blobs { + if finished.Compressed > maxBlobawoOverhead { + t.Errorf("blob size %d exceeds limit", finished.Compressed) + } + } +} + +func TestPackerEncryption(t *testing.T) { + log.Initialize(log.Config{}) + t.Parallel() + + identity := parseTestIdentity(t) + repos, packer := newTestPacker(t, defaultMaxBlobSize) + ctx := context.Background() + + data := bytes.Repeat([]byte("Test data for encryption!"), 100) + chunk := makeChunk(t, repos, data) + + err := packer.AddChunk(ctx, chunk) + if err != nil { + t.Fatalf("failed to add chunk: %v", err) + } + + err = packer.Flush(ctx) + if err != nil { + t.Fatalf("failed to flush: %v", err) + } + + blobs := packer.GetFinishedBlobs() + if len(blobs) != 1 { + t.Fatalf("expected 1 blob, got %d", len(blobs)) + } + + decompressed := decryptAndDecompress(t, blobs[0].Data, identity) + if !bytes.Equal(decompressed, data) { + t.Error("decrypted and decompressed data doesn't match original") + } } diff --git a/internal/blobgen/compress.go b/internal/blobgen/compress.go index 576b054..8eefc77 100644 --- a/internal/blobgen/compress.go +++ b/internal/blobgen/compress.go @@ -1,3 +1,6 @@ +// Package blobgen implements the blob data pipeline: streaming zstd +// compression, age encryption, and SHA256 content hashing for blob +// creation, plus the matching decrypt/decompress/verify reader. package blobgen import ( @@ -16,7 +19,9 @@ type CompressResult struct { } // CompressData compresses and encrypts data, returning the result with hash -func CompressData(data []byte, compressionLevel int, recipients []string) (*CompressResult, error) { +func CompressData( + data []byte, compressionLevel int, recipients []string, +) (*CompressResult, error) { var buf bytes.Buffer // Create writer @@ -47,8 +52,11 @@ func CompressData(data []byte, compressionLevel int, recipients []string) (*Comp }, nil } -// CompressStream compresses and encrypts from reader to writer, returning hash -func CompressStream(dst io.Writer, src io.Reader, compressionLevel int, recipients []string) (written int64, hash string, err error) { +// CompressStream compresses and encrypts from reader to writer, returning +// the number of uncompressed bytes written and the content hash. +func CompressStream( + dst io.Writer, src io.Reader, compressionLevel int, recipients []string, +) (int64, string, error) { // Create writer w, err := NewWriter(dst, compressionLevel, recipients) if err != nil { diff --git a/internal/blobgen/compress_test.go b/internal/blobgen/compress_test.go index 37c41e9..b019f11 100644 --- a/internal/blobgen/compress_test.go +++ b/internal/blobgen/compress_test.go @@ -1,4 +1,4 @@ -package blobgen +package blobgen_test import ( "bytes" @@ -8,6 +8,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "sneak.berlin/go/vaultik/internal/blobgen" ) // testRecipient is a static age recipient for tests. @@ -19,11 +20,14 @@ const testRecipient = "age1cplgrwj77ta54dnmydvvmzn64ltk83ankxl5sww04mrtmu62kv3s8 // the explicit Close() on the happy path combined with defer Close() would // cause a double close. func TestCompressStreamNoDoubleClose(t *testing.T) { + t.Parallel() + input := []byte("regression test data for issue #28 double-close fix") var buf bytes.Buffer - written, hash, err := CompressStream(&buf, bytes.NewReader(input), 3, []string{testRecipient}) + written, hash, err := blobgen.CompressStream( + &buf, bytes.NewReader(input), 3, []string{testRecipient}) require.NoError(t, err, "CompressStream should not return an error") assert.Positive(t, written, "expected bytes written > 0") assert.NotEmpty(t, hash, "expected non-empty hash") @@ -33,13 +37,16 @@ func TestCompressStreamNoDoubleClose(t *testing.T) { // TestCompressStreamLargeInput exercises CompressStream with a larger payload // to ensure no double-close issues surface under heavier I/O. func TestCompressStreamLargeInput(t *testing.T) { + t.Parallel() + data := make([]byte, 512*1024) // 512 KB _, err := rand.Read(data) require.NoError(t, err) var buf bytes.Buffer - written, hash, err := CompressStream(&buf, bytes.NewReader(data), 3, []string{testRecipient}) + written, hash, err := blobgen.CompressStream( + &buf, bytes.NewReader(data), 3, []string{testRecipient}) require.NoError(t, err) assert.Positive(t, written) assert.NotEmpty(t, hash) @@ -48,9 +55,12 @@ func TestCompressStreamLargeInput(t *testing.T) { // TestCompressStreamEmptyInput verifies CompressStream handles empty input // without double-close issues. func TestCompressStreamEmptyInput(t *testing.T) { + t.Parallel() + var buf bytes.Buffer - _, hash, err := CompressStream(&buf, strings.NewReader(""), 3, []string{testRecipient}) + _, hash, err := blobgen.CompressStream( + &buf, strings.NewReader(""), 3, []string{testRecipient}) require.NoError(t, err) assert.NotEmpty(t, hash) } @@ -58,8 +68,11 @@ func TestCompressStreamEmptyInput(t *testing.T) { // TestCompressDataNoDoubleClose mirrors the stream test for CompressData, // ensuring the explicit Close + error-path Close pattern is also safe. func TestCompressDataNoDoubleClose(t *testing.T) { + t.Parallel() + input := []byte("CompressData regression test for double-close") - result, err := CompressData(input, 3, []string{testRecipient}) + + result, err := blobgen.CompressData(input, 3, []string{testRecipient}) require.NoError(t, err) assert.Positive(t, result.CompressedSize) assert.Equal(t, result.UncompressedSize, int64(len(input))) diff --git a/internal/blobgen/reader.go b/internal/blobgen/reader.go index 6d7f2f8..1a7dc11 100644 --- a/internal/blobgen/reader.go +++ b/internal/blobgen/reader.go @@ -50,8 +50,8 @@ func NewReader(r io.Reader, identity age.Identity) (*Reader, error) { } // Read implements io.Reader -func (r *Reader) Read(p []byte) (n int, err error) { - n, err = r.teeReader.Read(p) +func (r *Reader) Read(p []byte) (int, error) { + n, err := r.teeReader.Read(p) r.bytesRead += int64(n) return n, err diff --git a/internal/blobgen/writer.go b/internal/blobgen/writer.go index 08a13fb..8305046 100644 --- a/internal/blobgen/writer.go +++ b/internal/blobgen/writer.go @@ -2,6 +2,7 @@ package blobgen import ( "crypto/sha256" + "errors" "fmt" "hash" "io" @@ -11,6 +12,21 @@ import ( "github.com/klauspost/compress/zstd" ) +// Zstd compression level bounds accepted by NewWriter. +const ( + minCompressionLevel = 1 + maxCompressionLevel = 19 +) + +// reservedCompressionCPUs is how many CPUs are left free of zstd +// compression work for I/O and hashing. +const reservedCompressionCPUs = 2 + +// ErrInvalidCompressionLevel is returned when the zstd compression level +// is outside the accepted 1-19 range. +var ErrInvalidCompressionLevel = errors.New( + "invalid compression level: must be between 1 and 19") + // Writer wraps compression and encryption with SHA256 hashing. // Data flows: input -> tee(hasher, compressor -> encryptor -> destination) // The hash is computed on the uncompressed input for deterministic content-addressing. @@ -23,9 +39,12 @@ type Writer struct { bytesWritten int64 } -// NewWriter creates a new Writer that compresses, encrypts, and hashes data. -// The hash is computed on the uncompressed input for deterministic content-addressing. -func NewWriter(w io.Writer, compressionLevel int, recipients []string) (*Writer, error) { +// NewWriter creates a new Writer that compresses, encrypts, and hashes +// data. The hash is computed on the uncompressed input for deterministic +// content-addressing. +func NewWriter( + w io.Writer, compressionLevel int, recipients []string, +) (*Writer, error) { // Validate compression level err := validateCompressionLevel(compressionLevel) if err != nil { @@ -54,7 +73,7 @@ func NewWriter(w io.Writer, compressionLevel int, recipients []string) (*Writer, } // Calculate compression concurrency: CPUs - 2, minimum 1 - concurrency := max(runtime.NumCPU()-2, 1) + concurrency := max(runtime.NumCPU()-reservedCompressionCPUs, 1) // Create compression writer with encryption as destination compressor, err := zstd.NewWriter(encWriter, @@ -80,8 +99,8 @@ func NewWriter(w io.Writer, compressionLevel int, recipients []string) (*Writer, } // Write implements io.Writer -func (w *Writer) Write(p []byte) (n int, err error) { - n, err = w.teeWriter.Write(p) +func (w *Writer) Write(p []byte) (int, error) { + n, err := w.teeWriter.Write(p) w.bytesWritten += int64(n) return n, err @@ -124,9 +143,10 @@ func (w *Writer) BytesWritten() int64 { func validateCompressionLevel(level int) error { // Zstd compression levels: 1-19 (default is 3) - // SpeedFastest = 1, SpeedDefault = 3, SpeedBetterCompression = 7, SpeedBestCompression = 11 - if level < 1 || level > 19 { - return fmt.Errorf("invalid compression level %d: must be between 1 and 19", level) + // SpeedFastest = 1, SpeedDefault = 3, SpeedBetterCompression = 7, + // SpeedBestCompression = 11 + if level < minCompressionLevel || level > maxCompressionLevel { + return fmt.Errorf("%w: got %d", ErrInvalidCompressionLevel, level) } return nil diff --git a/internal/blobgen/writer_test.go b/internal/blobgen/writer_test.go index f72e8cf..e218f52 100644 --- a/internal/blobgen/writer_test.go +++ b/internal/blobgen/writer_test.go @@ -1,4 +1,4 @@ -package blobgen +package blobgen_test import ( "bytes" @@ -9,12 +9,15 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "sneak.berlin/go/vaultik/internal/blobgen" ) // TestWriterHashIsDoubleHash verifies that Writer.Sum256() returns // the double hash SHA256(SHA256(plaintext)) for security. // Double hashing prevents attackers from confirming existence of known content. func TestWriterHashIsDoubleHash(t *testing.T) { + t.Parallel() + // Test data - random data that doesn't compress well testData := make([]byte, 1024*1024) // 1MB _, err := rand.Read(testData) @@ -27,7 +30,7 @@ func TestWriterHashIsDoubleHash(t *testing.T) { var encryptedBuf bytes.Buffer // Create blobgen writer - writer, err := NewWriter(&encryptedBuf, 3, []string{testRecipient}) + writer, err := blobgen.NewWriter(&encryptedBuf, 3, []string{testRecipient}) require.NoError(t, err) // Write test data @@ -67,6 +70,8 @@ func TestWriterHashIsDoubleHash(t *testing.T) { // TestWriterDeterministicHash verifies that the same input always produces // the same hash, even with non-deterministic encryption. func TestWriterDeterministicHash(t *testing.T) { + t.Parallel() + // Test data testData := []byte("Hello, World! This is test data for deterministic hashing.") @@ -76,13 +81,13 @@ func TestWriterDeterministicHash(t *testing.T) { // Create two writers and verify they produce the same hash var buf1, buf2 bytes.Buffer - writer1, err := NewWriter(&buf1, 3, []string{testRecipient}) + writer1, err := blobgen.NewWriter(&buf1, 3, []string{testRecipient}) require.NoError(t, err) _, err = writer1.Write(testData) require.NoError(t, err) require.NoError(t, writer1.Close()) - writer2, err := NewWriter(&buf2, 3, []string{testRecipient}) + writer2, err := blobgen.NewWriter(&buf2, 3, []string{testRecipient}) require.NoError(t, err) _, err = writer2.Write(testData) require.NoError(t, err) diff --git a/internal/chunker/chunker.go b/internal/chunker/chunker.go index c20065e..8fedb30 100644 --- a/internal/chunker/chunker.go +++ b/internal/chunker/chunker.go @@ -1,3 +1,6 @@ +// Package chunker splits input data into content-defined chunks using the +// FastCDC algorithm so that identical data sequences produce identical +// chunks regardless of their position in the file. package chunker import ( @@ -9,9 +12,10 @@ import ( "os" ) -// Chunk represents a single chunk of data produced by the content-defined chunking algorithm. -// Each chunk is identified by its SHA256 hash and contains the raw data along with -// its position and size information from the original file. +// Chunk represents a single chunk of data produced by the content-defined +// chunking algorithm. Each chunk is identified by its SHA256 hash and +// contains the raw data along with its position and size information from +// the original file. type Chunk struct { Hash string // Content hash of the chunk Data []byte // Chunk data @@ -29,6 +33,10 @@ type Chunker struct { maxChunkSize int } +// chunkSizeSpread is the FastCDC-recommended factor between the average +// chunk size and the minimum (avg/spread) and maximum (avg*spread) sizes. +const chunkSizeSpread = 4 + // NewChunker creates a new chunker with the specified average chunk size. // The actual chunk sizes will vary between avgChunkSize/4 and avgChunkSize*4 // as recommended by the FastCDC algorithm. Typical values for avgChunkSize @@ -37,17 +45,19 @@ func NewChunker(avgChunkSize int64) *Chunker { // FastCDC recommends min = avg/4 and max = avg*4 return &Chunker{ avgChunkSize: int(avgChunkSize), - minChunkSize: int(avgChunkSize / 4), - maxChunkSize: int(avgChunkSize * 4), + minChunkSize: int(avgChunkSize / chunkSizeSpread), + maxChunkSize: int(avgChunkSize * chunkSizeSpread), } } -// ChunkReader splits the reader into content-defined chunks and returns all chunks at once. -// This method loads all chunk data into memory, so it should only be used for -// reasonably sized inputs. For large files or streams, use ChunkReaderStreaming instead. +// ChunkReader splits the reader into content-defined chunks and returns all +// chunks at once. This method loads all chunk data into memory, so it should +// only be used for reasonably sized inputs. For large files or streams, use +// ChunkReaderStreaming instead. // Returns an error if chunking fails or if reading from the input fails. func (c *Chunker) ChunkReader(r io.Reader) ([]Chunk, error) { - chunker := AcquireReusableChunker(r, c.minChunkSize, c.avgChunkSize, c.maxChunkSize) + chunker := AcquireReusableChunker( + r, c.minChunkSize, c.avgChunkSize, c.maxChunkSize) defer chunker.Release() var chunks []Chunk @@ -86,21 +96,26 @@ func (c *Chunker) ChunkReader(r io.Reader) ([]Chunk, error) { // ChunkCallback is a function called for each chunk as it's processed. // The callback receives a Chunk containing the hash, data, offset, and size. -// If the callback returns an error, chunk processing stops and the error is propagated. +// If the callback returns an error, chunk processing stops and the error is +// propagated. type ChunkCallback func(chunk Chunk) error -// ChunkReaderStreaming splits the reader into chunks and calls the callback for each chunk. -// This is the preferred method for processing large files or streams as it doesn't -// accumulate all chunks in memory. The callback is invoked for each chunk as it's -// produced, allowing for streaming processing and immediate storage or transmission. -// Returns the SHA256 hash of the entire file content and an error if chunking fails, -// reading fails, or if the callback returns an error. -func (c *Chunker) ChunkReaderStreaming(r io.Reader, callback ChunkCallback) (string, error) { +// ChunkReaderStreaming splits the reader into chunks and calls the callback +// for each chunk. This is the preferred method for processing large files or +// streams as it doesn't accumulate all chunks in memory. The callback is +// invoked for each chunk as it's produced, allowing for streaming processing +// and immediate storage or transmission. +// Returns the SHA256 hash of the entire file content and an error if +// chunking fails, reading fails, or if the callback returns an error. +func (c *Chunker) ChunkReaderStreaming( + r io.Reader, callback ChunkCallback, +) (string, error) { // Create a tee reader to calculate full file hash while chunking fileHasher := sha256.New() teeReader := io.TeeReader(r, fileHasher) - chunker := AcquireReusableChunker(teeReader, c.minChunkSize, c.avgChunkSize, c.maxChunkSize) + chunker := AcquireReusableChunker( + teeReader, c.minChunkSize, c.avgChunkSize, c.maxChunkSize) defer chunker.Release() offset := int64(0) @@ -118,9 +133,10 @@ func (c *Chunker) ChunkReaderStreaming(r io.Reader, callback ChunkCallback) (str // Calculate chunk hash hash := sha256.Sum256(chunk.Data) - // Pass the data directly - caller must process it before we call Next() again - // (chunker reuses its internal buffer, but since we process synchronously - // and completely before continuing, no copy is needed) + // Pass the data directly - caller must process it before we call + // Next() again (chunker reuses its internal buffer, but since we + // process synchronously and completely before continuing, no copy + // is needed) err = callback(Chunk{ Hash: hex.EncodeToString(hash[:]), Data: chunk.Data, @@ -143,7 +159,7 @@ func (c *Chunker) ChunkReaderStreaming(r io.Reader, callback ChunkCallback) (str // For large files, consider using ChunkReaderStreaming with a file handle instead. // Returns an error if the file cannot be opened or if chunking fails. func (c *Chunker) ChunkFile(path string) ([]Chunk, error) { - file, err := os.Open(path) + file, err := os.Open(path) //nolint:gosec // G304: path is caller-supplied by design if err != nil { return nil, fmt.Errorf("opening file: %w", err) } diff --git a/internal/chunker/chunker_isolated_test.go b/internal/chunker/chunker_isolated_test.go index b5f86bf..5df44ae 100644 --- a/internal/chunker/chunker_isolated_test.go +++ b/internal/chunker/chunker_isolated_test.go @@ -1,11 +1,15 @@ -package chunker +package chunker_test import ( "bytes" "testing" + + "sneak.berlin/go/vaultik/internal/chunker" ) func TestChunkerExpectedChunkCount(t *testing.T) { + t.Parallel() + tests := []struct { name string fileSize int @@ -38,16 +42,19 @@ func TestChunkerExpectedChunkCount(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - chunker := NewChunker(tt.avgChunkSize) + t.Parallel() + + c := chunker.NewChunker(tt.avgChunkSize) // Create data with some variation to trigger chunk boundaries data := make([]byte, tt.fileSize) for i := range data { // Use a pattern that should create boundaries + //nolint:gosec // G115: intentional byte truncation data[i] = byte((i * 17) ^ (i >> 5)) } - chunks, err := chunker.ChunkReader(bytes.NewReader(data)) + chunks, err := c.ChunkReader(bytes.NewReader(data)) if err != nil { t.Fatalf("chunking failed: %v", err) } diff --git a/internal/chunker/chunker_test.go b/internal/chunker/chunker_test.go index 20ba3a3..f44919f 100644 --- a/internal/chunker/chunker_test.go +++ b/internal/chunker/chunker_test.go @@ -1,108 +1,120 @@ -package chunker +package chunker_test import ( "bytes" "crypto/rand" "testing" + + "sneak.berlin/go/vaultik/internal/chunker" ) -func TestChunker(t *testing.T) { - t.Run("small file produces single chunk", func(t *testing.T) { - chunker := NewChunker(1024 * 1024) // 1MB average - data := bytes.Repeat([]byte("hello"), 100) // 500 bytes +func TestChunkerSmallFileSingleChunk(t *testing.T) { + t.Parallel() - chunks, err := chunker.ChunkReader(bytes.NewReader(data)) - if err != nil { - t.Fatalf("chunking failed: %v", err) + c := chunker.NewChunker(1024 * 1024) // 1MB average + data := bytes.Repeat([]byte("hello"), 100) // 500 bytes + + chunks, err := c.ChunkReader(bytes.NewReader(data)) + if err != nil { + t.Fatalf("chunking failed: %v", err) + } + + if len(chunks) != 1 { + t.Errorf("expected 1 chunk, got %d", len(chunks)) + } + + if chunks[0].Size != int64(len(data)) { + t.Errorf("expected chunk size %d, got %d", len(data), chunks[0].Size) + } +} + +func TestChunkerLargeFileMultipleChunks(t *testing.T) { + t.Parallel() + + c := chunker.NewChunker(256 * 1024) // 256KB average chunk size + + // Generate 2MB of random data + data := make([]byte, 2*1024*1024) + + _, err := rand.Read(data) + if err != nil { + t.Fatalf("failed to generate random data: %v", err) + } + + chunks, err := c.ChunkReader(bytes.NewReader(data)) + if err != nil { + t.Fatalf("chunking failed: %v", err) + } + + // Should produce multiple chunks - with FastCDC we expect around 8 + // chunks for 2MB with 256KB average + if len(chunks) < 4 || len(chunks) > 16 { + t.Errorf("expected 4-16 chunks, got %d", len(chunks)) + } + + // Verify chunks reconstruct original data + var reconstructed []byte + for _, chunk := range chunks { + reconstructed = append(reconstructed, chunk.Data...) + } + + if !bytes.Equal(data, reconstructed) { + t.Error("reconstructed data doesn't match original") + } + + // Verify offsets + var expectedOffset int64 + + for i, chunk := range chunks { + if chunk.Offset != expectedOffset { + t.Errorf("chunk %d: expected offset %d, got %d", + i, expectedOffset, chunk.Offset) } - if len(chunks) != 1 { - t.Errorf("expected 1 chunk, got %d", len(chunks)) + expectedOffset += chunk.Size + } +} + +func TestChunkerDeterministic(t *testing.T) { + t.Parallel() + + chunker1 := chunker.NewChunker(256 * 1024) + chunker2 := chunker.NewChunker(256 * 1024) + + // Use deterministic data + data := bytes.Repeat([]byte("abcdefghijklmnopqrstuvwxyz"), 20000) // ~520KB + + chunks1, err := chunker1.ChunkReader(bytes.NewReader(data)) + if err != nil { + t.Fatalf("chunking failed: %v", err) + } + + chunks2, err := chunker2.ChunkReader(bytes.NewReader(data)) + if err != nil { + t.Fatalf("chunking failed: %v", err) + } + + // Should produce same chunks + if len(chunks1) != len(chunks2) { + t.Fatalf("different number of chunks: %d vs %d", + len(chunks1), len(chunks2)) + } + + for i := range chunks1 { + if chunks1[i].Hash != chunks2[i].Hash { + t.Errorf("chunk %d: different hashes", i) } - if chunks[0].Size != int64(len(data)) { - t.Errorf("expected chunk size %d, got %d", len(data), chunks[0].Size) + if chunks1[i].Size != chunks2[i].Size { + t.Errorf("chunk %d: different sizes", i) } - }) - - t.Run("large file produces multiple chunks", func(t *testing.T) { - chunker := NewChunker(256 * 1024) // 256KB average chunk size - - // Generate 2MB of random data - data := make([]byte, 2*1024*1024) - - _, err := rand.Read(data) - if err != nil { - t.Fatalf("failed to generate random data: %v", err) - } - - chunks, err := chunker.ChunkReader(bytes.NewReader(data)) - if err != nil { - t.Fatalf("chunking failed: %v", err) - } - - // Should produce multiple chunks - with FastCDC we expect around 8 chunks for 2MB with 256KB average - if len(chunks) < 4 || len(chunks) > 16 { - t.Errorf("expected 4-16 chunks, got %d", len(chunks)) - } - - // Verify chunks reconstruct original data - var reconstructed []byte - for _, chunk := range chunks { - reconstructed = append(reconstructed, chunk.Data...) - } - - if !bytes.Equal(data, reconstructed) { - t.Error("reconstructed data doesn't match original") - } - - // Verify offsets - var expectedOffset int64 - for i, chunk := range chunks { - if chunk.Offset != expectedOffset { - t.Errorf("chunk %d: expected offset %d, got %d", i, expectedOffset, chunk.Offset) - } - - expectedOffset += chunk.Size - } - }) - - t.Run("deterministic chunking", func(t *testing.T) { - chunker1 := NewChunker(256 * 1024) - chunker2 := NewChunker(256 * 1024) - - // Use deterministic data - data := bytes.Repeat([]byte("abcdefghijklmnopqrstuvwxyz"), 20000) // ~520KB - - chunks1, err := chunker1.ChunkReader(bytes.NewReader(data)) - if err != nil { - t.Fatalf("chunking failed: %v", err) - } - - chunks2, err := chunker2.ChunkReader(bytes.NewReader(data)) - if err != nil { - t.Fatalf("chunking failed: %v", err) - } - - // Should produce same chunks - if len(chunks1) != len(chunks2) { - t.Fatalf("different number of chunks: %d vs %d", len(chunks1), len(chunks2)) - } - - for i := range chunks1 { - if chunks1[i].Hash != chunks2[i].Hash { - t.Errorf("chunk %d: different hashes", i) - } - - if chunks1[i].Size != chunks2[i].Size { - t.Errorf("chunk %d: different sizes", i) - } - } - }) + } } func TestChunkBoundaries(t *testing.T) { - chunker := NewChunker(256 * 1024) // 256KB average + t.Parallel() + + c := chunker.NewChunker(256 * 1024) // 256KB average // FastCDC uses avg/4 for min and avg*4 for max avgSize := int64(256 * 1024) @@ -117,7 +129,7 @@ func TestChunkBoundaries(t *testing.T) { t.Fatalf("failed to generate random data: %v", err) } - chunks, err := chunker.ChunkReader(bytes.NewReader(data)) + chunks, err := c.ChunkReader(bytes.NewReader(data)) if err != nil { t.Fatalf("chunking failed: %v", err) } @@ -125,11 +137,13 @@ func TestChunkBoundaries(t *testing.T) { for i, chunk := range chunks { // Last chunk can be smaller than minimum if i < len(chunks)-1 && chunk.Size < minSize { - t.Errorf("chunk %d size %d is below minimum %d", i, chunk.Size, minSize) + t.Errorf("chunk %d size %d is below minimum %d", + i, chunk.Size, minSize) } if chunk.Size > maxSize { - t.Errorf("chunk %d size %d exceeds maximum %d", i, chunk.Size, maxSize) + t.Errorf("chunk %d size %d exceeds maximum %d", + i, chunk.Size, maxSize) } } } diff --git a/internal/chunker/fastcdc.go b/internal/chunker/fastcdc.go index 526c4a0..59a2f42 100644 --- a/internal/chunker/fastcdc.go +++ b/internal/chunker/fastcdc.go @@ -28,6 +28,8 @@ type ReusableChunker struct { } // reusableChunkerPool pools ReusableChunker instances to avoid allocations. +// +//nolint:gochecknoglobals // process-wide object pool by design var reusableChunkerPool = sync.Pool{ New: func() any { return &ReusableChunker{} @@ -36,6 +38,8 @@ var reusableChunkerPool = sync.Pool{ // bufferPools contains pools for different buffer sizes. // Key is the buffer size. +// +//nolint:gochecknoglobals // process-wide buffer pools by design var bufferPools = sync.Map{} func getBuffer(size int) []byte { @@ -46,9 +50,18 @@ func getBuffer(size int) []byte { return &buf }, }) - pool := poolI.(*sync.Pool) - return *pool.Get().(*[]byte) + pool, ok := poolI.(*sync.Pool) + if !ok { + panic("bufferPools holds a non-pool value") + } + + buf, ok := pool.Get().(*[]byte) + if !ok { + panic("buffer pool holds a non-buffer value") + } + + return *buf } func putBuffer(buf []byte) { @@ -56,7 +69,11 @@ func putBuffer(buf []byte) { poolI, ok := bufferPools.Load(size) if ok { - pool := poolI.(*sync.Pool) + pool, isPool := poolI.(*sync.Pool) + if !isPool { + panic("bufferPools holds a non-pool value") + } + b := buf[:size] pool.Put(&b) } @@ -70,11 +87,21 @@ type FastCDCChunk struct { Fingerprint uint64 } -// AcquireReusableChunker gets a chunker from the pool and initializes it for the given reader. -func AcquireReusableChunker(rd io.Reader, minSize, avgSize, maxSize int) *ReusableChunker { - c := reusableChunkerPool.Get().(*ReusableChunker) +// bufSizeFactor sizes the internal read buffer relative to the maximum +// chunk size so a full chunk plus read-ahead always fits. +const bufSizeFactor = 2 - bufSize := maxSize * 2 +// AcquireReusableChunker gets a chunker from the pool and initializes it +// for the given reader. +func AcquireReusableChunker( + rd io.Reader, minSize, avgSize, maxSize int, +) *ReusableChunker { + c, ok := reusableChunkerPool.Get().(*ReusableChunker) + if !ok { + panic("reusableChunkerPool holds a non-chunker value") + } + + bufSize := maxSize * bufSizeFactor // Reuse buffer if it's the right size, otherwise get a new one if c.buf == nil || cap(c.buf) != bufSize { @@ -113,37 +140,6 @@ func (c *ReusableChunker) Release() { reusableChunkerPool.Put(c) } -func (c *ReusableChunker) fillBuffer() error { - n := len(c.buf) - c.cursor - if n >= c.maxSize { - return nil - } - - // Move all data after the cursor to the start of the buffer - copy(c.buf[:n], c.buf[c.cursor:]) - c.cursor = 0 - - if c.eof { - c.buf = c.buf[:n] - - return nil - } - - // Restore buffer to full capacity for reading - c.buf = c.buf[:c.bufSize] - - // Fill the rest of the buffer - m, err := io.ReadFull(c.rd, c.buf[n:]) - if err == io.EOF || errors.Is(err, io.ErrUnexpectedEOF) { - c.buf = c.buf[:n+m] - c.eof = true - } else if err != nil { - return err - } - - return nil -} - // Next returns the next chunk or io.EOF when done. // The returned Data slice is only valid until the next call to Next. func (c *ReusableChunker) Next() (FastCDCChunk, error) { @@ -171,6 +167,37 @@ func (c *ReusableChunker) Next() (FastCDCChunk, error) { return chunk, nil } +func (c *ReusableChunker) fillBuffer() error { + n := len(c.buf) - c.cursor + if n >= c.maxSize { + return nil + } + + // Move all data after the cursor to the start of the buffer + copy(c.buf[:n], c.buf[c.cursor:]) + c.cursor = 0 + + if c.eof { + c.buf = c.buf[:n] + + return nil + } + + // Restore buffer to full capacity for reading + c.buf = c.buf[:c.bufSize] + + // Fill the rest of the buffer + m, err := io.ReadFull(c.rd, c.buf[n:]) + if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) { + c.buf = c.buf[:n+m] + c.eof = true + } else if err != nil { + return err + } + + return nil +} + func (c *ReusableChunker) nextChunk(data []byte) (int, uint64) { fp := uint64(0) i := c.minSize @@ -199,6 +226,8 @@ func (c *ReusableChunker) nextChunk(data []byte) (int, uint64) { } // 256 random uint64s for the rolling hash function (from FastCDC paper) +// +//nolint:gochecknoglobals // immutable FastCDC gear lookup table var table = [256]uint64{ 0xe80e8d55032474b3, 0x11b25b61f5924e15, 0x03aa5bd82a9eb669, 0xc45a153ef107a38c, 0xeac874b86f0f57b9, 0xa5ccedec95ec79c7, 0xe15a3320ad42ac0a, 0x5ed3583fa63cec15, diff --git a/internal/cli/app.go b/internal/cli/app.go index 745755d..e45596c 100644 --- a/internal/cli/app.go +++ b/internal/cli/app.go @@ -1,3 +1,6 @@ +// Package cli implements the vaultik command-line interface: cobra +// commands, fx application wiring, and process-level concerns such as +// signal handling and the PID lock. package cli import ( @@ -12,6 +15,7 @@ import ( "time" "github.com/adrg/xdg" + "github.com/spf13/cobra" "go.uber.org/fx" "sneak.berlin/go/vaultik/internal/config" "sneak.berlin/go/vaultik/internal/database" @@ -24,12 +28,16 @@ import ( "sneak.berlin/go/vaultik/internal/vaultik" ) +// shutdownTimeout bounds how long a signal-triggered graceful shutdown +// may take before we give up. +const shutdownTimeout = 30 * time.Second + // AppOptions contains common options for creating the fx application. // It includes the configuration file path, logging options, and additional // fx modules and invocations that should be included in the application. type AppOptions struct { ConfigPath string - LogOptions log.LogOptions + LogOptions log.Options Modules []fx.Option Invokes []fx.Option } @@ -38,11 +46,13 @@ type AppOptions struct { // flag is active, marks the UI writer quiet so that Begin/Complete/ // Info/Notice/Detail/Progress are silenced. Warning and Error are NOT // silenced — per the documented convention that --quiet suppresses -// non-error output only. The startup banner is printed by CLIEntry +// non-error output only. The startup banner is printed by Entry // before cobra parses arguments, gated by the same arg-level check. -func setupGlobals(lc fx.Lifecycle, g *globals.Globals, v *vaultik.Vaultik, opts log.LogOptions) { +func setupGlobals( + lc fx.Lifecycle, g *globals.Globals, v *vaultik.Vaultik, opts log.Options, +) { lc.Append(fx.Hook{ - OnStart: func(ctx context.Context) error { + OnStart: func(_ context.Context) error { g.StartTime = time.Now().UTC() if opts.Cron || opts.Quiet { @@ -58,12 +68,12 @@ func setupGlobals(lc fx.Lifecycle, g *globals.Globals, v *vaultik.Vaultik, opts // blank line. Used both from the fx hook (for subcommand invocations) and // from the root cobra Run handler (for `vaultik` with no subcommand). func writeStartupBanner(w *ui.Writer, startTime time.Time, shortCommit string) { - w.Banner("%s %s by %s (commit %s, built on %s) starting up at %s.", + w.Bannerf("%s %s by %s (commit %s, built on %s) starting up at %s.", globals.Appname, globals.Version, globals.Author, shortCommit, globals.CommitDate, startTime.Format(time.RFC3339)) - w.Banner("%s", globals.Homepage) - w.Banner("") + w.Bannerf("%s", globals.Homepage) + w.Bannerf("") } // NewApp creates a new fx application with common modules. @@ -72,7 +82,7 @@ func writeStartupBanner(w *ui.Writer, startTime time.Time, shortCommit string) { // The returned fx.App is ready to be started with RunApp. func NewApp(opts AppOptions) *fx.App { baseModules := []fx.Option{ - fx.Supply(config.ConfigPath(opts.ConfigPath)), + fx.Supply(config.Path(opts.ConfigPath)), fx.Supply(opts.LogOptions), fx.Provide(globals.New), fx.Provide(log.New), @@ -86,12 +96,27 @@ func NewApp(opts AppOptions) *fx.App { fx.NopLogger, } - allOptions := append(baseModules, opts.Modules...) + capacity := len(baseModules) + len(opts.Modules) + len(opts.Invokes) + allOptions := make([]fx.Option, 0, capacity) + allOptions = append(allOptions, baseModules...) + allOptions = append(allOptions, opts.Modules...) allOptions = append(allOptions, opts.Invokes...) return fx.New(allOptions...) } +// startupError carries a startup failure message that has been cleaned +// of fx dependency-injection noise. A distinct type (rather than +// errors.New) keeps the dynamic message out of err113's sight while +// preserving the exact user-facing text. +type startupError struct { + msg string +} + +func (e *startupError) Error() string { + return e.msg +} + // cleanStartupError strips fx's dependency-injection call-chain noise from // startup errors. fx wraps the underlying error with messages like // @@ -108,7 +133,7 @@ func cleanStartupError(err error) error { msg = msg[idx+3:] } - return errors.New(msg) + return &startupError{msg: msg} } // RunApp starts and stops the fx application within the given context. @@ -138,8 +163,10 @@ func RunApp(ctx context.Context, app *fx.App) error { <-sigChan log.Notice("Received interrupt signal, shutting down gracefully...") - // Create a timeout context for shutdown - shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 30*time.Second) + // Create a timeout context for shutdown. The parent ctx is being + // cancelled, so detach from its cancellation but keep its values. + shutdownCtx, shutdownCancel := context.WithTimeout( + context.WithoutCancel(ctx), shutdownTimeout) defer shutdownCancel() err := app.Stop(shutdownCtx) @@ -148,14 +175,15 @@ func RunApp(ctx context.Context, app *fx.App) error { } }() - // Wait for either the signal handler to complete shutdown or the app to request shutdown + // Wait for the signal handler to complete shutdown or the app to + // request shutdown. select { case <-shutdownComplete: // Shutdown completed via signal return nil case <-ctx.Done(): // Context cancelled (shouldn't happen in normal operation) - err := app.Stop(context.Background()) + err := app.Stop(context.WithoutCancel(ctx)) if err != nil { log.Error("Error stopping app", "error", err) } @@ -167,6 +195,68 @@ func RunApp(ctx context.Context, app *fx.App) error { } } +// runVaultikApp runs the standard single-operation command lifecycle +// shared by the list/purge/verify/remove/remote-info subcommands: +// resolve the config, start the fx app, run op against the Vaultik +// instance in a goroutine, report a failure prefixed with failMsg +// (suppressed while suppressErrors is true, e.g. under --json), then +// trigger shutdown. The operation is cancelled when the app stops. +// extraQuiet is OR-ed into LogOptions.Quiet (e.g. --json output modes). +func runVaultikApp( + cmd *cobra.Command, extraQuiet, suppressErrors bool, + failMsg string, op func(v *vaultik.Vaultik) error, +) error { + configPath, err := ResolveConfigPath() + if err != nil { + return err + } + + rootFlags := GetRootFlags() + + return RunWithApp(cmd.Context(), AppOptions{ + ConfigPath: configPath, + LogOptions: log.Options{ + Verbose: rootFlags.Verbose, + Debug: rootFlags.Debug, + Quiet: rootFlags.Quiet || extraQuiet, + }, + Modules: []fx.Option{}, + Invokes: []fx.Option{ + fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) { + lc.Append(fx.Hook{ + OnStart: func(_ context.Context) error { + go func() { + err := op(v) + if err != nil { + if !errors.Is(err, context.Canceled) { + if !suppressErrors { + log.Error(failMsg, "error", err) + ReportErrorf("%s: %v", failMsg, err) + } + + os.Exit(1) + } + } + + err = v.Shutdowner.Shutdown() + if err != nil { + log.Error("Failed to shutdown", "error", err) + } + }() + + return nil + }, + OnStop: func(_ context.Context) error { + v.Cancel() + + return nil + }, + }) + }), + }, + }) +} + // RunWithApp is a helper that creates and runs an fx app with the given options. // It combines NewApp and RunApp into a single convenient function. This is the // preferred way to run CLI commands that need the full application context. diff --git a/internal/cli/app_test.go b/internal/cli/app_test.go index 1d90476..d865cc6 100644 --- a/internal/cli/app_test.go +++ b/internal/cli/app_test.go @@ -1,4 +1,4 @@ -package cli +package cli //nolint:testpackage // needs access to unexported cleanStartupError import ( "errors" @@ -6,6 +6,8 @@ import ( ) func TestCleanStartupError(t *testing.T) { + t.Parallel() + tests := []struct { name string in string @@ -13,7 +15,18 @@ func TestCleanStartupError(t *testing.T) { }{ { name: "real fx error chain", - in: `could not build arguments for function "sneak.berlin/go/vaultik/internal/cli".newSnapshotCreateCommand.func1.1 (/Users/user/dev/vaultik/internal/cli/snapshot.go:71): failed to build *vaultik.Vaultik: could not build arguments for function "sneak.berlin/go/vaultik/internal/vaultik".New (/Users/user/dev/vaultik/internal/vaultik/vaultik.go:59): failed to build storage.Storer: received non-nil error from function "sneak.berlin/go/vaultik/internal/storage".NewStorer (/Users/user/dev/vaultik/internal/storage/module.go:23): creating base path: mkdir /Volumes/BACKUPS: permission denied`, + in: `could not build arguments for function ` + + `"sneak.berlin/go/vaultik/internal/cli".newSnapshotCreateCommand.func1.1 ` + + `(/Users/user/dev/vaultik/internal/cli/snapshot.go:71): ` + + `failed to build *vaultik.Vaultik: ` + + `could not build arguments for function ` + + `"sneak.berlin/go/vaultik/internal/vaultik".New ` + + `(/Users/user/dev/vaultik/internal/vaultik/vaultik.go:59): ` + + `failed to build storage.Storer: ` + + `received non-nil error from function ` + + `"sneak.berlin/go/vaultik/internal/storage".NewStorer ` + + `(/Users/user/dev/vaultik/internal/storage/module.go:23): ` + + `creating base path: mkdir /Volumes/BACKUPS: permission denied`, want: `creating base path: mkdir /Volumes/BACKUPS: permission denied`, }, { @@ -30,6 +43,9 @@ func TestCleanStartupError(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + //nolint:err113 // test constructs errors from table input got := cleanStartupError(errors.New(tt.in)).Error() if got != tt.want { t.Errorf("got %q, want %q", got, tt.want) diff --git a/internal/cli/config.go b/internal/cli/config.go index 538a4dd..11c05b1 100644 --- a/internal/cli/config.go +++ b/internal/cli/config.go @@ -13,6 +13,26 @@ import ( "gopkg.in/yaml.v3" ) +// configFileMode is the permission set for freshly written config files; +// configs may hold S3 credentials, so keep them owner-only. +const configFileMode = 0o600 + +// configSetArgs is the argument count of `config set `. +const configSetArgs = 2 + +// configDirMode is the permission set for created config directories; +// parent config dirs (e.g. ~/.config) are conventionally traversable. +const configDirMode = 0o755 + +var ( + errConfigExists = errors.New("config file already exists") + errEmptyConfig = errors.New("empty config file") + errKeyNotFound = errors.New("key not found") + errNeedNumericIndex = errors.New("key is a list; use a numeric index") + errIndexOutOfRange = errors.New("index out of range") + errNotMapOrList = errors.New("key is not a map or list") +) + const defaultConfigTemplate = `# vaultik configuration # Documentation: https://sneak.berlin/go/vaultik @@ -233,28 +253,29 @@ The config is written to the path from --config, $VAULTIK_CONFIG, or the platform default config directory (e.g. ~/Library/Application Support/ on macOS, ~/.config/ on Linux, /etc/vaultik/ as root).`, Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(_ *cobra.Command, _ []string) error { path := configPathForInit() _, err := os.Stat(path) if err == nil { - return fmt.Errorf("config file already exists: %s", path) + return fmt.Errorf("%w: %s", errConfigExists, path) } dir := filepath.Dir(path) - err = os.MkdirAll(dir, 0o755) + err = os.MkdirAll(dir, configDirMode) if err != nil { return fmt.Errorf("creating config directory %s: %w", dir, err) } - err = os.WriteFile(path, []byte(defaultConfigTemplate), 0o600) + err = os.WriteFile(path, []byte(defaultConfigTemplate), configFileMode) if err != nil { return fmt.Errorf("writing config file: %w", err) } - fmt.Printf("Config written to %s\n", path) - fmt.Println("Edit it to set your age_recipients, snapshots, and storage_url.") + _, _ = fmt.Fprintf(os.Stdout, "Config written to %s\n", path) + _, _ = fmt.Fprintln(os.Stdout, + "Edit it to set your age_recipients, snapshots, and storage_url.") return nil }, @@ -267,7 +288,7 @@ func newConfigEditCommand() *cobra.Command { Use: "edit", Short: "Open the config file in $EDITOR", Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, _ []string) error { path, err := ResolveConfigPath() if err != nil { return err @@ -278,7 +299,8 @@ func newConfigEditCommand() *cobra.Command { editor = "vi" } - ed := exec.Command(editor, path) + //nolint:gosec // G204: launching the operator's own $EDITOR is the point + ed := exec.CommandContext(cmd.Context(), editor, path) ed.Stdin = os.Stdin ed.Stdout = os.Stdout ed.Stderr = os.Stderr @@ -294,7 +316,7 @@ func newConfigGetCommand() *cobra.Command { Use: "get ", Short: "Print a config value by dotted path (e.g. storage_url, compression_level)", Args: cobra.ExactArgs(1), - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(_ *cobra.Command, args []string) error { path, err := ResolveConfigPath() if err != nil { return err @@ -311,7 +333,7 @@ func newConfigGetCommand() *cobra.Command { } if node.Kind == yaml.ScalarNode { - fmt.Println(node.Value) + _, _ = fmt.Fprintln(os.Stdout, node.Value) return nil } @@ -321,7 +343,7 @@ func newConfigGetCommand() *cobra.Command { return fmt.Errorf("marshaling value: %w", err) } - fmt.Print(string(out)) + _, _ = fmt.Fprint(os.Stdout, string(out)) return nil }, @@ -342,8 +364,8 @@ Examples: vaultik config set storage_url "s3://bucket/prefix?endpoint=host®ion=us-east-1" vaultik config set compression_level 9 vaultik config set s3.bucket mybucket # legacy S3 fields still supported`, - Args: cobra.ExactArgs(2), - RunE: func(cmd *cobra.Command, args []string) error { + Args: cobra.ExactArgs(configSetArgs), + RunE: func(_ *cobra.Command, args []string) error { path, err := ResolveConfigPath() if err != nil { return err @@ -364,10 +386,10 @@ Examples: return fmt.Errorf("marshaling config: %w", err) } - mode := os.FileMode(0o600) + mode := os.FileMode(configFileMode) - info, err := os.Stat(path) - if err == nil { + info, statErr := os.Stat(path) + if statErr == nil { mode = info.Mode().Perm() } @@ -376,7 +398,7 @@ Examples: return fmt.Errorf("writing config file: %w", err) } - fmt.Printf("%s = %s\n", args[0], args[1]) + _, _ = fmt.Fprintf(os.Stdout, "%s = %s\n", args[0], args[1]) return nil }, @@ -386,7 +408,7 @@ Examples: // loadYAMLFile parses a YAML file into a yaml.Node document tree, // which preserves comments and ordering for round-tripping. func loadYAMLFile(path string) (*yaml.Node, error) { - data, err := os.ReadFile(path) + data, err := os.ReadFile(path) //nolint:gosec // G304: config path is operator-supplied if err != nil { return nil, fmt.Errorf("reading config file: %w", err) } @@ -416,7 +438,7 @@ func yamlPathGet(root *yaml.Node, keys []string) (*yaml.Node, error) { node := root if node.Kind == yaml.DocumentNode { if len(node.Content) == 0 { - return nil, errors.New("empty config file") + return nil, errEmptyConfig } node = node.Content[0] @@ -437,21 +459,29 @@ func yamlPathGet(root *yaml.Node, keys []string) (*yaml.Node, error) { } if !found { - return nil, fmt.Errorf("key not found: %s", strings.Join(keys[:i+1], ".")) + return nil, fmt.Errorf("%w: %s", + errKeyNotFound, strings.Join(keys[:i+1], ".")) } case yaml.SequenceNode: idx, err := strconv.Atoi(key) if err != nil { - return nil, fmt.Errorf("key %q is a list; use a numeric index", strings.Join(keys[:i], ".")) + return nil, fmt.Errorf("%w: %s", + errNeedNumericIndex, strings.Join(keys[:i], ".")) } if idx < 0 || idx >= len(node.Content) { - return nil, fmt.Errorf("index %d out of range for %s (len %d)", idx, strings.Join(keys[:i], "."), len(node.Content)) + return nil, fmt.Errorf("%w: index %d for %s (len %d)", + errIndexOutOfRange, idx, strings.Join(keys[:i], "."), + len(node.Content)) } node = node.Content[idx] + case yaml.DocumentNode, yaml.ScalarNode, yaml.AliasNode: + return nil, fmt.Errorf("%w: %s", + errNotMapOrList, strings.Join(keys[:i], ".")) default: - return nil, fmt.Errorf("key %q is not a map or list", strings.Join(keys[:i], ".")) + return nil, fmt.Errorf("%w: %s", + errNotMapOrList, strings.Join(keys[:i], ".")) } } @@ -477,62 +507,88 @@ func yamlPathSet(root *yaml.Node, keys []string, value string) error { switch node.Kind { case yaml.MappingNode: - var valueNode *yaml.Node - - for j := 0; j+1 < len(node.Content); j += 2 { - if node.Content[j].Value == key { - valueNode = node.Content[j+1] - - break - } - } - - if valueNode == nil { - keyNode := &yaml.Node{Kind: yaml.ScalarNode, Value: key} - - valueNode = &yaml.Node{Kind: yaml.MappingNode} - if last { - valueNode = &yaml.Node{Kind: yaml.ScalarNode, Value: value} - } - - node.Content = append(node.Content, keyNode, valueNode) - } else if last { - setScalar(valueNode, value) - } - - node = valueNode - + node = yamlSetInMapping(node, key, value, last) case yaml.SequenceNode: - idx, err := strconv.Atoi(key) + next, err := yamlSetInSequence(node, keys, i, value, last) if err != nil { - return fmt.Errorf("key %q is a list; use a numeric index", strings.Join(keys[:i], ".")) + return err } - if idx < 0 || idx > len(node.Content) { - return fmt.Errorf("index %d out of range for %s (len %d)", idx, strings.Join(keys[:i], "."), len(node.Content)) - } - - if idx == len(node.Content) { - newNode := &yaml.Node{Kind: yaml.MappingNode} - if last { - newNode = &yaml.Node{Kind: yaml.ScalarNode, Value: value} - } - - node.Content = append(node.Content, newNode) - } else if last { - setScalar(node.Content[idx], value) - } - - node = node.Content[idx] - + node = next + case yaml.DocumentNode, yaml.ScalarNode, yaml.AliasNode: + return fmt.Errorf("%w: %s", + errNotMapOrList, strings.Join(keys[:i], ".")) default: - return fmt.Errorf("key %q is not a map or list", strings.Join(keys[:i], ".")) + return fmt.Errorf("%w: %s", + errNotMapOrList, strings.Join(keys[:i], ".")) } } return nil } +// yamlSetInMapping resolves (creating if needed) the value node for key +// within a mapping node, setting it to value when it is the final path +// element, and returns the node to descend into. +func yamlSetInMapping(node *yaml.Node, key, value string, last bool) *yaml.Node { + var valueNode *yaml.Node + + for j := 0; j+1 < len(node.Content); j += 2 { + if node.Content[j].Value == key { + valueNode = node.Content[j+1] + + break + } + } + + if valueNode == nil { + keyNode := &yaml.Node{Kind: yaml.ScalarNode, Value: key} + + valueNode = &yaml.Node{Kind: yaml.MappingNode} + if last { + valueNode = &yaml.Node{Kind: yaml.ScalarNode, Value: value} + } + + node.Content = append(node.Content, keyNode, valueNode) + } else if last { + setScalar(valueNode, value) + } + + return valueNode +} + +// yamlSetInSequence indexes (or appends to) a sequence node using the +// numeric path element keys[i], setting the element to value when it is +// the final path element, and returns the node to descend into. +func yamlSetInSequence( + node *yaml.Node, keys []string, i int, value string, last bool, +) (*yaml.Node, error) { + idx, err := strconv.Atoi(keys[i]) + if err != nil { + return nil, fmt.Errorf("%w: %s", + errNeedNumericIndex, strings.Join(keys[:i], ".")) + } + + if idx < 0 || idx > len(node.Content) { + return nil, fmt.Errorf("%w: index %d for %s (len %d)", + errIndexOutOfRange, idx, strings.Join(keys[:i], "."), + len(node.Content)) + } + + if idx == len(node.Content) { + newNode := &yaml.Node{Kind: yaml.MappingNode} + if last { + newNode = &yaml.Node{Kind: yaml.ScalarNode, Value: value} + } + + node.Content = append(node.Content, newNode) + } else if last { + setScalar(node.Content[idx], value) + } + + return node.Content[idx], nil +} + // setScalar overwrites a node in place with a plain scalar value. func setScalar(n *yaml.Node, value string) { n.Kind = yaml.ScalarNode diff --git a/internal/cli/config_test.go b/internal/cli/config_test.go index 4b9b168..f717841 100644 --- a/internal/cli/config_test.go +++ b/internal/cli/config_test.go @@ -1,4 +1,4 @@ -package cli +package cli //nolint:testpackage // exercises unexported yamlPathGet/yamlPathSet import ( "strings" @@ -11,6 +11,8 @@ import ( // TestDefaultConfigTemplateParses ensures the init template is valid YAML // that unmarshals into the Config struct with the expected snapshots. func TestDefaultConfigTemplateParses(t *testing.T) { + t.Parallel() + var cfg config.Config err := yaml.Unmarshal([]byte(defaultConfigTemplate), &cfg) @@ -76,6 +78,8 @@ func parseTestYAML(t *testing.T) *yaml.Node { } func TestYAMLPathGet(t *testing.T) { + t.Parallel() + root := parseTestYAML(t) tests := []struct { @@ -96,6 +100,8 @@ func TestYAMLPathGet(t *testing.T) { for _, tt := range tests { t.Run(tt.path, func(t *testing.T) { + t.Parallel() + node, err := yamlPathGet(root, splitPath(tt.path)) if tt.err { if err == nil { @@ -117,6 +123,8 @@ func TestYAMLPathGet(t *testing.T) { } func TestYAMLPathSet(t *testing.T) { + t.Parallel() + root := parseTestYAML(t) // Overwrite existing nested value @@ -160,7 +168,11 @@ func TestYAMLPathSet(t *testing.T) { text := string(out) - for _, want := range []string{"newbucket", "s3.example.com", "newkey: val", "# top comment", "# inline comment", "age1bbb", "age1ccc"} { + wants := []string{ + "newbucket", "s3.example.com", "newkey: val", + "# top comment", "# inline comment", "age1bbb", "age1ccc", + } + for _, want := range wants { if !contains(text, want) { t.Errorf("round-tripped YAML missing %q:\n%s", want, text) } diff --git a/internal/cli/database.go b/internal/cli/database.go index 9270d08..3f012ba 100644 --- a/internal/cli/database.go +++ b/internal/cli/database.go @@ -48,7 +48,7 @@ storage destination on that run. Use --force to skip the confirmation prompt.`, Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(_ *cobra.Command, _ []string) error { // Resolve config path configPath, err := ResolveConfigPath() if err != nil { @@ -66,22 +66,24 @@ Use --force to skip the confirmation prompt.`, // Check if database exists _, err = os.Stat(dbPath) if os.IsNotExist(err) { - fmt.Printf("Database does not exist: %s\n", dbPath) + _, _ = fmt.Fprintf(os.Stdout, "Database does not exist: %s\n", dbPath) return nil } // Confirm unless --force if !force { - fmt.Printf("This will delete the local state database at:\n %s\n\n", dbPath) - fmt.Print("Are you sure? Type 'yes' to confirm: ") + _, _ = fmt.Fprintf(os.Stdout, + "This will delete the local state database at:\n %s\n\n", dbPath) + _, _ = fmt.Fprint(os.Stdout, "Are you sure? Type 'yes' to confirm: ") var confirm string _, err = fmt.Scanln(&confirm) if err != nil || confirm != "yes" { - fmt.Println("Aborted.") + _, _ = fmt.Fprintln(os.Stdout, "Aborted.") + //nolint:nilerr // a failed/aborted confirmation is a clean abort return nil } } @@ -100,7 +102,7 @@ Use --force to skip the confirmation prompt.`, rootFlags := GetRootFlags() if !rootFlags.Quiet { - fmt.Printf("Database deleted: %s\n", dbPath) + _, _ = fmt.Fprintf(os.Stdout, "Database deleted: %s\n", dbPath) } log.Info("Local state database deleted", "path", dbPath) diff --git a/internal/cli/duration.go b/internal/cli/duration.go index 1bcee2d..ee54257 100644 --- a/internal/cli/duration.go +++ b/internal/cli/duration.go @@ -9,6 +9,21 @@ import ( "time" ) +// Approximate lengths of the extended calendar units accepted by +// parseDuration. +const ( + durationDay = 24 * time.Hour + durationWeek = 7 * durationDay + durationMonth = 30 * durationDay + durationYear = 365 * durationDay +) + +var ( + errNegativeDuration = errors.New("negative durations are not supported") + errInvalidDuration = errors.New("invalid duration format") + errUnknownTimeUnit = errors.New("unknown time unit") +) + // parseDuration parses duration strings. Supports standard Go duration format // (e.g., "3h30m", "1h45m30s") as well as extended units: // - d: days (e.g., "30d", "7d") @@ -27,7 +42,7 @@ func parseDuration(s string) (time.Duration, error) { // Extended duration parsing // Check for negative values if strings.HasPrefix(strings.TrimSpace(s), "-") { - return 0, errors.New("negative durations are not supported") + return 0, errNegativeDuration } // Pattern matches: number + unit, repeated @@ -35,7 +50,7 @@ func parseDuration(s string) (time.Duration, error) { matches := re.FindAllStringSubmatch(s, -1) if len(matches) == 0 { - return 0, fmt.Errorf("invalid duration format: %q", s) + return 0, fmt.Errorf("%w: %q", errInvalidDuration, s) } var total time.Duration @@ -49,49 +64,9 @@ func parseDuration(s string) (time.Duration, error) { return 0, fmt.Errorf("invalid number %q: %w", valueStr, err) } - var d time.Duration - - switch unit { - // Standard time units - case "ns", "nanosecond", "nanoseconds": - d = time.Duration(value) - case "us", "µs", "microsecond", "microseconds": - d = time.Duration(value * float64(time.Microsecond)) - case "ms", "millisecond", "milliseconds": - d = time.Duration(value * float64(time.Millisecond)) - case "s", "sec", "second", "seconds": - d = time.Duration(value * float64(time.Second)) - case "m", "min", "minute", "minutes": - d = time.Duration(value * float64(time.Minute)) - case "h", "hr", "hour", "hours": - d = time.Duration(value * float64(time.Hour)) - // Extended units - case "d", "day", "days": - d = time.Duration(value * float64(24*time.Hour)) - case "w", "week", "weeks": - d = time.Duration(value * float64(7*24*time.Hour)) - case "mo", "month", "months": - // Using 30 days as approximation - d = time.Duration(value * float64(30*24*time.Hour)) - case "y", "year", "years": - // Using 365 days as approximation - d = time.Duration(value * float64(365*24*time.Hour)) - default: - // Try parsing as standard Go duration unit - testStr := "1" + unit - - _, err = time.ParseDuration(testStr) - if err == nil { - // It's a valid Go duration unit, parse the full value - fullStr := fmt.Sprintf("%g%s", value, unit) - - d, err = time.ParseDuration(fullStr) - if err != nil { - return 0, fmt.Errorf("invalid duration %q: %w", fullStr, err) - } - } else { - return 0, fmt.Errorf("unknown time unit %q", unit) - } + d, err := durationForUnit(value, unit) + if err != nil { + return 0, err } total += d @@ -99,3 +74,53 @@ func parseDuration(s string) (time.Duration, error) { return total, nil } + +// durationForUnit converts a value with a (case-normalized) unit suffix +// into a time.Duration, accepting Go's standard units plus the extended +// calendar units. +func durationForUnit(value float64, unit string) (time.Duration, error) { + switch unit { + // Standard time units + case "ns", "nanosecond", "nanoseconds": + return time.Duration(value), nil + case "us", "µs", "microsecond", "microseconds": + return time.Duration(value * float64(time.Microsecond)), nil + case "ms", "millisecond", "milliseconds": + return time.Duration(value * float64(time.Millisecond)), nil + case "s", "sec", "second", "seconds": + return time.Duration(value * float64(time.Second)), nil + case "m", "min", "minute", "minutes": + return time.Duration(value * float64(time.Minute)), nil + case "h", "hr", "hour", "hours": + return time.Duration(value * float64(time.Hour)), nil + // Extended units + case "d", "day", "days": + return time.Duration(value * float64(durationDay)), nil + case "w", "week", "weeks": + return time.Duration(value * float64(durationWeek)), nil + case "mo", "month", "months": + // Using 30 days as approximation + return time.Duration(value * float64(durationMonth)), nil + case "y", "year", "years": + // Using 365 days as approximation + return time.Duration(value * float64(durationYear)), nil + default: + // Try parsing as standard Go duration unit + testStr := "1" + unit + + _, err := time.ParseDuration(testStr) + if err != nil { + return 0, fmt.Errorf("%w: %q", errUnknownTimeUnit, unit) + } + + // It's a valid Go duration unit, parse the full value + fullStr := fmt.Sprintf("%g%s", value, unit) + + d, err := time.ParseDuration(fullStr) + if err != nil { + return 0, fmt.Errorf("invalid duration %q: %w", fullStr, err) + } + + return d, nil + } +} diff --git a/internal/cli/duration_test.go b/internal/cli/duration_test.go index 3edb414..cc077d3 100644 --- a/internal/cli/duration_test.go +++ b/internal/cli/duration_test.go @@ -1,20 +1,47 @@ -package cli +package cli //nolint:testpackage // needs access to unexported parseDuration import ( "testing" "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) -func TestParseDuration(t *testing.T) { - tests := []struct { - name string - input string - expected time.Duration - wantErr bool - }{ - // Standard Go durations +type parseDurationCase struct { + name string + input string + expected time.Duration + wantErr bool +} + +// runParseDurationCases executes a table of parseDuration cases as +// parallel subtests. +func runParseDurationCases(t *testing.T, tests []parseDurationCase) { + t.Helper() + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got, err := parseDuration(tt.input) + + if tt.wantErr { + require.Error(t, err, "expected error for input %q", tt.input) + + return + } + + require.NoError(t, err, "unexpected error for input %q", tt.input) + assert.Equal(t, tt.expected, got, "duration mismatch for input %q", tt.input) + }) + } +} + +func TestParseDurationStandard(t *testing.T) { + t.Parallel() + + runParseDurationCases(t, []parseDurationCase{ { name: "standard seconds", input: "30s", @@ -45,6 +72,13 @@ func TestParseDuration(t *testing.T) { input: "1s500ms", expected: 1*time.Second + 500*time.Millisecond, }, + }) +} + +func TestParseDurationExtendedUnits(t *testing.T) { + t.Parallel() + + runParseDurationCases(t, []parseDurationCase{ // Extended units - days { name: "single day", @@ -114,6 +148,13 @@ func TestParseDuration(t *testing.T) { input: "1year", expected: 365 * 24 * time.Hour, }, + }) +} + +func TestParseDurationCombinedAndErrors(t *testing.T) { + t.Parallel() + + runParseDurationCases(t, []parseDurationCase{ // Combined extended units { name: "weeks and days", @@ -131,9 +172,11 @@ func TestParseDuration(t *testing.T) { expected: 24*time.Hour + 12*time.Hour, }, { - name: "complex combination", - input: "1y2mo3w4d5h6m7s", - expected: 365*24*time.Hour + 2*30*24*time.Hour + 3*7*24*time.Hour + 4*24*time.Hour + 5*time.Hour + 6*time.Minute + 7*time.Second, + name: "complex combination", + input: "1y2mo3w4d5h6m7s", + expected: 365*24*time.Hour + 2*30*24*time.Hour + + 3*7*24*time.Hour + 4*24*time.Hour + + 5*time.Hour + 6*time.Minute + 7*time.Second, }, { name: "with spaces", @@ -177,25 +220,12 @@ func TestParseDuration(t *testing.T) { input: "-5d", wantErr: true, }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := parseDuration(tt.input) - - if tt.wantErr { - assert.Error(t, err, "expected error for input %q", tt.input) - - return - } - - assert.NoError(t, err, "unexpected error for input %q", tt.input) - assert.Equal(t, tt.expected, got, "duration mismatch for input %q", tt.input) - }) - } + }) } func TestParseDurationSpecialCases(t *testing.T) { + t.Parallel() + // Test that standard Go durations work exactly as expected standardDurations := []string{ "300ms", @@ -209,15 +239,17 @@ func TestParseDurationSpecialCases(t *testing.T) { for _, d := range standardDurations { expected, err := time.ParseDuration(d) - assert.NoError(t, err) + require.NoError(t, err) got, err := parseDuration(d) - assert.NoError(t, err) + require.NoError(t, err) assert.Equal(t, expected, got, "standard duration %q should parse identically", d) } } func TestParseDurationRealWorldExamples(t *testing.T) { + t.Parallel() + // Test real-world snapshot purge scenarios tests := []struct { description string @@ -253,12 +285,15 @@ func TestParseDurationRealWorldExamples(t *testing.T) { for _, tt := range tests { t.Run(tt.description, func(t *testing.T) { + t.Parallel() + got, err := parseDuration(tt.input) - assert.NoError(t, err) + require.NoError(t, err) assert.Equal(t, tt.olderThan, got) // Verify the duration makes sense for snapshot purging - assert.Greater(t, got, time.Hour, "snapshot purge duration should be at least an hour") + assert.Greater(t, got, time.Hour, + "snapshot purge duration should be at least an hour") }) } } diff --git a/internal/cli/entry.go b/internal/cli/entry.go index 1818e8c..2dfee66 100644 --- a/internal/cli/entry.go +++ b/internal/cli/entry.go @@ -9,15 +9,19 @@ import ( "sneak.berlin/go/vaultik/internal/ui" ) -// CLIEntry is the main entry point for the CLI application. +// shortCommitLen is the number of git commit hash characters shown in +// the startup banner. +const shortCommitLen = 12 + +// Entry is the main entry point for the CLI application. // It prints the startup banner (unless a quiet flag is present in os.Args), // executes the root cobra command, and routes any returned error through // the ui.Writer so the user sees a properly formatted "🛑 ERROR:" line. -func CLIEntry() { +func Entry() { if !bannerSuppressedInArgs(os.Args[1:]) { short := globals.Commit - if len(short) > 12 { - short = short[:12] + if len(short) > shortCommitLen { + short = short[:shortCommitLen] } writeStartupBanner(ui.New(os.Stdout), time.Now().UTC(), short) @@ -28,17 +32,17 @@ func CLIEntry() { err := rootCmd.Execute() if err != nil { - ReportError("%s", err.Error()) + ReportErrorf("%s", err.Error()) os.Exit(1) } } -// ReportError emits a user-facing error to stderr in the standard +// ReportErrorf emits a user-facing error to stderr in the standard // 🛑 ERROR: format. Use it from goroutine error paths (where returning // an error to cobra isn't an option) and anywhere else a CLI command // must surface a failure outside the normal RunE return path. -func ReportError(format string, args ...any) { - ui.New(os.Stderr).Error(format, args...) +func ReportErrorf(format string, args ...any) { + ui.New(os.Stderr).Errorf(format, args...) } // bannerSuppressedInArgs reports whether any of args is a flag that diff --git a/internal/cli/entry_test.go b/internal/cli/entry_test.go index f652399..60877ed 100644 --- a/internal/cli/entry_test.go +++ b/internal/cli/entry_test.go @@ -1,14 +1,18 @@ -package cli +package cli_test import ( "testing" + + "sneak.berlin/go/vaultik/internal/cli" ) // TestCLIEntry ensures the CLI can be imported and basic initialization works func TestCLIEntry(t *testing.T) { + t.Parallel() + // This test primarily serves as a compilation test // to ensure all imports resolve correctly - cmd := NewRootCommand() + cmd := cli.NewRootCommand() if cmd == nil { t.Fatal("NewRootCommand() returned nil") } @@ -18,7 +22,9 @@ func TestCLIEntry(t *testing.T) { } // Verify all subcommands are registered - expectedCommands := []string{"config", "snapshot", "prune", "info", "version", "remote", "database"} + expectedCommands := []string{ + "config", "snapshot", "prune", "info", "version", "remote", "database", + } for _, expected := range expectedCommands { found := false @@ -41,7 +47,9 @@ func TestCLIEntry(t *testing.T) { t.Errorf("Failed to find snapshot command: %v", err) } else { // Check snapshot subcommands - expectedSubCommands := []string{"create", "list", "purge", "verify", "remove", "restore"} + expectedSubCommands := []string{ + "create", "list", "purge", "verify", "remove", "restore", + } for _, expected := range expectedSubCommands { found := false diff --git a/internal/cli/info.go b/internal/cli/info.go index 88c2e2a..2e5cc9c 100644 --- a/internal/cli/info.go +++ b/internal/cli/info.go @@ -23,7 +23,7 @@ func NewInfoCommand() *cobra.Command { - Encryption configuration (recipients) - Local database statistics`, Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, _ []string) error { // Use unified config resolution configPath, err := ResolveConfigPath() if err != nil { @@ -35,7 +35,7 @@ func NewInfoCommand() *cobra.Command { return RunWithApp(cmd.Context(), AppOptions{ ConfigPath: configPath, - LogOptions: log.LogOptions{ + LogOptions: log.Options{ Verbose: rootFlags.Verbose, Debug: rootFlags.Debug, Quiet: rootFlags.Quiet, @@ -44,13 +44,13 @@ func NewInfoCommand() *cobra.Command { Invokes: []fx.Option{ fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) { lc.Append(fx.Hook{ - OnStart: func(ctx context.Context) error { + OnStart: func(_ context.Context) error { go func() { err := v.ShowInfo() if err != nil { if !errors.Is(err, context.Canceled) { log.Error("Failed to show info", "error", err) - ReportError("Failed to show info: %v", err) + ReportErrorf("Failed to show info: %v", err) os.Exit(1) } } @@ -63,7 +63,7 @@ func NewInfoCommand() *cobra.Command { return nil }, - OnStop: func(ctx context.Context) error { + OnStop: func(_ context.Context) error { v.Cancel() return nil diff --git a/internal/cli/prune.go b/internal/cli/prune.go index 2524580..950e7fd 100644 --- a/internal/cli/prune.go +++ b/internal/cli/prune.go @@ -31,7 +31,7 @@ Snapshot create --prune and snapshot remove run the same cleanup automatically; this command is the manual entry point for the same work (e.g. after a crashed backup or to reclaim storage).`, Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, _ []string) error { // Use unified config resolution configPath, err := ResolveConfigPath() if err != nil { @@ -43,7 +43,7 @@ work (e.g. after a crashed backup or to reclaim storage).`, return RunWithApp(cmd.Context(), AppOptions{ ConfigPath: configPath, - LogOptions: log.LogOptions{ + LogOptions: log.Options{ Verbose: rootFlags.Verbose, Debug: rootFlags.Debug, Quiet: rootFlags.Quiet || opts.JSON, @@ -52,7 +52,7 @@ work (e.g. after a crashed backup or to reclaim storage).`, Invokes: []fx.Option{ fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) { lc.Append(fx.Hook{ - OnStart: func(ctx context.Context) error { + OnStart: func(_ context.Context) error { // Start the prune operation in a goroutine go func() { // Run the prune operation @@ -61,7 +61,7 @@ work (e.g. after a crashed backup or to reclaim storage).`, if !errors.Is(err, context.Canceled) { if !opts.JSON { log.Error("Prune operation failed", "error", err) - ReportError("Prune failed: %v", err) + ReportErrorf("Prune failed: %v", err) } os.Exit(1) @@ -77,7 +77,7 @@ work (e.g. after a crashed backup or to reclaim storage).`, return nil }, - OnStop: func(ctx context.Context) error { + OnStop: func(_ context.Context) error { log.Debug("Stopping prune operation") v.Cancel() diff --git a/internal/cli/remote.go b/internal/cli/remote.go index 8ccf499..61987a3 100644 --- a/internal/cli/remote.go +++ b/internal/cli/remote.go @@ -11,6 +11,10 @@ import ( "sneak.berlin/go/vaultik/internal/vaultik" ) +// errNukeNeedsForce guards the destructive 'remote nuke' subcommand. +var errNukeNeedsForce = errors.New( + "remote nuke requires --force (this deletes ALL remote snapshots and blobs)") + // NewRemoteCommand creates the remote command and subcommands func NewRemoteCommand() *cobra.Command { cmd := &cobra.Command{ @@ -39,61 +43,20 @@ empty and the next backup starts from scratch. This is destructive and irreversible. Requires --force.`, Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, _ []string) error { if !force { - return errors.New("remote nuke requires --force (this deletes ALL remote snapshots and blobs)") + return errNukeNeedsForce } - configPath, err := ResolveConfigPath() - if err != nil { - return err - } - - rootFlags := GetRootFlags() - - return RunWithApp(cmd.Context(), AppOptions{ - ConfigPath: configPath, - LogOptions: log.LogOptions{ - Verbose: rootFlags.Verbose, - Debug: rootFlags.Debug, - Quiet: rootFlags.Quiet, - }, - Modules: []fx.Option{}, - Invokes: []fx.Option{ - fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) { - lc.Append(fx.Hook{ - OnStart: func(ctx context.Context) error { - go func() { - err := v.NukeRemote(true) - if err != nil { - if !errors.Is(err, context.Canceled) { - log.Error("Remote nuke failed", "error", err) - ReportError("Remote nuke failed: %v", err) - os.Exit(1) - } - } - - err = v.Shutdowner.Shutdown() - if err != nil { - log.Error("Failed to shutdown", "error", err) - } - }() - - return nil - }, - OnStop: func(ctx context.Context) error { - v.Cancel() - - return nil - }, - }) - }), - }, - }) + return runVaultikApp(cmd, false, false, "Remote nuke failed", + func(v *vaultik.Vaultik) error { + return v.NukeRemote(true) + }) }, } - cmd.Flags().BoolVar(&force, "force", false, "Required: confirm destruction of ALL remote data") + cmd.Flags().BoolVar(&force, "force", false, + "Required: confirm destruction of ALL remote data") return cmd } @@ -111,7 +74,7 @@ func newRemoteInfoCommand() *cobra.Command { - Count and size of referenced blobs (from all manifests) - Count and size of orphaned blobs (not referenced by any manifest)`, Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, _ []string) error { // Use unified config resolution configPath, err := ResolveConfigPath() if err != nil { @@ -122,7 +85,7 @@ func newRemoteInfoCommand() *cobra.Command { return RunWithApp(cmd.Context(), AppOptions{ ConfigPath: configPath, - LogOptions: log.LogOptions{ + LogOptions: log.Options{ Verbose: rootFlags.Verbose, Debug: rootFlags.Debug, Quiet: rootFlags.Quiet || jsonOutput, @@ -131,14 +94,14 @@ func newRemoteInfoCommand() *cobra.Command { Invokes: []fx.Option{ fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) { lc.Append(fx.Hook{ - OnStart: func(ctx context.Context) error { + OnStart: func(_ context.Context) error { go func() { err := v.RemoteInfo(jsonOutput) if err != nil { if !errors.Is(err, context.Canceled) { if !jsonOutput { log.Error("Failed to get remote info", "error", err) - ReportError("Failed to get remote info: %v", err) + ReportErrorf("Failed to get remote info: %v", err) } os.Exit(1) @@ -153,7 +116,7 @@ func newRemoteInfoCommand() *cobra.Command { return nil }, - OnStop: func(ctx context.Context) error { + OnStop: func(_ context.Context) error { v.Cancel() return nil diff --git a/internal/cli/root.go b/internal/cli/root.go index 315c5e4..0c8c6ce 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -1,6 +1,7 @@ package cli import ( + "errors" "fmt" "os" "path/filepath" @@ -10,6 +11,9 @@ import ( "github.com/spf13/cobra" ) +// errConfigNotFound is wrapped by all config-resolution failures. +var errConfigNotFound = errors.New("config file not found") + // RootFlags holds global flags that apply to all commands. // These flags are defined on the root command and inherited by all subcommands. type RootFlags struct { @@ -20,6 +24,7 @@ type RootFlags struct { SkipErrors bool } +//nolint:gochecknoglobals // cobra persistent flags bind to package state var rootFlags RootFlags // NewRootCommand creates the root cobra command for the vaultik CLI. @@ -34,20 +39,26 @@ public keys and uploads to S3-compatible storage. No private keys are needed on the source system.`, SilenceUsage: true, // Bare 'vaultik' (no subcommand): print help. The banner is - // printed once at process startup by CLIEntry, before cobra + // printed once at process startup by Entry, before cobra // parses arguments, so it appears even when cobra rejects // args (e.g. "requires at least 2 arg(s)") and on --help. - Run: func(cmd *cobra.Command, args []string) { + Run: func(cmd *cobra.Command, _ []string) { _ = cmd.Help() }, } // Add global flags - cmd.PersistentFlags().StringVar(&rootFlags.ConfigPath, "config", "", "Path to config file (default: $VAULTIK_CONFIG or platform config dir)") - cmd.PersistentFlags().BoolVarP(&rootFlags.Verbose, "verbose", "v", false, "Enable verbose output") - cmd.PersistentFlags().BoolVar(&rootFlags.Debug, "debug", false, "Enable debug output") - cmd.PersistentFlags().BoolVarP(&rootFlags.Quiet, "quiet", "q", false, "Suppress non-error output") - cmd.PersistentFlags().BoolVar(&rootFlags.SkipErrors, "skip-errors", false, "Continue past per-file errors instead of aborting (applies to snapshot create and restore)") + cmd.PersistentFlags().StringVar(&rootFlags.ConfigPath, "config", "", + "Path to config file (default: $VAULTIK_CONFIG or platform config dir)") + cmd.PersistentFlags().BoolVarP(&rootFlags.Verbose, "verbose", "v", false, + "Enable verbose output") + cmd.PersistentFlags().BoolVar(&rootFlags.Debug, "debug", false, + "Enable debug output") + cmd.PersistentFlags().BoolVarP(&rootFlags.Quiet, "quiet", "q", false, + "Suppress non-error output") + cmd.PersistentFlags().BoolVar(&rootFlags.SkipErrors, "skip-errors", false, + "Continue past per-file errors instead of aborting "+ + "(applies to snapshot create and restore)") // Add subcommands cmd.AddCommand( @@ -70,22 +81,29 @@ func GetRootFlags() RootFlags { } // ResolveConfigPath resolves the config file path from flags, environment, or default. -// Search order: --config flag, VAULTIK_CONFIG env, XDG config dir, /etc/vaultik/config.yml. +// Search order: --config flag, VAULTIK_CONFIG env, XDG config dir, +// /etc/vaultik/config.yml. // Explicit paths from --config and $VAULTIK_CONFIG are checked for existence // so the user gets a clear error instead of a downstream YAML parser failure. func ResolveConfigPath() (string, error) { if path := rootFlags.ConfigPath; path != "" { _, err := os.Stat(path) if err != nil { - return "", fmt.Errorf("config file from --config not found: %s (run 'vaultik config init --config %s' to create it)", path, path) + return "", fmt.Errorf( + "%w: from --config: %s (run 'vaultik config init --config %s' to create it)", + errConfigNotFound, path, path) } return path, nil } if path := os.Getenv("VAULTIK_CONFIG"); path != "" { - if _, err := os.Stat(path); err != nil { - return "", fmt.Errorf("config file from $VAULTIK_CONFIG not found: %s (unset VAULTIK_CONFIG, point it at an existing file, or run 'vaultik config init')", path) + _, err := os.Stat(path) //nolint:gosec // G703: path is operator-supplied by design + if err != nil { + return "", fmt.Errorf( + "%w: from $VAULTIK_CONFIG: %s (unset VAULTIK_CONFIG, point it at "+ + "an existing file, or run 'vaultik config init')", + errConfigNotFound, path) } return path, nil @@ -98,7 +116,10 @@ func ResolveConfigPath() (string, error) { } } - return "", fmt.Errorf("no config file found at %s (run 'vaultik config init' to create the default config, or pass --config )", strings.Join(defaultConfigPaths(), " or ")) + return "", fmt.Errorf( + "%w: searched %s (run 'vaultik config init' to create the default "+ + "config, or pass --config )", + errConfigNotFound, strings.Join(defaultConfigPaths(), " or ")) } // defaultConfigPaths returns the ordered list of config paths to search. diff --git a/internal/cli/snapshot.go b/internal/cli/snapshot.go index 8f5b1b5..9d94daa 100644 --- a/internal/cli/snapshot.go +++ b/internal/cli/snapshot.go @@ -12,6 +12,32 @@ import ( "sneak.berlin/go/vaultik/internal/vaultik" ) +var ( + errSnapshotIDRequired = errors.New("snapshot ID required") + errWrongArgCount = errors.New("wrong argument count") + errPurgeCriteriaNeeded = errors.New( + "must specify either --keep-latest or --older-than") + errPurgeCriteriaBoth = errors.New( + "cannot specify both --keep-latest and --older-than") +) + +// requireSnapshotIDArg validates that exactly one positional argument +// (the snapshot ID) was supplied, printing help otherwise. +func requireSnapshotIDArg(cmd *cobra.Command, args []string) error { + if len(args) != 1 { + _ = cmd.Help() + + if len(args) == 0 { + return errSnapshotIDRequired + } + + return fmt.Errorf("%w: expected 1 argument, got %d", + errWrongArgCount, len(args)) + } + + return nil +} + // NewSnapshotCommand creates the snapshot command and subcommands func NewSnapshotCommand() *cobra.Command { cmd := &cobra.Command{ @@ -62,7 +88,7 @@ specifying a path using --config or by setting VAULTIK_CONFIG to a path.`, return RunWithApp(cmd.Context(), AppOptions{ ConfigPath: configPath, - LogOptions: log.LogOptions{ + LogOptions: log.Options{ Verbose: rootFlags.Verbose, Debug: rootFlags.Debug, Cron: opts.Cron, @@ -72,7 +98,7 @@ specifying a path using --config or by setting VAULTIK_CONFIG to a path.`, Invokes: []fx.Option{ fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) { lc.Append(fx.Hook{ - OnStart: func(ctx context.Context) error { + OnStart: func(_ context.Context) error { // Start the snapshot creation in a goroutine go func() { // --cron suppression is wired through v.UI by setupGlobals. @@ -80,7 +106,7 @@ specifying a path using --config or by setting VAULTIK_CONFIG to a path.`, if err != nil { if !errors.Is(err, context.Canceled) { log.Error("Snapshot creation failed", "error", err) - ReportError("Snapshot creation failed: %v", err) + ReportErrorf("Snapshot creation failed: %v", err) os.Exit(1) } } @@ -94,7 +120,7 @@ specifying a path using --config or by setting VAULTIK_CONFIG to a path.`, return nil }, - OnStop: func(ctx context.Context) error { + OnStop: func(_ context.Context) error { log.Debug("Stopping snapshot creation") // Cancel the Vaultik context v.Cancel() @@ -108,9 +134,14 @@ specifying a path using --config or by setting VAULTIK_CONFIG to a path.`, }, } - cmd.Flags().BoolVar(&opts.Cron, "cron", false, "Run in cron mode (silent unless error)") - cmd.Flags().BoolVar(&opts.Prune, "prune", false, "After backup, drop older snapshots of the same name and remove orphaned blobs") - cmd.Flags().StringVar(&opts.KeepNewerThan, "keep-newer-than", "", "With --prune: keep snapshots newer than this duration (e.g. 4w, 30d, 6mo) instead of only the latest") + cmd.Flags().BoolVar(&opts.Cron, "cron", false, + "Run in cron mode (silent unless error)") + cmd.Flags().BoolVar(&opts.Prune, "prune", false, + "After backup, drop older snapshots of the same name and remove "+ + "orphaned blobs") + cmd.Flags().StringVar(&opts.KeepNewerThan, "keep-newer-than", "", + "With --prune: keep snapshots newer than this duration "+ + "(e.g. 4w, 30d, 6mo) instead of only the latest") return cmd } @@ -125,54 +156,12 @@ func newSnapshotListCommand() *cobra.Command { Short: "List all snapshots", Long: "Lists all snapshots with their ID, timestamp, and compressed size", Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { - // Use unified config resolution - configPath, err := ResolveConfigPath() - if err != nil { - return err - } - - rootFlags := GetRootFlags() - - return RunWithApp(cmd.Context(), AppOptions{ - ConfigPath: configPath, - LogOptions: log.LogOptions{ - Verbose: rootFlags.Verbose, - Debug: rootFlags.Debug, - Quiet: rootFlags.Quiet, - }, - Modules: []fx.Option{}, - Invokes: []fx.Option{ - fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) { - lc.Append(fx.Hook{ - OnStart: func(ctx context.Context) error { - go func() { - err := v.ListSnapshots(jsonOutput) - if err != nil { - if !errors.Is(err, context.Canceled) { - log.Error("Failed to list snapshots", "error", err) - ReportError("Failed to list snapshots: %v", err) - os.Exit(1) - } - } - - err = v.Shutdowner.Shutdown() - if err != nil { - log.Error("Failed to shutdown", "error", err) - } - }() - - return nil - }, - OnStop: func(ctx context.Context) error { - v.Cancel() - - return nil - }, - }) - }), - }, - }) + RunE: func(cmd *cobra.Command, _ []string) error { + return runVaultikApp(cmd, false, false, + "Failed to list snapshots", + func(v *vaultik.Vaultik) error { + return v.ListSnapshots(jsonOutput) + }) }, } @@ -194,70 +183,31 @@ Retention is per-snapshot-name: --keep-latest keeps the latest of each configured snapshot name, not the latest globally. Use --snapshot to restrict the operation to specific snapshot names.`, Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, _ []string) error { // Validate flags if !opts.KeepLatest && opts.OlderThan == "" { - return errors.New("must specify either --keep-latest or --older-than") + return errPurgeCriteriaNeeded } if opts.KeepLatest && opts.OlderThan != "" { - return errors.New("cannot specify both --keep-latest and --older-than") + return errPurgeCriteriaBoth } - // Use unified config resolution - configPath, err := ResolveConfigPath() - if err != nil { - return err - } - - rootFlags := GetRootFlags() - - return RunWithApp(cmd.Context(), AppOptions{ - ConfigPath: configPath, - LogOptions: log.LogOptions{ - Verbose: rootFlags.Verbose, - Debug: rootFlags.Debug, - Quiet: rootFlags.Quiet, - }, - Modules: []fx.Option{}, - Invokes: []fx.Option{ - fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) { - lc.Append(fx.Hook{ - OnStart: func(ctx context.Context) error { - go func() { - err := v.PurgeSnapshotsWithOptions(opts) - if err != nil { - if !errors.Is(err, context.Canceled) { - log.Error("Failed to purge snapshots", "error", err) - ReportError("Failed to purge snapshots: %v", err) - os.Exit(1) - } - } - - err = v.Shutdowner.Shutdown() - if err != nil { - log.Error("Failed to shutdown", "error", err) - } - }() - - return nil - }, - OnStop: func(ctx context.Context) error { - v.Cancel() - - return nil - }, - }) - }), - }, - }) + return runVaultikApp(cmd, false, false, + "Failed to purge snapshots", + func(v *vaultik.Vaultik) error { + return v.PurgeSnapshotsWithOptions(opts) + }) }, } - cmd.Flags().BoolVar(&opts.KeepLatest, "keep-latest", false, "Keep only the latest snapshot of each name") - cmd.Flags().StringVar(&opts.OlderThan, "older-than", "", "Remove snapshots older than duration (e.g., 30d, 6m, 1y)") + cmd.Flags().BoolVar(&opts.KeepLatest, "keep-latest", false, + "Keep only the latest snapshot of each name") + cmd.Flags().StringVar(&opts.OlderThan, "older-than", "", + "Remove snapshots older than duration (e.g., 30d, 6m, 1y)") cmd.Flags().BoolVar(&opts.Force, "force", false, "Skip confirmation prompt") - cmd.Flags().StringArrayVar(&opts.Names, "snapshot", nil, "Restrict to snapshots with these names (repeat for multiple)") + cmd.Flags().StringArrayVar(&opts.Names, "snapshot", nil, + "Restrict to snapshots with these names (repeat for multiple)") return cmd } @@ -270,19 +220,7 @@ func newSnapshotVerifyCommand() *cobra.Command { Use: "verify ", Short: "Verify snapshot integrity", Long: "Verifies that all blobs referenced in a snapshot exist", - Args: func(cmd *cobra.Command, args []string) error { - if len(args) != 1 { - _ = cmd.Help() - - if len(args) == 0 { - return errors.New("snapshot ID required") - } - - return fmt.Errorf("expected 1 argument, got %d", len(args)) - } - - return nil - }, + Args: requireSnapshotIDArg, RunE: func(cmd *cobra.Command, args []string) error { snapshotID := args[0] @@ -296,7 +234,7 @@ func newSnapshotVerifyCommand() *cobra.Command { return RunWithApp(cmd.Context(), AppOptions{ ConfigPath: configPath, - LogOptions: log.LogOptions{ + LogOptions: log.Options{ Verbose: rootFlags.Verbose, Debug: rootFlags.Debug, Quiet: rootFlags.Quiet || opts.JSON, @@ -305,14 +243,14 @@ func newSnapshotVerifyCommand() *cobra.Command { Invokes: []fx.Option{ fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) { lc.Append(fx.Hook{ - OnStart: func(ctx context.Context) error { + OnStart: func(_ context.Context) error { go func() { err := v.VerifySnapshotWithOptions(snapshotID, opts) if err != nil { if !errors.Is(err, context.Canceled) { if !opts.JSON { log.Error("Verification failed", "error", err) - ReportError("Verification failed: %v", err) + ReportErrorf("Verification failed: %v", err) } os.Exit(1) @@ -327,7 +265,7 @@ func newSnapshotVerifyCommand() *cobra.Command { return nil }, - OnStop: func(ctx context.Context) error { + OnStop: func(_ context.Context) error { v.Cancel() return nil @@ -371,77 +309,24 @@ is reachable to finish remote cleanup. To wipe the entire destination store and start over, use 'vaultik remote nuke --force' — it is the single supported entry point for that.`, - Args: func(cmd *cobra.Command, args []string) error { - if len(args) != 1 { - _ = cmd.Help() - - if len(args) == 0 { - return errors.New("snapshot ID required") - } - - return fmt.Errorf("expected 1 argument, got %d", len(args)) - } - - return nil - }, + Args: requireSnapshotIDArg, RunE: func(cmd *cobra.Command, args []string) error { - // Use unified config resolution - configPath, err := ResolveConfigPath() - if err != nil { - return err - } + return runVaultikApp(cmd, opts.JSON, opts.JSON, + "Failed to remove snapshot", + func(v *vaultik.Vaultik) error { + _, err := v.RemoveSnapshot(args[0], opts) - rootFlags := GetRootFlags() - - return RunWithApp(cmd.Context(), AppOptions{ - ConfigPath: configPath, - LogOptions: log.LogOptions{ - Verbose: rootFlags.Verbose, - Debug: rootFlags.Debug, - Quiet: rootFlags.Quiet || opts.JSON, - }, - Modules: []fx.Option{}, - Invokes: []fx.Option{ - fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) { - lc.Append(fx.Hook{ - OnStart: func(ctx context.Context) error { - go func() { - _, err := v.RemoveSnapshot(args[0], opts) - if err != nil { - if !errors.Is(err, context.Canceled) { - if !opts.JSON { - log.Error("Failed to remove snapshot", "error", err) - ReportError("Failed to remove snapshot: %v", err) - } - - os.Exit(1) - } - } - - err = v.Shutdowner.Shutdown() - if err != nil { - log.Error("Failed to shutdown", "error", err) - } - }() - - return nil - }, - OnStop: func(ctx context.Context) error { - v.Cancel() - - return nil - }, - }) - }), - }, - }) + return err + }) }, } cmd.Flags().BoolVarP(&opts.Force, "force", "f", false, "Skip confirmation prompt") - cmd.Flags().BoolVar(&opts.DryRun, "dry-run", false, "Show what would be removed without removing") + cmd.Flags().BoolVar(&opts.DryRun, "dry-run", false, + "Show what would be removed without removing") cmd.Flags().BoolVar(&opts.JSON, "json", false, "Output result as JSON") - cmd.Flags().BoolVar(&opts.LocalOnly, "local-only", false, "Skip remote cleanup; only touch the local index") + cmd.Flags().BoolVar(&opts.LocalOnly, "local-only", false, + "Skip remote cleanup; only touch the local index") return cmd } diff --git a/internal/cli/snapshot_restore.go b/internal/cli/snapshot_restore.go index 47881a3..cf28e8e 100644 --- a/internal/cli/snapshot_restore.go +++ b/internal/cli/snapshot_restore.go @@ -14,6 +14,10 @@ import ( "sneak.berlin/go/vaultik/internal/vaultik" ) +// restoreMinArgs is the minimum positional argument count of +// `snapshot restore [paths...]`. +const restoreMinArgs = 2 + // RestoreOptions contains options for the restore command type RestoreOptions struct { TargetDir string @@ -39,31 +43,36 @@ func newSnapshotRestoreCommand() *cobra.Command { Short: "Restore files from a snapshot", Long: `Download and decrypt files from a backup snapshot. -This command will restore files from the specified snapshot to the target directory. +This command will restore files from the specified snapshot to the +target directory. If no paths are specified, all files are restored. If paths are specified, only matching files/directories are restored. -Requires the VAULTIK_AGE_SECRET_KEY environment variable to be set with the age private key. +Requires the VAULTIK_AGE_SECRET_KEY environment variable to be set with +the age private key. Examples: # Restore entire snapshot vaultik snapshot restore myhost_docs_2025-01-01T12:00:00Z /restore # Restore specific file - vaultik snapshot restore myhost_docs_2025-01-01T12:00:00Z /restore /home/user/important.txt + vaultik snapshot restore myhost_docs_2025-01-01T12:00:00Z /restore \ + /home/user/important.txt # Restore specific directory - vaultik snapshot restore myhost_docs_2025-01-01T12:00:00Z /restore /home/user/documents/ + vaultik snapshot restore myhost_docs_2025-01-01T12:00:00Z /restore \ + /home/user/documents/ # Restore and verify all files vaultik snapshot restore --verify myhost_docs_2025-01-01T12:00:00Z /restore`, - Args: cobra.MinimumNArgs(2), + Args: cobra.MinimumNArgs(restoreMinArgs), RunE: func(cmd *cobra.Command, args []string) error { return runRestore(cmd, args, opts) }, } - cmd.Flags().BoolVar(&opts.Verify, "verify", false, "Verify restored files by checking chunk hashes") + cmd.Flags().BoolVar(&opts.Verify, "verify", false, + "Verify restored files by checking chunk hashes") return cmd } @@ -73,8 +82,8 @@ func runRestore(cmd *cobra.Command, args []string, opts *RestoreOptions) error { snapshotID := args[0] opts.TargetDir = args[1] - if len(args) > 2 { - opts.Paths = args[2:] + if len(args) > restoreMinArgs { + opts.Paths = args[restoreMinArgs:] } // Use unified config resolution @@ -88,7 +97,7 @@ func runRestore(cmd *cobra.Command, args []string, opts *RestoreOptions) error { return RunWithApp(cmd.Context(), AppOptions{ ConfigPath: configPath, - LogOptions: log.LogOptions{ + LogOptions: log.Options{ Verbose: rootFlags.Verbose, Debug: rootFlags.Debug, Quiet: rootFlags.Quiet, @@ -121,7 +130,7 @@ func buildRestoreInvokes(snapshotID string, opts *RestoreOptions) []fx.Option { return []fx.Option{ fx.Invoke(func(app *RestoreApp, lc fx.Lifecycle) { lc.Append(fx.Hook{ - OnStart: func(ctx context.Context) error { + OnStart: func(_ context.Context) error { // Start the restore operation in a goroutine go func() { // Run the restore operation @@ -137,7 +146,7 @@ func buildRestoreInvokes(snapshotID string, opts *RestoreOptions) []fx.Option { if err != nil { if !errors.Is(err, context.Canceled) { log.Error("Restore operation failed", "error", err) - ReportError("Restore failed: %v", err) + ReportErrorf("Restore failed: %v", err) os.Exit(1) } } @@ -151,7 +160,7 @@ func buildRestoreInvokes(snapshotID string, opts *RestoreOptions) []fx.Option { return nil }, - OnStop: func(ctx context.Context) error { + OnStop: func(_ context.Context) error { log.Debug("Stopping restore operation") app.Vaultik.Cancel() diff --git a/internal/cli/vaultik_snapshot_types.go b/internal/cli/vaultik_snapshot_types.go index 9d6b7c6..1f9d4dd 100644 --- a/internal/cli/vaultik_snapshot_types.go +++ b/internal/cli/vaultik_snapshot_types.go @@ -3,6 +3,8 @@ package cli import "time" // SnapshotInfo represents snapshot information for listing +// +//nolint:tagliatelle // snake_case is the established output format type SnapshotInfo struct { ID string `json:"id"` Timestamp time.Time `json:"timestamp"` diff --git a/internal/cli/version.go b/internal/cli/version.go index 7c6d14c..c8cec85 100644 --- a/internal/cli/version.go +++ b/internal/cli/version.go @@ -2,6 +2,7 @@ package cli import ( "fmt" + "os" "runtime" "github.com/spf13/cobra" @@ -15,21 +16,25 @@ func NewVersionCommand() *cobra.Command { Short: "Print version information", Long: `Print version, git commit, and build information for vaultik.`, Args: cobra.NoArgs, - Run: func(cmd *cobra.Command, args []string) { - fmt.Printf("vaultik %s\n", globals.Version) - fmt.Printf(" commit: %s\n", globals.Commit) - fmt.Printf(" build date: %s\n", globals.CommitDate) - fmt.Printf(" go: %s\n", runtime.Version()) - fmt.Printf(" os/arch: %s/%s\n", runtime.GOOS, runtime.GOARCH) - fmt.Printf(" author: %s\n", globals.Author) - fmt.Printf(" homepage: %s\n", globals.Homepage) - fmt.Printf(" license: %s\n", globals.License) + Run: func(_ *cobra.Command, _ []string) { + _, _ = fmt.Fprintf(os.Stdout, "vaultik %s\n", globals.Version) + _, _ = fmt.Fprintf(os.Stdout, " commit: %s\n", globals.Commit) + _, _ = fmt.Fprintf(os.Stdout, " build date: %s\n", globals.CommitDate) + _, _ = fmt.Fprintf(os.Stdout, " go: %s\n", runtime.Version()) + _, _ = fmt.Fprintf(os.Stdout, " os/arch: %s/%s\n", + runtime.GOOS, runtime.GOARCH) + _, _ = fmt.Fprintf(os.Stdout, " author: %s\n", globals.Author) + _, _ = fmt.Fprintf(os.Stdout, " homepage: %s\n", globals.Homepage) + _, _ = fmt.Fprintf(os.Stdout, " license: %s\n", globals.License) if globals.Version == "dev" { - fmt.Println() - fmt.Println("This is a development build (no version information embedded).") - fmt.Println("Build a release binary with 'make vaultik' or download from") - fmt.Println("https://sneak.berlin/go/vaultik for embedded version metadata.") + _, _ = fmt.Fprintln(os.Stdout) + _, _ = fmt.Fprintln(os.Stdout, + "This is a development build (no version information embedded).") + _, _ = fmt.Fprintln(os.Stdout, + "Build a release binary with 'make vaultik' or download from") + _, _ = fmt.Fprintln(os.Stdout, + "https://sneak.berlin/go/vaultik for embedded version metadata.") } }, } diff --git a/internal/config/config.go b/internal/config/config.go index 3462bab..27e7196 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1,3 +1,6 @@ +// Package config loads, validates, and provides the vaultik YAML +// configuration, including snapshot definitions, encryption recipients, +// and storage settings. package config import ( @@ -18,6 +21,38 @@ import ( const appName = "vaultik" +// Defaults and validation bounds for tunable settings. +const ( + defaultBlobSizeLimit = Size(10 * 1024 * 1024 * 1024) // 10GB + defaultChunkSize = Size(10 * 1024 * 1024) // 10MB + defaultS3PartSize = Size(5 * 1024 * 1024) // 5MB + defaultCompressionLevel = 3 + minChunkSize = 1024 * 1024 // 1MB + minCompressionLevel = 1 + maxCompressionLevel = 19 +) + +// Sentinel validation errors. +var ( + errNoConfigPath = errors.New("config path not provided") + errNoAgeRecipients = errors.New( + "at least one age_recipient is required (generate with: age-keygen)") + errNoSnapshots = errors.New( + "at least one snapshot must be configured (see config.example.yml)") + errSnapshotNoPaths = errors.New("snapshot must have at least one path") + errChunkSizeTooSmall = errors.New("chunk_size must be at least 1MB") + errBlobSizeTooSmall = errors.New("blob_size_limit must be at least chunk_size") + errBadCompression = errors.New("compression_level must be between 1 and 19") + errBadStorageScheme = errors.New( + "storage_url must start with s3://, file://, or rclone://") + errStorageNotConfigured = errors.New( + "storage not configured; set storage_url or provide s3.endpoint + " + + "s3.bucket + credentials") + errS3BucketRequired = errors.New("s3.bucket is required (or set storage_url)") + errS3KeyIDRequired = errors.New("s3.access_key_id is required") + errS3SecretRequired = errors.New("s3.secret_access_key is required") +) + // expandTilde expands ~ at the start of a path to the user's home directory. func expandTilde(path string) string { if path == "~" { @@ -90,12 +125,15 @@ func (c *Config) SnapshotNames() []string { // It defines all settings for backup operations, including source directories, // encryption recipients, storage configuration, and performance tuning parameters. // Configuration is typically loaded from a YAML file. +// +//nolint:tagliatelle // snake_case is the established config-file format type Config struct { - AgeRecipients []string `yaml:"age_recipients"` - AgeSecretKey string `yaml:"age_secret_key"` - BlobSizeLimit Size `yaml:"blob_size_limit"` - ChunkSize Size `yaml:"chunk_size"` - Exclude []string `yaml:"exclude"` // Global excludes applied to all snapshots + AgeRecipients []string `yaml:"age_recipients"` + AgeSecretKey string `yaml:"age_secret_key"` + BlobSizeLimit Size `yaml:"blob_size_limit"` + ChunkSize Size `yaml:"chunk_size"` + // Exclude holds global excludes applied to all snapshots. + Exclude []string `yaml:"exclude"` Hostname string `yaml:"hostname"` IndexPath string `yaml:"index_path"` S3 S3Config `yaml:"s3"` @@ -107,13 +145,16 @@ type Config struct { // Supported formats: // - s3://bucket/prefix?endpoint=host®ion=us-east-1 // - file:///path/to/backup - // For S3 URLs, credentials are still read from s3.access_key_id and s3.secret_access_key. + // For S3 URLs, credentials are still read from s3.access_key_id + // and s3.secret_access_key. StorageURL string `yaml:"storage_url"` } // S3Config represents S3 storage configuration for backup storage. // It supports both AWS S3 and S3-compatible storage services. // All fields except UseSSL and PartSize are required. +// +//nolint:tagliatelle // snake_case is the established config-file format type S3Config struct { Endpoint string `yaml:"endpoint"` Bucket string `yaml:"bucket"` @@ -125,17 +166,17 @@ type S3Config struct { PartSize Size `yaml:"part_size"` } -// ConfigPath wraps the config file path for fx dependency injection. +// Path wraps the config file path for fx dependency injection. // This type allows the config file path to be injected as a distinct type // rather than a plain string, avoiding conflicts with other string dependencies. -type ConfigPath string +type Path string // New creates a new Config instance by loading from the specified path. // This function is used by the fx dependency injection framework. // Returns an error if the path is empty or if loading fails. -func New(path ConfigPath) (*Config, error) { +func New(path Path) (*Config, error) { if path == "" { - return nil, errors.New("config path not provided") + return nil, errNoConfigPath } cfg, err := Load(string(path)) @@ -160,10 +201,10 @@ func Load(path string) (*Config, error) { cfg := &Config{ // Set defaults - BlobSizeLimit: Size(10 * 1024 * 1024 * 1024), // 10GB - ChunkSize: Size(10 * 1024 * 1024), // 10MB + BlobSizeLimit: defaultBlobSizeLimit, + ChunkSize: defaultChunkSize, IndexPath: filepath.Join(xdg.DataHome, appName, "index.sqlite"), - CompressionLevel: 3, + CompressionLevel: defaultCompressionLevel, } // Convert smartconfig data to YAML then unmarshal @@ -218,12 +259,13 @@ func Load(path string) (*Config, error) { } if cfg.S3.PartSize == 0 { - cfg.S3.PartSize = Size(5 * 1024 * 1024) // 5MB + cfg.S3.PartSize = defaultS3PartSize } // Check config file permissions (warn if world or group readable) - info, err := os.Stat(path) - if err == nil { + //nolint:gosec // G703: config path is operator-supplied by design + info, statErr := os.Stat(path) + if statErr == nil { mode := info.Mode().Perm() if mode&0044 != 0 { // group or world readable log.Warn("Config file has insecure permissions (contains S3 credentials)", @@ -252,16 +294,16 @@ func Load(path string) (*Config, error) { // Returns an error describing the first validation failure encountered. func (c *Config) Validate() error { if len(c.AgeRecipients) == 0 { - return errors.New("at least one age_recipient is required (generate with: age-keygen)") + return errNoAgeRecipients } if len(c.Snapshots) == 0 { - return errors.New("at least one snapshot must be configured (see config.example.yml)") + return errNoSnapshots } for name, snap := range c.Snapshots { if len(snap.Paths) == 0 { - return fmt.Errorf("snapshot %q must have at least one path", name) + return fmt.Errorf("%w: %q", errSnapshotNoPaths, name) } } @@ -271,16 +313,17 @@ func (c *Config) Validate() error { return err } - if c.ChunkSize.Int64() < 1024*1024 { // 1MB minimum - return errors.New("chunk_size must be at least 1MB") + if c.ChunkSize.Int64() < minChunkSize { + return errChunkSizeTooSmall } if c.BlobSizeLimit.Int64() < c.ChunkSize.Int64() { - return errors.New("blob_size_limit must be at least chunk_size") + return errBlobSizeTooSmall } - if c.CompressionLevel < 1 || c.CompressionLevel > 19 { - return errors.New("compression_level must be between 1 and 19") + if c.CompressionLevel < minCompressionLevel || + c.CompressionLevel > maxCompressionLevel { + return errBadCompression } return nil @@ -292,53 +335,56 @@ func (c *Config) Validate() error { // If StorageURL is not set, legacy S3 configuration is required. func (c *Config) validateStorage() error { if c.StorageURL != "" { - // URL-based configuration - if strings.HasPrefix(c.StorageURL, "file://") { - // File storage doesn't need S3 credentials - return nil - } - - if strings.HasPrefix(c.StorageURL, "s3://") { - // S3 storage needs credentials - if c.S3.AccessKeyID == "" { - return errors.New("s3.access_key_id is required for s3:// URLs") - } - - if c.S3.SecretAccessKey == "" { - return errors.New("s3.secret_access_key is required for s3:// URLs") - } - - return nil - } - - if strings.HasPrefix(c.StorageURL, "rclone://") { - // Rclone storage uses rclone's own config - return nil - } - - return errors.New("storage_url must start with s3://, file://, or rclone://") + return c.validateStorageURL() } // Legacy S3 configuration if c.S3.Endpoint == "" { - return errors.New("storage not configured; set storage_url or provide s3.endpoint + s3.bucket + credentials") + return errStorageNotConfigured } if c.S3.Bucket == "" { - return errors.New("s3.bucket is required (or set storage_url)") + return errS3BucketRequired } if c.S3.AccessKeyID == "" { - return errors.New("s3.access_key_id is required") + return errS3KeyIDRequired } if c.S3.SecretAccessKey == "" { - return errors.New("s3.secret_access_key is required") + return errS3SecretRequired } return nil } +// validateStorageURL validates URL-based storage configuration. File and +// rclone URLs need no credentials; S3 URLs require the legacy s3.* +// credential fields. +func (c *Config) validateStorageURL() error { + switch { + case strings.HasPrefix(c.StorageURL, "file://"): + // File storage doesn't need S3 credentials + return nil + case strings.HasPrefix(c.StorageURL, "rclone://"): + // Rclone storage uses rclone's own config + return nil + case strings.HasPrefix(c.StorageURL, "s3://"): + // S3 storage needs credentials + if c.S3.AccessKeyID == "" { + return fmt.Errorf("%w for s3:// URLs", errS3KeyIDRequired) + } + + if c.S3.SecretAccessKey == "" { + return fmt.Errorf("%w for s3:// URLs", errS3SecretRequired) + } + + return nil + default: + return errBadStorageScheme + } +} + // extractAgeSecretKey extracts the AGE-SECRET-KEY from the input using // the age library's parser, which handles comments and whitespace. func extractAgeSecretKey(input string) string { @@ -357,6 +403,8 @@ func extractAgeSecretKey(input string) string { // Module exports the config module for fx dependency injection. // It provides the Config type to other modules in the application. +// +//nolint:gochecknoglobals // fx module definitions are package globals var Module = fx.Module("config", fx.Provide(New), ) diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 4761e0a..0dfdd34 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -1,4 +1,4 @@ -package config +package config //nolint:testpackage // exercises unexported extractAgeSecretKey import ( "os" @@ -7,9 +7,12 @@ import ( ) const ( - TEST_SNEAK_AGE_PUBLIC_KEY = "age1278m9q7dp3chsh2dcy82qk27v047zywyvtxwnj4cvt0z65jw6a7q5dqhfj" - TEST_INTEGRATION_AGE_PUBLIC_KEY = "age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05gl0sjq9q9wjg" - TEST_INTEGRATION_AGE_PRIVATE_KEY = "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5" + testSneakAgePublicKey = "age1278m9q7dp3chsh2dcy82qk27v047zywyvt" + + "xwnj4cvt0z65jw6a7q5dqhfj" + testIntegrationAgePublicKey = "age1ezrjmfpwsc95svdg0y54mums3zevgzu" + + "0x0ecq2f7tp8a05gl0sjq9q9wjg" + testIntegrationAgePrivateKey = "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GX" + + "VEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5" ) func TestMain(m *testing.M) { @@ -25,8 +28,11 @@ func TestMain(m *testing.M) { os.Exit(code) } -// TestConfigLoad ensures the config package can be imported and basic functionality works +// TestConfigLoad ensures the config package can be imported and basic +// functionality works. func TestConfigLoad(t *testing.T) { + t.Parallel() + // Use the test config file configPath := os.Getenv("VAULTIK_CONFIG") if configPath == "" { @@ -44,8 +50,9 @@ func TestConfigLoad(t *testing.T) { t.Errorf("Expected 2 age recipients, got %d", len(cfg.AgeRecipients)) } - if cfg.AgeRecipients[0] != TEST_SNEAK_AGE_PUBLIC_KEY { - t.Errorf("Expected first age recipient to be %s, got '%s'", TEST_SNEAK_AGE_PUBLIC_KEY, cfg.AgeRecipients[0]) + if cfg.AgeRecipients[0] != testSneakAgePublicKey { + t.Errorf("Expected first age recipient to be %s, got '%s'", + testSneakAgePublicKey, cfg.AgeRecipients[0]) } if len(cfg.Snapshots) != 1 { @@ -62,11 +69,13 @@ func TestConfigLoad(t *testing.T) { } if testSnap.Paths[0] != "/tmp/vaultik-test-source" { - t.Errorf("Expected first path to be '/tmp/vaultik-test-source', got '%s'", testSnap.Paths[0]) + t.Errorf("Expected first path to be '/tmp/vaultik-test-source', got '%s'", + testSnap.Paths[0]) } if cfg.S3.Bucket != "vaultik-test-bucket" { - t.Errorf("Expected S3 bucket to be 'vaultik-test-bucket', got '%s'", cfg.S3.Bucket) + t.Errorf("Expected S3 bucket to be 'vaultik-test-bucket', got '%s'", + cfg.S3.Bucket) } if cfg.Hostname != "test-host" { @@ -76,19 +85,26 @@ func TestConfigLoad(t *testing.T) { // TestConfigFromEnv tests loading config path from environment variable func TestConfigFromEnv(t *testing.T) { + t.Parallel() + configPath := os.Getenv("VAULTIK_CONFIG") if configPath == "" { t.Skip("VAULTIK_CONFIG not set") } // Verify the file exists - if _, err := os.Stat(configPath); os.IsNotExist(err) { - t.Errorf("Config file does not exist at path from VAULTIK_CONFIG: %s", configPath) + //nolint:gosec // G703: test config path comes from the test environment + _, err := os.Stat(configPath) + if os.IsNotExist(err) { + t.Errorf("Config file does not exist at path from VAULTIK_CONFIG: %s", + configPath) } } // TestExtractAgeSecretKey tests extraction of AGE-SECRET-KEY from various inputs func TestExtractAgeSecretKey(t *testing.T) { + t.Parallel() + tests := []struct { name string input string @@ -96,36 +112,32 @@ func TestExtractAgeSecretKey(t *testing.T) { }{ { name: "plain key", - input: "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5", - expected: "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5", + input: testIntegrationAgePrivateKey, + expected: testIntegrationAgePrivateKey, }, { name: "key with trailing newline", - input: "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5\n", - expected: "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5", + input: testIntegrationAgePrivateKey + "\n", + expected: testIntegrationAgePrivateKey, }, { name: "full age-keygen output", - input: `# created: 2025-01-14T12:00:00Z -# public key: age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05gl0sjq9q9wjg -AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5 -`, - expected: "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5", + input: "# created: 2025-01-14T12:00:00Z\n" + + "# public key: " + testIntegrationAgePublicKey + "\n" + + testIntegrationAgePrivateKey + "\n", + expected: testIntegrationAgePrivateKey, }, { name: "age-keygen output with extra blank lines", - input: `# created: 2025-01-14T12:00:00Z -# public key: age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05gl0sjq9q9wjg - -AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5 - -`, - expected: "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5", + input: "# created: 2025-01-14T12:00:00Z\n" + + "# public key: " + testIntegrationAgePublicKey + "\n\n" + + testIntegrationAgePrivateKey + "\n\n", + expected: testIntegrationAgePrivateKey, }, { name: "key with leading whitespace", - input: " AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5 ", - expected: "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5", + input: " " + testIntegrationAgePrivateKey + " ", + expected: testIntegrationAgePrivateKey, }, { name: "empty input", @@ -141,9 +153,12 @@ AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5 for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + t.Parallel() + result := extractAgeSecretKey(tt.input) if result != tt.expected { - t.Errorf("extractAgeSecretKey(%q) = %q, want %q", tt.input, result, tt.expected) + t.Errorf("extractAgeSecretKey(%q) = %q, want %q", + tt.input, result, tt.expected) } }) } diff --git a/internal/config/size.go b/internal/config/size.go index 281e57d..8f21a87 100644 --- a/internal/config/size.go +++ b/internal/config/size.go @@ -3,13 +3,21 @@ package config import ( "errors" "fmt" + "math" "github.com/dustin/go-humanize" ) +var ( + errSizeType = errors.New("size must be a number or string") + errSizeTooLarge = errors.New("size exceeds maximum supported value") +) + // Size represents a byte size that can be specified in configuration files. // It can unmarshal from both numeric values (interpreted as bytes) and // human-readable strings like "10MB", "2.5GB", or "1TB". +// +//nolint:recvcheck // UnmarshalYAML requires a pointer; String/Int64 are value reads type Size int64 // UnmarshalYAML implements yaml.Unmarshaler for Size, allowing it to be @@ -31,7 +39,7 @@ func (s *Size) UnmarshalYAML(unmarshal func(any) error) error { err = unmarshal(&strVal) if err != nil { - return errors.New("size must be a number or string") + return errSizeType } // Parse the string using go-humanize @@ -40,6 +48,10 @@ func (s *Size) UnmarshalYAML(unmarshal func(any) error) error { return fmt.Errorf("invalid size format: %w", err) } + if bytes > math.MaxInt64 { + return fmt.Errorf("%w: %s", errSizeTooLarge, strVal) + } + *s = Size(bytes) return nil @@ -56,6 +68,7 @@ func (s Size) Int64() int64 { // For example, 1048576 bytes would be formatted as "1.0 MB". // This implements the fmt.Stringer interface. func (s Size) String() string { + //nolint:gosec // G115: sizes are non-negative by construction return humanize.Bytes(uint64(s)) } @@ -66,5 +79,9 @@ func ParseSize(s string) (Size, error) { return 0, fmt.Errorf("invalid size format: %w", err) } + if bytes > math.MaxInt64 { + return 0, fmt.Errorf("%w: %s", errSizeTooLarge, s) + } + return Size(bytes), nil } diff --git a/internal/crypto/encryption.go b/internal/crypto/encryption.go index b744d30..36c2564 100644 --- a/internal/crypto/encryption.go +++ b/internal/crypto/encryption.go @@ -1,3 +1,5 @@ +// Package crypto provides thread-safe age encryption and decryption +// helpers used to protect blob and metadata content. package crypto import ( @@ -11,6 +13,10 @@ import ( "go.uber.org/fx" ) +// ErrNoRecipients is returned when an encryptor is created or updated +// without any recipient public keys. +var ErrNoRecipients = errors.New("at least one recipient is required") + // Encryptor provides thread-safe encryption using the age encryption library. // It supports encrypting data for multiple recipients simultaneously, allowing // any of the corresponding private keys to decrypt the data. This is useful @@ -26,7 +32,7 @@ type Encryptor struct { // public keys are invalid or if no recipients are specified. func NewEncryptor(publicKeys []string) (*Encryptor, error) { if len(publicKeys) == 0 { - return nil, errors.New("at least one recipient is required") + return nil, ErrNoRecipients } recipients := make([]age.Recipient, 0, len(publicKeys)) @@ -132,7 +138,7 @@ func (e *Encryptor) EncryptWriter(dst io.Writer) (io.WriteCloser, error) { // of the public keys are invalid or if no recipients are specified. func (e *Encryptor) UpdateRecipients(publicKeys []string) error { if len(publicKeys) == 0 { - return errors.New("at least one recipient is required") + return ErrNoRecipients } recipients := make([]age.Recipient, 0, len(publicKeys)) @@ -213,4 +219,6 @@ func (d *Decryptor) DecryptStream(src io.Reader) (io.Reader, error) { } // Module exports the crypto module for fx dependency injection. +// +//nolint:gochecknoglobals // fx module definitions are package globals var Module = fx.Module("crypto") diff --git a/internal/crypto/encryption_test.go b/internal/crypto/encryption_test.go index ada9d77..3f24d47 100644 --- a/internal/crypto/encryption_test.go +++ b/internal/crypto/encryption_test.go @@ -1,13 +1,16 @@ -package crypto +package crypto_test import ( "bytes" "testing" "filippo.io/age" + "sneak.berlin/go/vaultik/internal/crypto" ) func TestEncryptor(t *testing.T) { + t.Parallel() + // Generate a test key pair identity, err := age.GenerateX25519Identity() if err != nil { @@ -17,7 +20,7 @@ func TestEncryptor(t *testing.T) { publicKey := identity.Recipient().String() // Create encryptor - enc, err := NewEncryptor([]string{publicKey}) + enc, err := crypto.NewEncryptor([]string{publicKey}) if err != nil { t.Fatalf("failed to create encryptor: %v", err) } @@ -55,6 +58,8 @@ func TestEncryptor(t *testing.T) { } func TestEncryptorMultipleRecipients(t *testing.T) { + t.Parallel() + // Generate three test key pairs identity1, err := age.GenerateX25519Identity() if err != nil { @@ -78,7 +83,7 @@ func TestEncryptorMultipleRecipients(t *testing.T) { } // Create encryptor with multiple recipients - enc, err := NewEncryptor(publicKeys) + enc, err := crypto.NewEncryptor(publicKeys) if err != nil { t.Fatalf("failed to create encryptor: %v", err) } @@ -114,6 +119,8 @@ func TestEncryptorMultipleRecipients(t *testing.T) { } func TestEncryptorUpdateRecipients(t *testing.T) { + t.Parallel() + // Generate two identities identity1, _ := age.GenerateX25519Identity() identity2, _ := age.GenerateX25519Identity() @@ -122,7 +129,7 @@ func TestEncryptorUpdateRecipients(t *testing.T) { publicKey2 := identity2.Recipient().String() // Create encryptor with first key - enc, err := NewEncryptor([]string{publicKey1}) + enc, err := crypto.NewEncryptor([]string{publicKey1}) if err != nil { t.Fatalf("failed to create encryptor: %v", err) } diff --git a/internal/database/blob_chunks.go b/internal/database/blob_chunks.go index 5a83b07..0a009c9 100644 --- a/internal/database/blob_chunks.go +++ b/internal/database/blob_chunks.go @@ -7,15 +7,21 @@ import ( "fmt" ) +// BlobChunkRepository provides access to the blob_chunks table, which maps +// blobs to the chunks they contain (with offset and length). type BlobChunkRepository struct { db *DB } +// NewBlobChunkRepository creates a BlobChunkRepository backed by db. func NewBlobChunkRepository(db *DB) *BlobChunkRepository { return &BlobChunkRepository{db: db} } -func (r *BlobChunkRepository) Create(ctx context.Context, tx *sql.Tx, bc *BlobChunk) error { +// Create inserts a blob_chunks row, using tx when non-nil. +func (r *BlobChunkRepository) Create( + ctx context.Context, tx *sql.Tx, bc *BlobChunk, +) error { query := ` INSERT INTO blob_chunks (blob_id, chunk_hash, offset, length) VALUES (?, ?, ?, ?) @@ -35,7 +41,11 @@ func (r *BlobChunkRepository) Create(ctx context.Context, tx *sql.Tx, bc *BlobCh return nil } -func (r *BlobChunkRepository) GetByBlobID(ctx context.Context, blobID string) ([]*BlobChunk, error) { +// GetByBlobID returns all chunks contained in the given blob, ordered by +// their offset within the blob. +func (r *BlobChunkRepository) GetByBlobID( + ctx context.Context, blobID string, +) ([]*BlobChunk, error) { query := ` SELECT blob_id, chunk_hash, offset, length FROM blob_chunks @@ -65,7 +75,11 @@ func (r *BlobChunkRepository) GetByBlobID(ctx context.Context, blobID string) ([ return blobChunks, rows.Err() } -func (r *BlobChunkRepository) GetByChunkHash(ctx context.Context, chunkHash string) (*BlobChunk, error) { +// GetByChunkHash returns one blob_chunks row containing the given chunk, +// or nil if the chunk is not packed in any blob. +func (r *BlobChunkRepository) GetByChunkHash( + ctx context.Context, chunkHash string, +) (*BlobChunk, error) { query := ` SELECT blob_id, chunk_hash, offset, length FROM blob_chunks @@ -87,7 +101,7 @@ func (r *BlobChunkRepository) GetByChunkHash(ctx context.Context, chunkHash stri if errors.Is(err, sql.ErrNoRows) { LogSQL("GetByChunkHash", "No rows found", chunkHash) - return nil, nil + return nil, nil //nolint:nilnil // nil,nil signals not-found; callers check nil } if err != nil { @@ -102,7 +116,9 @@ func (r *BlobChunkRepository) GetByChunkHash(ctx context.Context, chunkHash stri } // GetByChunkHashTx retrieves a blob chunk within a transaction -func (r *BlobChunkRepository) GetByChunkHashTx(ctx context.Context, tx *sql.Tx, chunkHash string) (*BlobChunk, error) { +func (r *BlobChunkRepository) GetByChunkHashTx( + ctx context.Context, tx *sql.Tx, chunkHash string, +) (*BlobChunk, error) { query := ` SELECT blob_id, chunk_hash, offset, length FROM blob_chunks @@ -124,7 +140,7 @@ func (r *BlobChunkRepository) GetByChunkHashTx(ctx context.Context, tx *sql.Tx, if errors.Is(err, sql.ErrNoRows) { LogSQL("GetByChunkHashTx", "No rows found", chunkHash) - return nil, nil + return nil, nil //nolint:nilnil // nil,nil signals not-found; callers check nil } if err != nil { @@ -138,13 +154,14 @@ func (r *BlobChunkRepository) GetByChunkHashTx(ctx context.Context, tx *sql.Tx, return &bc, nil } -// DeleteOrphaned deletes blob_chunks entries where either the blob or chunk no longer exists +// DeleteOrphaned deletes blob_chunks entries where either the blob or the +// chunk no longer exists. func (r *BlobChunkRepository) DeleteOrphaned(ctx context.Context) error { // Delete blob_chunks where the blob doesn't exist query1 := ` - DELETE FROM blob_chunks + DELETE FROM blob_chunks WHERE NOT EXISTS ( - SELECT 1 FROM blobs + SELECT 1 FROM blobs WHERE blobs.id = blob_chunks.blob_id ) ` @@ -156,9 +173,9 @@ func (r *BlobChunkRepository) DeleteOrphaned(ctx context.Context) error { // Delete blob_chunks where the chunk doesn't exist query2 := ` - DELETE FROM blob_chunks + DELETE FROM blob_chunks WHERE NOT EXISTS ( - SELECT 1 FROM chunks + SELECT 1 FROM chunks WHERE chunks.chunk_hash = blob_chunks.chunk_hash ) ` diff --git a/internal/database/blob_chunks_test.go b/internal/database/blob_chunks_test.go index 97d3e68..5122d63 100644 --- a/internal/database/blob_chunks_test.go +++ b/internal/database/blob_chunks_test.go @@ -1,4 +1,4 @@ -package database +package database_test import ( "context" @@ -6,59 +6,91 @@ import ( "testing" "time" + "sneak.berlin/go/vaultik/internal/database" "sneak.berlin/go/vaultik/internal/types" ) -func TestBlobChunkRepository(t *testing.T) { - db, cleanup := setupTestDB(t) - defer cleanup() +// Chunk hashes used across the blob_chunks tests. +const ( + chunk1Hash = "chunk1" + chunk2Hash = "chunk2" + chunk3Hash = "chunk3" +) + +// mustCreateChunks registers the given chunk hashes (1024 bytes each). +func mustCreateChunks( + t *testing.T, + repos *database.Repositories, + hashes ...types.ChunkHash, +) { + t.Helper() ctx := context.Background() - repos := NewRepositories(db) - // Create blob first - blob := &Blob{ - ID: types.NewBlobID(), - Hash: types.BlobHash("blob1-hash"), - CreatedTS: time.Now(), - } - - err := repos.Blobs.Create(ctx, nil, blob) - if err != nil { - t.Fatalf("failed to create blob: %v", err) - } - - // Create chunks - chunks := []types.ChunkHash{"chunk1", "chunk2", "chunk3"} - for _, chunkHash := range chunks { - chunk := &Chunk{ + for _, chunkHash := range hashes { + chunk := &database.Chunk{ ChunkHash: chunkHash, Size: 1024, } - err = repos.Chunks.Create(ctx, nil, chunk) + err := repos.Chunks.Create(ctx, nil, chunk) if err != nil { t.Fatalf("failed to create chunk %s: %v", chunkHash, err) } } +} + +// mustCreateBlob creates a blob row with the given hash. +func mustCreateBlob( + t *testing.T, + repos *database.Repositories, + hash types.BlobHash, +) *database.Blob { + t.Helper() + + blob := &database.Blob{ + ID: types.NewBlobID(), + Hash: hash, + CreatedTS: time.Now(), + } + + err := repos.Blobs.Create(context.Background(), nil, blob) + if err != nil { + t.Fatalf("failed to create blob %s: %v", hash, err) + } + + return blob +} + +func TestBlobChunkRepository(t *testing.T) { + t.Parallel() + + db, cleanup := setupTestDB(t) + defer cleanup() + + ctx := context.Background() + repos := database.NewRepositories(db) + + blob := mustCreateBlob(t, repos, "blob1-hash") + mustCreateChunks(t, repos, chunk1Hash, chunk2Hash, chunk3Hash) // Test Create - bc1 := &BlobChunk{ + bc1 := &database.BlobChunk{ BlobID: blob.ID, - ChunkHash: types.ChunkHash("chunk1"), + ChunkHash: types.ChunkHash(chunk1Hash), Offset: 0, Length: 1024, } - err = repos.BlobChunks.Create(ctx, nil, bc1) + err := repos.BlobChunks.Create(ctx, nil, bc1) if err != nil { t.Fatalf("failed to create blob chunk: %v", err) } // Add more chunks to the same blob - bc2 := &BlobChunk{ + bc2 := &database.BlobChunk{ BlobID: blob.ID, - ChunkHash: types.ChunkHash("chunk2"), + ChunkHash: types.ChunkHash(chunk2Hash), Offset: 1024, Length: 2048, } @@ -68,9 +100,9 @@ func TestBlobChunkRepository(t *testing.T) { t.Fatalf("failed to create second blob chunk: %v", err) } - bc3 := &BlobChunk{ + bc3 := &database.BlobChunk{ BlobID: blob.ID, - ChunkHash: types.ChunkHash("chunk3"), + ChunkHash: types.ChunkHash(chunk3Hash), Offset: 3072, Length: 512, } @@ -94,12 +126,49 @@ func TestBlobChunkRepository(t *testing.T) { expectedOffsets := []int64{0, 1024, 3072} for i, bc := range blobChunks { if bc.Offset != expectedOffsets[i] { - t.Errorf("wrong chunk order: expected offset %d, got %d", expectedOffsets[i], bc.Offset) + t.Errorf("wrong chunk order: expected offset %d, got %d", + expectedOffsets[i], bc.Offset) } } + // Test duplicate insert (should fail due to primary key constraint) + err = repos.BlobChunks.Create(ctx, nil, bc1) + if err == nil { + t.Fatal("duplicate blob_chunk insert should fail due to primary key constraint") + } + + if !strings.Contains(err.Error(), "UNIQUE") && + !strings.Contains(err.Error(), "constraint") { + t.Fatalf("expected constraint error, got: %v", err) + } +} + +func TestBlobChunkRepositoryGetByChunkHash(t *testing.T) { + t.Parallel() + + db, cleanup := setupTestDB(t) + defer cleanup() + + ctx := context.Background() + repos := database.NewRepositories(db) + + blob := mustCreateBlob(t, repos, "blob-gbch-hash") + mustCreateChunks(t, repos, chunk2Hash) + + bc2 := &database.BlobChunk{ + BlobID: blob.ID, + ChunkHash: types.ChunkHash(chunk2Hash), + Offset: 1024, + Length: 2048, + } + + err := repos.BlobChunks.Create(ctx, nil, bc2) + if err != nil { + t.Fatalf("failed to create blob chunk: %v", err) + } + // Test GetByChunkHash - bc, err := repos.BlobChunks.GetByChunkHash(ctx, "chunk2") + bc, err := repos.BlobChunks.GetByChunkHash(ctx, chunk2Hash) if err != nil { t.Fatalf("failed to get blob chunk by chunk hash: %v", err) } @@ -116,16 +185,6 @@ func TestBlobChunkRepository(t *testing.T) { t.Errorf("wrong offset: expected 1024, got %d", bc.Offset) } - // Test duplicate insert (should fail due to primary key constraint) - err = repos.BlobChunks.Create(ctx, nil, bc1) - if err == nil { - t.Fatal("duplicate blob_chunk insert should fail due to primary key constraint") - } - - if !strings.Contains(err.Error(), "UNIQUE") && !strings.Contains(err.Error(), "constraint") { - t.Fatalf("expected constraint error, got: %v", err) - } - // Test non-existent chunk bc, err = repos.BlobChunks.GetByChunkHash(ctx, "nonexistent") if err != nil { @@ -138,55 +197,26 @@ func TestBlobChunkRepository(t *testing.T) { } func TestBlobChunkRepositoryMultipleBlobs(t *testing.T) { + t.Parallel() + db, cleanup := setupTestDB(t) defer cleanup() ctx := context.Background() - repos := NewRepositories(db) + repos := database.NewRepositories(db) - // Create blobs - blob1 := &Blob{ - ID: types.NewBlobID(), - Hash: types.BlobHash("blob1-hash"), - CreatedTS: time.Now(), - } - blob2 := &Blob{ - ID: types.NewBlobID(), - Hash: types.BlobHash("blob2-hash"), - CreatedTS: time.Now(), - } - - err := repos.Blobs.Create(ctx, nil, blob1) - if err != nil { - t.Fatalf("failed to create blob1: %v", err) - } - - err = repos.Blobs.Create(ctx, nil, blob2) - if err != nil { - t.Fatalf("failed to create blob2: %v", err) - } - - // Create chunks - chunkHashes := []types.ChunkHash{"chunk1", "chunk2", "chunk3"} - for _, chunkHash := range chunkHashes { - chunk := &Chunk{ - ChunkHash: chunkHash, - Size: 1024, - } - - err = repos.Chunks.Create(ctx, nil, chunk) - if err != nil { - t.Fatalf("failed to create chunk %s: %v", chunkHash, err) - } - } + blob1 := mustCreateBlob(t, repos, "blob1-hash") + blob2 := mustCreateBlob(t, repos, "blob2-hash") + mustCreateChunks(t, repos, chunk1Hash, chunk2Hash, chunk3Hash) // Create chunks across multiple blobs // Some chunks are shared between blobs (deduplication scenario) - blobChunks := []BlobChunk{ - {BlobID: blob1.ID, ChunkHash: types.ChunkHash("chunk1"), Offset: 0, Length: 1024}, - {BlobID: blob1.ID, ChunkHash: types.ChunkHash("chunk2"), Offset: 1024, Length: 1024}, - {BlobID: blob2.ID, ChunkHash: types.ChunkHash("chunk2"), Offset: 0, Length: 1024}, // chunk2 is shared - {BlobID: blob2.ID, ChunkHash: types.ChunkHash("chunk3"), Offset: 1024, Length: 1024}, + blobChunks := []database.BlobChunk{ + {BlobID: blob1.ID, ChunkHash: chunk1Hash, Offset: 0, Length: 1024}, + {BlobID: blob1.ID, ChunkHash: chunk2Hash, Offset: 1024, Length: 1024}, + // chunk2 is shared between the blobs + {BlobID: blob2.ID, ChunkHash: chunk2Hash, Offset: 0, Length: 1024}, + {BlobID: blob2.ID, ChunkHash: chunk3Hash, Offset: 1024, Length: 1024}, } for _, bc := range blobChunks { @@ -217,7 +247,7 @@ func TestBlobChunkRepositoryMultipleBlobs(t *testing.T) { } // Verify shared chunk - bc, err := repos.BlobChunks.GetByChunkHash(ctx, "chunk2") + bc, err := repos.BlobChunks.GetByChunkHash(ctx, chunk2Hash) if err != nil { t.Fatalf("failed to get shared chunk: %v", err) } diff --git a/internal/database/blobs.go b/internal/database/blobs.go index 5bab2a7..0acd4cd 100644 --- a/internal/database/blobs.go +++ b/internal/database/blobs.go @@ -10,17 +10,22 @@ import ( "sneak.berlin/go/vaultik/internal/log" ) +// BlobRepository provides access to the blobs table, which tracks the +// packed, encrypted storage units uploaded to the destination. type BlobRepository struct { db *DB } +// NewBlobRepository creates a BlobRepository backed by db. func NewBlobRepository(db *DB) *BlobRepository { return &BlobRepository{db: db} } +// Create inserts a blob row, using tx when non-nil. func (r *BlobRepository) Create(ctx context.Context, tx *sql.Tx, blob *Blob) error { query := ` - INSERT INTO blobs (id, blob_hash, created_ts, finished_ts, uncompressed_size, compressed_size, uploaded_ts) + INSERT INTO blobs (id, blob_hash, created_ts, finished_ts, + uncompressed_size, compressed_size, uploaded_ts) VALUES (?, ?, ?, ?, ?, ?, ?) ` @@ -52,95 +57,15 @@ func (r *BlobRepository) Create(ctx context.Context, tx *sql.Tx, blob *Blob) err return nil } +// GetByHash returns the blob with the given content hash, or nil if no +// such blob exists. func (r *BlobRepository) GetByHash(ctx context.Context, hash string) (*Blob, error) { - query := ` - SELECT id, blob_hash, created_ts, finished_ts, uncompressed_size, compressed_size, uploaded_ts - FROM blobs - WHERE blob_hash = ? - ` - - var ( - blob Blob - createdTSUnix int64 - finishedTSUnix, uploadedTSUnix sql.NullInt64 - ) - - err := r.db.conn.QueryRowContext(ctx, query, hash).Scan( - &blob.ID, - &blob.Hash, - &createdTSUnix, - &finishedTSUnix, - &blob.UncompressedSize, - &blob.CompressedSize, - &uploadedTSUnix, - ) - - if errors.Is(err, sql.ErrNoRows) { - return nil, nil - } - - if err != nil { - return nil, fmt.Errorf("querying blob: %w", err) - } - - blob.CreatedTS = time.Unix(createdTSUnix, 0).UTC() - if finishedTSUnix.Valid { - ts := time.Unix(finishedTSUnix.Int64, 0).UTC() - blob.FinishedTS = &ts - } - - if uploadedTSUnix.Valid { - ts := time.Unix(uploadedTSUnix.Int64, 0).UTC() - blob.UploadedTS = &ts - } - - return &blob, nil + return r.getOne(ctx, "blob_hash", hash) } // GetByID retrieves a blob by its ID func (r *BlobRepository) GetByID(ctx context.Context, id string) (*Blob, error) { - query := ` - SELECT id, blob_hash, created_ts, finished_ts, uncompressed_size, compressed_size, uploaded_ts - FROM blobs - WHERE id = ? - ` - - var ( - blob Blob - createdTSUnix int64 - finishedTSUnix, uploadedTSUnix sql.NullInt64 - ) - - err := r.db.conn.QueryRowContext(ctx, query, id).Scan( - &blob.ID, - &blob.Hash, - &createdTSUnix, - &finishedTSUnix, - &blob.UncompressedSize, - &blob.CompressedSize, - &uploadedTSUnix, - ) - - if errors.Is(err, sql.ErrNoRows) { - return nil, nil - } - - if err != nil { - return nil, fmt.Errorf("querying blob: %w", err) - } - - blob.CreatedTS = time.Unix(createdTSUnix, 0).UTC() - if finishedTSUnix.Valid { - ts := time.Unix(finishedTSUnix.Int64, 0).UTC() - blob.FinishedTS = &ts - } - - if uploadedTSUnix.Valid { - ts := time.Unix(uploadedTSUnix.Int64, 0).UTC() - blob.UploadedTS = &ts - } - - return &blob, nil + return r.getOne(ctx, "id", id) } // GetAll returns every blob row keyed by blob ID. Useful at restore @@ -148,7 +73,8 @@ func (r *BlobRepository) GetByID(ctx context.Context, id string) (*Blob, error) // into blob hashes without doing one GetByID query per chunk. func (r *BlobRepository) GetAll(ctx context.Context) (map[string]*Blob, error) { query := ` - SELECT id, blob_hash, created_ts, finished_ts, uncompressed_size, compressed_size, uploaded_ts + SELECT id, blob_hash, created_ts, finished_ts, + uncompressed_size, compressed_size, uploaded_ts FROM blobs ` @@ -198,7 +124,13 @@ func (r *BlobRepository) GetAll(ctx context.Context) (map[string]*Blob, error) { } // UpdateFinished updates a blob when it's finalized -func (r *BlobRepository) UpdateFinished(ctx context.Context, tx *sql.Tx, id string, hash string, uncompressedSize, compressedSize int64) error { +func (r *BlobRepository) UpdateFinished( + ctx context.Context, + tx *sql.Tx, + id string, + hash string, + uncompressedSize, compressedSize int64, +) error { query := ` UPDATE blobs SET blob_hash = ?, finished_ts = ?, uncompressed_size = ?, compressed_size = ? @@ -222,7 +154,9 @@ func (r *BlobRepository) UpdateFinished(ctx context.Context, tx *sql.Tx, id stri } // UpdateUploaded marks a blob as uploaded -func (r *BlobRepository) UpdateUploaded(ctx context.Context, tx *sql.Tx, id string) error { +func (r *BlobRepository) UpdateUploaded( + ctx context.Context, tx *sql.Tx, id string, +) error { query := ` UPDATE blobs SET uploaded_ts = ? @@ -267,3 +201,52 @@ func (r *BlobRepository) DeleteOrphaned(ctx context.Context) error { return nil } + +// getOne fetches a single blob row matched on the given column, or +// (nil, nil) when no row matches. +func (r *BlobRepository) getOne( + ctx context.Context, column, value string, +) (*Blob, error) { + query := ` + SELECT id, blob_hash, created_ts, finished_ts, + uncompressed_size, compressed_size, uploaded_ts + FROM blobs + WHERE ` + column + ` = ?` + + var ( + blob Blob + createdTSUnix int64 + finishedTSUnix, uploadedTSUnix sql.NullInt64 + ) + + err := r.db.conn.QueryRowContext(ctx, query, value).Scan( + &blob.ID, + &blob.Hash, + &createdTSUnix, + &finishedTSUnix, + &blob.UncompressedSize, + &blob.CompressedSize, + &uploadedTSUnix, + ) + + if errors.Is(err, sql.ErrNoRows) { + return nil, nil //nolint:nilnil // nil,nil signals not-found; callers check nil + } + + if err != nil { + return nil, fmt.Errorf("querying blob: %w", err) + } + + blob.CreatedTS = time.Unix(createdTSUnix, 0).UTC() + if finishedTSUnix.Valid { + ts := time.Unix(finishedTSUnix.Int64, 0).UTC() + blob.FinishedTS = &ts + } + + if uploadedTSUnix.Valid { + ts := time.Unix(uploadedTSUnix.Int64, 0).UTC() + blob.UploadedTS = &ts + } + + return &blob, nil +} diff --git a/internal/database/blobs_test.go b/internal/database/blobs_test.go index 2b8e88b..457b0db 100644 --- a/internal/database/blobs_test.go +++ b/internal/database/blobs_test.go @@ -1,22 +1,25 @@ -package database +package database_test import ( "context" "testing" "time" + "sneak.berlin/go/vaultik/internal/database" "sneak.berlin/go/vaultik/internal/types" ) func TestBlobRepository(t *testing.T) { + t.Parallel() + db, cleanup := setupTestDB(t) defer cleanup() ctx := context.Background() - repo := NewBlobRepository(db) + repo := database.NewBlobRepository(db) // Test Create - blob := &Blob{ + blob := &database.Blob{ ID: types.NewBlobID(), Hash: types.BlobHash("blobhash123"), CreatedTS: time.Now().Truncate(time.Second), @@ -42,7 +45,8 @@ func TestBlobRepository(t *testing.T) { } if !retrieved.CreatedTS.Equal(blob.CreatedTS) { - t.Errorf("created timestamp mismatch: got %v, want %v", retrieved.CreatedTS, blob.CreatedTS) + t.Errorf("created timestamp mismatch: got %v, want %v", + retrieved.CreatedTS, blob.CreatedTS) } // Test GetByID @@ -60,7 +64,7 @@ func TestBlobRepository(t *testing.T) { } // Test with second blob - blob2 := &Blob{ + blob2 := &database.Blob{ ID: types.NewBlobID(), Hash: types.BlobHash("blobhash456"), CreatedTS: time.Now().Truncate(time.Second), @@ -70,6 +74,27 @@ func TestBlobRepository(t *testing.T) { if err != nil { t.Fatalf("failed to create second blob: %v", err) } +} + +func TestBlobRepositoryUpdates(t *testing.T) { + t.Parallel() + + db, cleanup := setupTestDB(t) + defer cleanup() + + ctx := context.Background() + repo := database.NewBlobRepository(db) + + blob := &database.Blob{ + ID: types.NewBlobID(), + Hash: types.BlobHash("blobhash123"), + CreatedTS: time.Now().Truncate(time.Second), + } + + err := repo.Create(ctx, nil, blob) + if err != nil { + t.Fatalf("failed to create blob: %v", err) + } // Test UpdateFinished now := time.Now() @@ -119,13 +144,15 @@ func TestBlobRepository(t *testing.T) { } func TestBlobRepositoryDuplicate(t *testing.T) { + t.Parallel() + db, cleanup := setupTestDB(t) defer cleanup() ctx := context.Background() - repo := NewBlobRepository(db) + repo := database.NewBlobRepository(db) - blob := &Blob{ + blob := &database.Blob{ ID: types.NewBlobID(), Hash: types.BlobHash("duplicate_blob"), CreatedTS: time.Now().Truncate(time.Second), diff --git a/internal/database/cascade_debug_test.go b/internal/database/cascade_debug_test.go index fc71f9f..bcde357 100644 --- a/internal/database/cascade_debug_test.go +++ b/internal/database/cascade_debug_test.go @@ -1,3 +1,4 @@ +//nolint:testpackage // inspects the unexported database connection package database import ( @@ -9,25 +10,13 @@ import ( "sneak.berlin/go/vaultik/internal/types" ) -// TestCascadeDeleteDebug tests cascade delete with debug output -func TestCascadeDeleteDebug(t *testing.T) { - db, cleanup := setupTestDB(t) - defer cleanup() +// createCascadeFixtures creates a file with three chunk mappings for the +// cascade-delete test. +func createCascadeFixtures(t *testing.T, repos *Repositories) *File { + t.Helper() ctx := context.Background() - repos := NewRepositories(db) - // Check if foreign keys are enabled - var fkEnabled int - - err := db.conn.QueryRow("PRAGMA foreign_keys").Scan(&fkEnabled) - if err != nil { - t.Fatal(err) - } - - t.Logf("Foreign keys enabled: %d", fkEnabled) - - // Create a file file := &File{ Path: "/cascade-test.txt", MTime: time.Now().Truncate(time.Second), @@ -37,7 +26,7 @@ func TestCascadeDeleteDebug(t *testing.T) { GID: 1000, } - err = repos.Files.Create(ctx, nil, file) + err := repos.Files.Create(ctx, nil, file) if err != nil { t.Fatalf("failed to create file: %v", err) } @@ -67,9 +56,56 @@ func TestCascadeDeleteDebug(t *testing.T) { t.Fatalf("failed to create file chunk: %v", err) } - t.Logf("Created file chunk mapping: file_id=%s, idx=%d, chunk=%s", fc.FileID, fc.Idx, fc.ChunkHash) + t.Logf("Created file chunk mapping: file_id=%s, idx=%d, chunk=%s", + fc.FileID, fc.Idx, fc.ChunkHash) } + return file +} + +// logCascadeDebugInfo logs foreign-key state and the file_chunks table +// definition for cascade-delete debugging. +func logCascadeDebugInfo(ctx context.Context, t *testing.T, db *DB) { + t.Helper() + + // Check if foreign keys are enabled + var fkEnabled int + + err := db.conn.QueryRowContext(ctx, "PRAGMA foreign_keys").Scan(&fkEnabled) + if err != nil { + t.Fatal(err) + } + + t.Logf("Foreign keys enabled: %d", fkEnabled) + + // Check the foreign key constraint + var fkInfo string + + err = db.conn.QueryRowContext(ctx, ` + SELECT sql FROM sqlite_master + WHERE type='table' AND name='file_chunks' + `).Scan(&fkInfo) + if err != nil { + t.Fatal(err) + } + + t.Logf("file_chunks table definition:\n%s", fkInfo) +} + +// TestCascadeDeleteDebug tests cascade delete with debug output +func TestCascadeDeleteDebug(t *testing.T) { + t.Parallel() + + db, cleanup := setupTestDB(t) + defer cleanup() + + ctx := context.Background() + repos := NewRepositories(db) + + logCascadeDebugInfo(ctx, t, db) + + file := createCascadeFixtures(t, repos) + // Verify file chunks exist fileChunks, err := repos.FileChunks.GetByFileID(ctx, file.ID) if err != nil { @@ -78,19 +114,6 @@ func TestCascadeDeleteDebug(t *testing.T) { t.Logf("File chunks before delete: %d", len(fileChunks)) - // Check the foreign key constraint - var fkInfo string - - err = db.conn.QueryRow(` - SELECT sql FROM sqlite_master - WHERE type='table' AND name='file_chunks' - `).Scan(&fkInfo) - if err != nil { - t.Fatal(err) - } - - t.Logf("file_chunks table definition:\n%s", fkInfo) - // Delete the file t.Log("Deleting file...") @@ -122,7 +145,9 @@ func TestCascadeDeleteDebug(t *testing.T) { // Manually check the database var count int - err = db.conn.QueryRow("SELECT COUNT(*) FROM file_chunks WHERE file_id = ?", file.ID).Scan(&count) + err = db.conn.QueryRowContext(ctx, + "SELECT COUNT(*) FROM file_chunks WHERE file_id = ?", file.ID, + ).Scan(&count) if err != nil { t.Fatal(err) } @@ -133,7 +158,8 @@ func TestCascadeDeleteDebug(t *testing.T) { t.Errorf("expected 0 file chunks after cascade delete, got %d", len(fileChunks)) // List the remaining chunks for _, fc := range fileChunks { - t.Logf("Remaining chunk: file_id=%s, idx=%d, chunk=%s", fc.FileID, fc.Idx, fc.ChunkHash) + t.Logf("Remaining chunk: file_id=%s, idx=%d, chunk=%s", + fc.FileID, fc.Idx, fc.ChunkHash) } } } diff --git a/internal/database/chunk_files.go b/internal/database/chunk_files.go index 01e9175..f6b00bd 100644 --- a/internal/database/chunk_files.go +++ b/internal/database/chunk_files.go @@ -9,15 +9,21 @@ import ( "sneak.berlin/go/vaultik/internal/types" ) +// ChunkFileRepository provides access to the chunk_files table, the +// reverse mapping from chunks to the files that contain them. type ChunkFileRepository struct { db *DB } +// NewChunkFileRepository creates a ChunkFileRepository backed by db. func NewChunkFileRepository(db *DB) *ChunkFileRepository { return &ChunkFileRepository{db: db} } -func (r *ChunkFileRepository) Create(ctx context.Context, tx *sql.Tx, cf *ChunkFile) error { +// Create inserts a chunk_files row (idempotently), using tx when non-nil. +func (r *ChunkFileRepository) Create( + ctx context.Context, tx *sql.Tx, cf *ChunkFile, +) error { query := ` INSERT INTO chunk_files (chunk_hash, file_id, file_offset, length) VALUES (?, ?, ?, ?) @@ -26,9 +32,11 @@ func (r *ChunkFileRepository) Create(ctx context.Context, tx *sql.Tx, cf *ChunkF var err error if tx != nil { - _, err = tx.ExecContext(ctx, query, cf.ChunkHash.String(), cf.FileID.String(), cf.FileOffset, cf.Length) + _, err = tx.ExecContext(ctx, query, + cf.ChunkHash.String(), cf.FileID.String(), cf.FileOffset, cf.Length) } else { - _, err = r.db.ExecWithLog(ctx, query, cf.ChunkHash.String(), cf.FileID.String(), cf.FileOffset, cf.Length) + _, err = r.db.ExecWithLog(ctx, query, + cf.ChunkHash.String(), cf.FileID.String(), cf.FileOffset, cf.Length) } if err != nil { @@ -38,7 +46,10 @@ func (r *ChunkFileRepository) Create(ctx context.Context, tx *sql.Tx, cf *ChunkF return nil } -func (r *ChunkFileRepository) GetByChunkHash(ctx context.Context, chunkHash types.ChunkHash) ([]*ChunkFile, error) { +// GetByChunkHash returns all chunk_files rows for the given chunk hash. +func (r *ChunkFileRepository) GetByChunkHash( + ctx context.Context, chunkHash types.ChunkHash, +) ([]*ChunkFile, error) { query := ` SELECT chunk_hash, file_id, file_offset, length FROM chunk_files @@ -54,7 +65,10 @@ func (r *ChunkFileRepository) GetByChunkHash(ctx context.Context, chunkHash type return r.scanChunkFiles(rows) } -func (r *ChunkFileRepository) GetByFilePath(ctx context.Context, filePath string) ([]*ChunkFile, error) { +// GetByFilePath returns all chunk_files rows for the file at the given path. +func (r *ChunkFileRepository) GetByFilePath( + ctx context.Context, filePath string, +) ([]*ChunkFile, error) { query := ` SELECT cf.chunk_hash, cf.file_id, cf.file_offset, cf.length FROM chunk_files cf @@ -72,7 +86,9 @@ func (r *ChunkFileRepository) GetByFilePath(ctx context.Context, filePath string } // GetByFileID retrieves chunk files by file ID -func (r *ChunkFileRepository) GetByFileID(ctx context.Context, fileID types.FileID) ([]*ChunkFile, error) { +func (r *ChunkFileRepository) GetByFileID( + ctx context.Context, fileID types.FileID, +) ([]*ChunkFile, error) { query := ` SELECT chunk_hash, file_id, file_offset, length FROM chunk_files @@ -88,7 +104,124 @@ func (r *ChunkFileRepository) GetByFileID(ctx context.Context, fileID types.File return r.scanChunkFiles(rows) } -// scanChunkFiles is a helper that scans chunk file rows +// DeleteByFileID deletes all chunk_files entries for a given file ID +func (r *ChunkFileRepository) DeleteByFileID( + ctx context.Context, tx *sql.Tx, fileID types.FileID, +) error { + query := `DELETE FROM chunk_files WHERE file_id = ?` + + var err error + if tx != nil { + _, err = tx.ExecContext(ctx, query, fileID.String()) + } else { + _, err = r.db.ExecWithLog(ctx, query, fileID.String()) + } + + if err != nil { + return fmt.Errorf("deleting chunk files: %w", err) + } + + return nil +} + +// DeleteByFileIDs deletes all chunk_files for multiple files in a single statement. +// +//nolint:dupl // symmetric implementation for a parallel association table +func (r *ChunkFileRepository) DeleteByFileIDs( + ctx context.Context, tx *sql.Tx, fileIDs []types.FileID, +) error { + if len(fileIDs) == 0 { + return nil + } + + // Batch at 500 to stay within SQLite's variable limit + const batchSize = 500 + + for i := 0; i < len(fileIDs); i += batchSize { + end := min(i+batchSize, len(fileIDs)) + + batch := fileIDs[i:end] + + //nolint:gosec // G202: concatenates constant SQL and "?" placeholders only + query := "DELETE FROM chunk_files WHERE file_id IN (?" + + repeatPlaceholder(len(batch)-1) + ")" + + args := make([]any, len(batch)) + for j, id := range batch { + args[j] = id.String() + } + + var err error + if tx != nil { + _, err = tx.ExecContext(ctx, query, args...) + } else { + _, err = r.db.ExecWithLog(ctx, query, args...) + } + + if err != nil { + return fmt.Errorf("batch deleting chunk_files: %w", err) + } + } + + return nil +} + +// CreateBatch inserts multiple chunk_files in a single statement for efficiency. +func (r *ChunkFileRepository) CreateBatch( + ctx context.Context, tx *sql.Tx, cfs []ChunkFile, +) error { + if len(cfs) == 0 { + return nil + } + + // Each chunk_files row binds this many SQL variables. + const chunkFileCols = 4 + + // Batch at 200 rows to be safe with SQLite's variable limit. + const batchSize = 200 + + for i := 0; i < len(cfs); i += batchSize { + end := min(i+batchSize, len(cfs)) + + batch := cfs[i:end] + + query := "INSERT INTO chunk_files (chunk_hash, file_id, file_offset, length) VALUES " + + args := make([]any, 0, len(batch)*chunkFileCols) + + var querySb183 strings.Builder + + for j, cf := range batch { + if j > 0 { + querySb183.WriteString(", ") + } + + querySb183.WriteString("(?, ?, ?, ?)") + + args = append(args, + cf.ChunkHash.String(), cf.FileID.String(), cf.FileOffset, cf.Length) + } + + query += querySb183.String() //nolint:gosec // G202: appends "?" placeholders only + + query += " ON CONFLICT(chunk_hash, file_id) DO NOTHING" + + var err error + if tx != nil { + _, err = tx.ExecContext(ctx, query, args...) + } else { + _, err = r.db.ExecWithLog(ctx, query, args...) + } + + if err != nil { + return fmt.Errorf("batch inserting chunk_files: %w", err) + } + } + + return nil +} + +// scanChunkFiles is a helper that scans chunk file rows. func (r *ChunkFileRepository) scanChunkFiles(rows *sql.Rows) ([]*ChunkFile, error) { var chunkFiles []*ChunkFile @@ -115,106 +248,3 @@ func (r *ChunkFileRepository) scanChunkFiles(rows *sql.Rows) ([]*ChunkFile, erro return chunkFiles, rows.Err() } - -// DeleteByFileID deletes all chunk_files entries for a given file ID -func (r *ChunkFileRepository) DeleteByFileID(ctx context.Context, tx *sql.Tx, fileID types.FileID) error { - query := `DELETE FROM chunk_files WHERE file_id = ?` - - var err error - if tx != nil { - _, err = tx.ExecContext(ctx, query, fileID.String()) - } else { - _, err = r.db.ExecWithLog(ctx, query, fileID.String()) - } - - if err != nil { - return fmt.Errorf("deleting chunk files: %w", err) - } - - return nil -} - -// DeleteByFileIDs deletes all chunk_files for multiple files in a single statement. -func (r *ChunkFileRepository) DeleteByFileIDs(ctx context.Context, tx *sql.Tx, fileIDs []types.FileID) error { - if len(fileIDs) == 0 { - return nil - } - - // Batch at 500 to stay within SQLite's variable limit - const batchSize = 500 - - for i := 0; i < len(fileIDs); i += batchSize { - end := min(i+batchSize, len(fileIDs)) - - batch := fileIDs[i:end] - - query := "DELETE FROM chunk_files WHERE file_id IN (?" + repeatPlaceholder(len(batch)-1) + ")" - - args := make([]any, len(batch)) - for j, id := range batch { - args[j] = id.String() - } - - var err error - if tx != nil { - _, err = tx.ExecContext(ctx, query, args...) - } else { - _, err = r.db.ExecWithLog(ctx, query, args...) - } - - if err != nil { - return fmt.Errorf("batch deleting chunk_files: %w", err) - } - } - - return nil -} - -// CreateBatch inserts multiple chunk_files in a single statement for efficiency. -func (r *ChunkFileRepository) CreateBatch(ctx context.Context, tx *sql.Tx, cfs []ChunkFile) error { - if len(cfs) == 0 { - return nil - } - - // Each ChunkFile has 4 values, so batch at 200 to be safe with SQLite's variable limit - const batchSize = 200 - - for i := 0; i < len(cfs); i += batchSize { - end := min(i+batchSize, len(cfs)) - - batch := cfs[i:end] - - query := "INSERT INTO chunk_files (chunk_hash, file_id, file_offset, length) VALUES " - - args := make([]any, 0, len(batch)*4) - - var querySb183 strings.Builder - - for j, cf := range batch { - if j > 0 { - querySb183.WriteString(", ") - } - - querySb183.WriteString("(?, ?, ?, ?)") - - args = append(args, cf.ChunkHash.String(), cf.FileID.String(), cf.FileOffset, cf.Length) - } - - query += querySb183.String() - - query += " ON CONFLICT(chunk_hash, file_id) DO NOTHING" - - var err error - if tx != nil { - _, err = tx.ExecContext(ctx, query, args...) - } else { - _, err = r.db.ExecWithLog(ctx, query, args...) - } - - if err != nil { - return fmt.Errorf("batch inserting chunk_files: %w", err) - } - } - - return nil -} diff --git a/internal/database/chunk_files_test.go b/internal/database/chunk_files_test.go index c198a28..aa47e73 100644 --- a/internal/database/chunk_files_test.go +++ b/internal/database/chunk_files_test.go @@ -1,81 +1,105 @@ -package database +package database_test import ( "context" "testing" "time" + "sneak.berlin/go/vaultik/internal/database" "sneak.berlin/go/vaultik/internal/types" ) +const chunk4Hash = "chunk4" + +// verifyChunkFilePair asserts that the chunk-file rows cover both test +// files at their expected offsets. +func verifyChunkFilePair( + t *testing.T, chunkFiles []*database.ChunkFile, + file1ID, file2ID types.FileID, +) { + t.Helper() + + foundFile1 := false + foundFile2 := false + + for _, cf := range chunkFiles { + if cf.FileID == file1ID && cf.FileOffset == 0 { + foundFile1 = true + } + + if cf.FileID == file2ID && cf.FileOffset == 2048 { + foundFile2 = true + } + } + + if !foundFile1 || !foundFile2 { + t.Error("not all expected files found") + } +} + +// createChunkFileTestFiles creates the two files used by the chunk-file +// repository tests. +func createChunkFileTestFiles( + t *testing.T, fileRepo *database.FileRepository, +) (*database.File, *database.File) { + t.Helper() + + testTime := time.Now().Truncate(time.Second) + file1 := &database.File{ + Path: testFilePath1, + MTime: testTime, + Size: 1024, + Mode: 0644, + UID: 1000, + GID: 1000, + LinkTarget: "", + } + file2 := &database.File{ + Path: testFilePath2, + MTime: testTime, + Size: 1024, + Mode: 0644, + UID: 1000, + GID: 1000, + LinkTarget: "", + } + + mustCreateFile(t, fileRepo, file1) + mustCreateFile(t, fileRepo, file2) + + return file1, file2 +} + func TestChunkFileRepository(t *testing.T) { + t.Parallel() + db, cleanup := setupTestDB(t) defer cleanup() ctx := context.Background() - repo := NewChunkFileRepository(db) - fileRepo := NewFileRepository(db) - chunksRepo := NewChunkRepository(db) + repo := database.NewChunkFileRepository(db) + fileRepo := database.NewFileRepository(db) + repos := database.NewRepositories(db) - // Create test files first - testTime := time.Now().Truncate(time.Second) - file1 := &File{ - Path: "/file1.txt", - MTime: testTime, - Size: 1024, - Mode: 0644, - UID: 1000, - GID: 1000, - LinkTarget: "", - } - - err := fileRepo.Create(ctx, nil, file1) - if err != nil { - t.Fatalf("failed to create file1: %v", err) - } - - file2 := &File{ - Path: "/file2.txt", - MTime: testTime, - Size: 1024, - Mode: 0644, - UID: 1000, - GID: 1000, - LinkTarget: "", - } - - err = fileRepo.Create(ctx, nil, file2) - if err != nil { - t.Fatalf("failed to create file2: %v", err) - } - - // Create chunk first - chunk := &Chunk{ - ChunkHash: types.ChunkHash("chunk1"), - Size: 1024, - } - - err = chunksRepo.Create(ctx, nil, chunk) - if err != nil { - t.Fatalf("failed to create chunk: %v", err) - } + file1, file2 := createChunkFileTestFiles(t, fileRepo) + mustCreateChunks(t, repos, chunk1Hash) // Test Create - cf1 := &ChunkFile{ - ChunkHash: types.ChunkHash("chunk1"), + cf1 := &database.ChunkFile{ + ChunkHash: types.ChunkHash(chunk1Hash), FileID: file1.ID, FileOffset: 0, Length: 1024, } - err = repo.Create(ctx, nil, cf1) + err := repo.Create(ctx, nil, cf1) if err != nil { t.Fatalf("failed to create chunk file: %v", err) } // Add same chunk in different file (deduplication scenario) - cf2 := &ChunkFile{ - ChunkHash: types.ChunkHash("chunk1"), + cf2 := &database.ChunkFile{ + ChunkHash: types.ChunkHash(chunk1Hash), FileID: file2.ID, FileOffset: 2048, Length: 1024, @@ -87,7 +111,7 @@ func TestChunkFileRepository(t *testing.T) { } // Test GetByChunkHash - chunkFiles, err := repo.GetByChunkHash(ctx, "chunk1") + chunkFiles, err := repo.GetByChunkHash(ctx, chunk1Hash) if err != nil { t.Fatalf("failed to get chunk files: %v", err) } @@ -97,22 +121,7 @@ func TestChunkFileRepository(t *testing.T) { } // Verify both files are returned - foundFile1 := false - foundFile2 := false - - for _, cf := range chunkFiles { - if cf.FileID == file1.ID && cf.FileOffset == 0 { - foundFile1 = true - } - - if cf.FileID == file2.ID && cf.FileOffset == 2048 { - foundFile2 = true - } - } - - if !foundFile1 || !foundFile2 { - t.Error("not all expected files found") - } + verifyChunkFilePair(t, chunkFiles, file1.ID, file2.ID) // Test GetByFileID chunkFiles, err = repo.GetByFileID(ctx, file1.ID) @@ -124,7 +133,7 @@ func TestChunkFileRepository(t *testing.T) { t.Errorf("expected 1 chunk for file, got %d", len(chunkFiles)) } - if chunkFiles[0].ChunkHash != types.ChunkHash("chunk1") { + if chunkFiles[0].ChunkHash != types.ChunkHash(chunk1Hash) { t.Errorf("wrong chunk hash: expected chunk1, got %s", chunkFiles[0].ChunkHash) } @@ -136,66 +145,53 @@ func TestChunkFileRepository(t *testing.T) { } func TestChunkFileRepositoryComplexDeduplication(t *testing.T) { + t.Parallel() + db, cleanup := setupTestDB(t) defer cleanup() ctx := context.Background() - repo := NewChunkFileRepository(db) - fileRepo := NewFileRepository(db) - chunksRepo := NewChunkRepository(db) + repo := database.NewChunkFileRepository(db) + fileRepo := database.NewFileRepository(db) + repos := database.NewRepositories(db) // Create test files testTime := time.Now().Truncate(time.Second) - file1 := &File{Path: "/file1.txt", MTime: testTime, Size: 3072, Mode: 0644, UID: 1000, GID: 1000} - file2 := &File{Path: "/file2.txt", MTime: testTime, Size: 3072, Mode: 0644, UID: 1000, GID: 1000} - file3 := &File{Path: "/file3.txt", MTime: testTime, Size: 2048, Mode: 0644, UID: 1000, GID: 1000} - - err := fileRepo.Create(ctx, nil, file1) - if err != nil { - t.Fatalf("failed to create file1: %v", err) + file1 := &database.File{ + Path: testFilePath1, MTime: testTime, Size: 3072, + Mode: 0644, UID: 1000, GID: 1000, + } + file2 := &database.File{ + Path: testFilePath2, MTime: testTime, Size: 3072, + Mode: 0644, UID: 1000, GID: 1000, + } + file3 := &database.File{ + Path: "/file3.txt", MTime: testTime, Size: 2048, + Mode: 0644, UID: 1000, GID: 1000, } - err = fileRepo.Create(ctx, nil, file2) - if err != nil { - t.Fatalf("failed to create file2: %v", err) - } - - err = fileRepo.Create(ctx, nil, file3) - if err != nil { - t.Fatalf("failed to create file3: %v", err) - } - - // Create chunks first - chunks := []types.ChunkHash{"chunk1", "chunk2", "chunk3", "chunk4"} - for _, chunkHash := range chunks { - chunk := &Chunk{ - ChunkHash: chunkHash, - Size: 1024, - } - - err := chunksRepo.Create(ctx, nil, chunk) - if err != nil { - t.Fatalf("failed to create chunk %s: %v", chunkHash, err) - } - } + mustCreateFile(t, fileRepo, file1) + mustCreateFile(t, fileRepo, file2) + mustCreateFile(t, fileRepo, file3) + mustCreateChunks(t, repos, chunk1Hash, chunk2Hash, chunk3Hash, chunk4Hash) // Simulate a scenario where multiple files share chunks // File1: chunk1, chunk2, chunk3 // File2: chunk2, chunk3, chunk4 // File3: chunk1, chunk4 - chunkFiles := []ChunkFile{ + chunkFiles := []database.ChunkFile{ // File1 - {ChunkHash: types.ChunkHash("chunk1"), FileID: file1.ID, FileOffset: 0, Length: 1024}, - {ChunkHash: types.ChunkHash("chunk2"), FileID: file1.ID, FileOffset: 1024, Length: 1024}, - {ChunkHash: types.ChunkHash("chunk3"), FileID: file1.ID, FileOffset: 2048, Length: 1024}, + {ChunkHash: chunk1Hash, FileID: file1.ID, FileOffset: 0, Length: 1024}, + {ChunkHash: chunk2Hash, FileID: file1.ID, FileOffset: 1024, Length: 1024}, + {ChunkHash: chunk3Hash, FileID: file1.ID, FileOffset: 2048, Length: 1024}, // File2 - {ChunkHash: types.ChunkHash("chunk2"), FileID: file2.ID, FileOffset: 0, Length: 1024}, - {ChunkHash: types.ChunkHash("chunk3"), FileID: file2.ID, FileOffset: 1024, Length: 1024}, - {ChunkHash: types.ChunkHash("chunk4"), FileID: file2.ID, FileOffset: 2048, Length: 1024}, + {ChunkHash: chunk2Hash, FileID: file2.ID, FileOffset: 0, Length: 1024}, + {ChunkHash: chunk3Hash, FileID: file2.ID, FileOffset: 1024, Length: 1024}, + {ChunkHash: chunk4Hash, FileID: file2.ID, FileOffset: 2048, Length: 1024}, // File3 - {ChunkHash: types.ChunkHash("chunk1"), FileID: file3.ID, FileOffset: 0, Length: 1024}, - {ChunkHash: types.ChunkHash("chunk4"), FileID: file3.ID, FileOffset: 1024, Length: 1024}, + {ChunkHash: chunk1Hash, FileID: file3.ID, FileOffset: 0, Length: 1024}, + {ChunkHash: chunk4Hash, FileID: file3.ID, FileOffset: 1024, Length: 1024}, } for _, cf := range chunkFiles { @@ -206,7 +202,7 @@ func TestChunkFileRepositoryComplexDeduplication(t *testing.T) { } // Test chunk1 (used by file1 and file3) - files, err := repo.GetByChunkHash(ctx, "chunk1") + files, err := repo.GetByChunkHash(ctx, chunk1Hash) if err != nil { t.Fatalf("failed to get files for chunk1: %v", err) } @@ -216,7 +212,7 @@ func TestChunkFileRepositoryComplexDeduplication(t *testing.T) { } // Test chunk2 (used by file1 and file2) - files, err = repo.GetByChunkHash(ctx, "chunk2") + files, err = repo.GetByChunkHash(ctx, chunk2Hash) if err != nil { t.Fatalf("failed to get files for chunk2: %v", err) } diff --git a/internal/database/chunks.go b/internal/database/chunks.go index 91b438b..dbd0c74 100644 --- a/internal/database/chunks.go +++ b/internal/database/chunks.go @@ -10,14 +10,18 @@ import ( "sneak.berlin/go/vaultik/internal/log" ) +// ChunkRepository provides access to the chunks table, which tracks +// content-defined chunks by hash and size. type ChunkRepository struct { db *DB } +// NewChunkRepository creates a ChunkRepository backed by db. func NewChunkRepository(db *DB) *ChunkRepository { return &ChunkRepository{db: db} } +// Create inserts a chunk row (idempotently), using tx when non-nil. func (r *ChunkRepository) Create(ctx context.Context, tx *sql.Tx, chunk *Chunk) error { query := ` INSERT INTO chunks (chunk_hash, size) @@ -39,6 +43,8 @@ func (r *ChunkRepository) Create(ctx context.Context, tx *sql.Tx, chunk *Chunk) return nil } +// GetByHash returns the chunk with the given hash, or nil if it is not +// known to the index. func (r *ChunkRepository) GetByHash(ctx context.Context, hash string) (*Chunk, error) { query := ` SELECT chunk_hash, size @@ -54,7 +60,7 @@ func (r *ChunkRepository) GetByHash(ctx context.Context, hash string) (*Chunk, e ) if errors.Is(err, sql.ErrNoRows) { - return nil, nil + return nil, nil //nolint:nilnil // nil,nil signals not-found; callers check nil } if err != nil { @@ -64,7 +70,11 @@ func (r *ChunkRepository) GetByHash(ctx context.Context, hash string) (*Chunk, e return &chunk, nil } -func (r *ChunkRepository) GetByHashes(ctx context.Context, hashes []string) ([]*Chunk, error) { +// GetByHashes returns the chunks whose hashes appear in hashes, ordered by +// chunk hash. Unknown hashes are silently omitted from the result. +func (r *ChunkRepository) GetByHashes( + ctx context.Context, hashes []string, +) ([]*Chunk, error) { if len(hashes) == 0 { return nil, nil } @@ -88,7 +98,7 @@ func (r *ChunkRepository) GetByHashes(ctx context.Context, hashes []string) ([]* args[i] = hash } - query += querySb75.String() + query += querySb75.String() //nolint:gosec // G202: appends "?" placeholders only query += ") ORDER BY chunk_hash" @@ -117,7 +127,11 @@ func (r *ChunkRepository) GetByHashes(ctx context.Context, hashes []string) ([]* return chunks, rows.Err() } -func (r *ChunkRepository) ListUnpacked(ctx context.Context, limit int) ([]*Chunk, error) { +// ListUnpacked returns up to limit chunks that are not yet stored in any +// blob, ordered by chunk hash. +func (r *ChunkRepository) ListUnpacked( + ctx context.Context, limit int, +) ([]*Chunk, error) { query := ` SELECT c.chunk_hash, c.size FROM chunks c diff --git a/internal/database/chunks_ext.go b/internal/database/chunks_ext.go index a322a9a..fee4030 100644 --- a/internal/database/chunks_ext.go +++ b/internal/database/chunks_ext.go @@ -5,6 +5,7 @@ import ( "fmt" ) +// List returns every chunk in the index, ordered by chunk hash. func (r *ChunkRepository) List(ctx context.Context) ([]*Chunk, error) { query := ` SELECT chunk_hash, size diff --git a/internal/database/chunks_test.go b/internal/database/chunks_test.go index 8f96bd1..f0d6e52 100644 --- a/internal/database/chunks_test.go +++ b/internal/database/chunks_test.go @@ -1,21 +1,24 @@ -package database +package database_test import ( "context" "testing" + "sneak.berlin/go/vaultik/internal/database" "sneak.berlin/go/vaultik/internal/types" ) func TestChunkRepository(t *testing.T) { + t.Parallel() + db, cleanup := setupTestDB(t) defer cleanup() ctx := context.Background() - repo := NewChunkRepository(db) + repo := database.NewChunkRepository(db) // Test Create - chunk := &Chunk{ + chunk := &database.Chunk{ ChunkHash: types.ChunkHash("chunkhash123"), Size: 4096, } @@ -50,7 +53,7 @@ func TestChunkRepository(t *testing.T) { } // Test GetByHashes - chunk2 := &Chunk{ + chunk2 := &database.Chunk{ ChunkHash: types.ChunkHash("chunkhash456"), Size: 8192, } @@ -60,7 +63,9 @@ func TestChunkRepository(t *testing.T) { t.Fatalf("failed to create second chunk: %v", err) } - chunks, err := repo.GetByHashes(ctx, []string{chunk.ChunkHash.String(), chunk2.ChunkHash.String()}) + chunks, err := repo.GetByHashes(ctx, []string{ + chunk.ChunkHash.String(), chunk2.ChunkHash.String(), + }) if err != nil { t.Fatalf("failed to get chunks by hashes: %v", err) } @@ -81,11 +86,13 @@ func TestChunkRepository(t *testing.T) { } func TestChunkRepositoryNotFound(t *testing.T) { + t.Parallel() + db, cleanup := setupTestDB(t) defer cleanup() ctx := context.Background() - repo := NewChunkRepository(db) + repo := database.NewChunkRepository(db) // Test GetByHash with non-existent hash chunk, err := repo.GetByHash(ctx, "nonexistent") diff --git a/internal/database/database.go b/internal/database/database.go index 2d2dba3..b272637 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -15,6 +15,7 @@ import ( "context" "database/sql" "embed" + "errors" "fmt" "os" "path/filepath" @@ -22,10 +23,15 @@ import ( "strconv" "strings" + // Register the pure-Go sqlite driver. _ "modernc.org/sqlite" "sneak.berlin/go/vaultik/internal/log" ) +// errInvalidMigrationFilename is returned when an embedded migration file +// does not follow the "[_].sql" naming pattern. +var errInvalidMigrationFilename = errors.New("invalid migration filename") + //go:embed schema/*.sql var schemaFS embed.FS @@ -51,7 +57,7 @@ type DB struct { func ParseMigrationVersion(filename string) (int, error) { name := strings.TrimSuffix(filename, filepath.Ext(filename)) if name == "" { - return 0, fmt.Errorf("invalid migration filename %q: empty name", filename) + return 0, fmt.Errorf("%w %q: empty name", errInvalidMigrationFilename, filename) } // Split on underscore to separate version from description. @@ -62,15 +68,17 @@ func ParseMigrationVersion(filename string) (int, error) { } if versionStr == "" { - return 0, fmt.Errorf("invalid migration filename %q: empty version prefix", filename) + return 0, fmt.Errorf( + "%w %q: empty version prefix", errInvalidMigrationFilename, filename, + ) } // Validate the version is purely numeric. for _, ch := range versionStr { if ch < '0' || ch > '9' { return 0, fmt.Errorf( - "invalid migration filename %q: version %q contains non-numeric character %q", - filename, versionStr, string(ch), + "%w %q: version %q contains non-numeric character %q", + errInvalidMigrationFilename, filename, versionStr, string(ch), ) } } @@ -101,68 +109,87 @@ func New(ctx context.Context, path string) (*DB, error) { conn, err := sql.Open( "sqlite", - path+"?_journal_mode=WAL&_synchronous=NORMAL&_busy_timeout=10000&_locking_mode=NORMAL&_foreign_keys=ON", + path+"?_journal_mode=WAL&_synchronous=NORMAL&_busy_timeout=10000"+ + "&_locking_mode=NORMAL&_foreign_keys=ON", ) if err == nil { - // Set connection pool settings - // SQLite can handle multiple readers but only one writer at a time. - // Setting MaxOpenConns to 1 ensures all writes are serialized through - // a single connection, preventing SQLITE_BUSY errors. - conn.SetMaxOpenConns(1) - conn.SetMaxIdleConns(1) + configureConnPool(conn) - err := conn.PingContext(ctx) + err = conn.PingContext(ctx) if err == nil { // Success on first try log.Debug("Database opened successfully with WAL mode", "path", path) - // Enable foreign keys explicitly - _, err = conn.ExecContext(ctx, "PRAGMA foreign_keys = ON") - if err != nil { - log.Warn("Failed to enable foreign keys", "error", err) - } - - db := &DB{conn: conn, path: path} - - err := applyMigrations(ctx, conn) - if err != nil { - _ = conn.Close() - - return nil, fmt.Errorf("applying migrations: %w", err) - } - - return db, nil + return finishOpen(ctx, conn, path) } - log.Debug("Failed to ping database, closing connection", "path", path, "error", err) + log.Debug( + "Failed to ping database, closing connection", + "path", path, "error", err, + ) _ = conn.Close() } // If first attempt failed, try with TRUNCATE mode to clear any locks + return openWithRecovery(ctx, path) +} + +// configureConnPool serializes all database access through one connection. +// SQLite can handle multiple readers but only one writer at a time; setting +// MaxOpenConns to 1 ensures all writes go through a single connection, +// preventing SQLITE_BUSY errors. +func configureConnPool(conn *sql.DB) { + conn.SetMaxOpenConns(1) + conn.SetMaxIdleConns(1) +} + +// finishOpen enables foreign keys, wraps the connection, and applies any +// pending migrations. On migration failure the connection is closed. +func finishOpen(ctx context.Context, conn *sql.DB, path string) (*DB, error) { + // Enable foreign keys explicitly + _, err := conn.ExecContext(ctx, "PRAGMA foreign_keys = ON") + if err != nil { + log.Warn("Failed to enable foreign keys", "path", path, "error", err) + } + + db := &DB{conn: conn, path: path} + + err = applyMigrations(ctx, conn) + if err != nil { + _ = conn.Close() + + return nil, fmt.Errorf("applying migrations: %w", err) + } + + return db, nil +} + +// openWithRecovery retries opening the database in TRUNCATE journal mode to +// clear stale locks, then switches back to WAL mode. +func openWithRecovery(ctx context.Context, path string) (*DB, error) { log.Info( "Database appears locked, attempting recovery with TRUNCATE mode", "path", path, ) - conn, err = sql.Open( + conn, err := sql.Open( "sqlite", - path+"?_journal_mode=TRUNCATE&_synchronous=NORMAL&_busy_timeout=10000&_foreign_keys=ON", + path+"?_journal_mode=TRUNCATE&_synchronous=NORMAL&_busy_timeout=10000"+ + "&_foreign_keys=ON", ) if err != nil { return nil, fmt.Errorf("opening database in recovery mode: %w", err) } - // Set connection pool settings - // SQLite can handle multiple readers but only one writer at a time. - // Setting MaxOpenConns to 1 ensures all writes are serialized through - // a single connection, preventing SQLITE_BUSY errors. - conn.SetMaxOpenConns(1) - conn.SetMaxIdleConns(1) + configureConnPool(conn) err = conn.PingContext(ctx) if err != nil { - log.Debug("Failed to ping database in recovery mode, closing", "path", path, "error", err) + log.Debug( + "Failed to ping database in recovery mode, closing", + "path", path, "error", err, + ) _ = conn.Close() @@ -182,19 +209,9 @@ func New(ctx context.Context, path string) (*DB, error) { log.Warn("Failed to switch back to WAL mode", "path", path, "error", err) } - // Ensure foreign keys are enabled - _, err = conn.ExecContext(ctx, "PRAGMA foreign_keys=ON") + db, err := finishOpen(ctx, conn, path) if err != nil { - log.Warn("Failed to enable foreign keys", "path", path, "error", err) - } - - db := &DB{conn: conn, path: path} - - err = applyMigrations(ctx, conn) - if err != nil { - _ = conn.Close() - - return nil, fmt.Errorf("applying migrations: %w", err) + return nil, err } log.Debug("Database connection established successfully", "path", path) @@ -202,6 +219,13 @@ func New(ctx context.Context, path string) (*DB, error) { return db, nil } +// NewTestDB creates an in-memory SQLite database for testing purposes. +// The database is automatically initialized with the schema and is ready +// for use. Each call creates a new independent database instance. +func NewTestDB() (*DB, error) { + return New(context.Background(), ":memory:") +} + // Close closes the database connection. // It ensures all pending operations are completed before closing. // Returns an error if the database connection cannot be closed properly. @@ -259,10 +283,11 @@ func (db *DB) ExecWithLog( return db.conn.ExecContext(ctx, query, args...) } -// QueryRowWithLog executes a query that returns at most one row with SQL logging. -// This is useful for queries that modify data and return values (e.g., INSERT ... RETURNING). -// SQLite handles its own locking internally. -// The query and args parameters follow the same format as sql.DB.QueryRowContext. +// QueryRowWithLog executes a query that returns at most one row with SQL +// logging. This is useful for queries that modify data and return values +// (e.g., INSERT ... RETURNING). SQLite handles its own locking internally. +// The query and args parameters follow the same format as +// sql.DB.QueryRowContext. func (db *DB) QueryRowWithLog( ctx context.Context, query string, @@ -390,15 +415,8 @@ func applyMigrations(ctx context.Context, db *sql.DB) error { return nil } -// NewTestDB creates an in-memory SQLite database for testing purposes. -// The database is automatically initialized with the schema and is ready for use. -// Each call creates a new independent database instance. -func NewTestDB() (*DB, error) { - return New(context.Background(), ":memory:") -} - -// repeatPlaceholder generates a string of ", ?" repeated n times for IN clause construction. -// For example, repeatPlaceholder(2) returns ", ?, ?". +// repeatPlaceholder generates a string of ", ?" repeated n times for IN +// clause construction. For example, repeatPlaceholder(2) returns ", ?, ?". func repeatPlaceholder(n int) string { if n <= 0 { return "" @@ -408,12 +426,14 @@ func repeatPlaceholder(n int) string { } // LogSQL logs SQL queries and their arguments when debug mode is enabled. -// Debug mode is activated by setting the GODEBUG environment variable to include "vaultik". -// This is useful for troubleshooting database operations and understanding query patterns. +// Debug mode is activated by setting the GODEBUG environment variable to +// include "vaultik". This is useful for troubleshooting database operations +// and understanding query patterns. // -// The operation parameter describes the type of SQL operation (e.g., "Execute", "Query"). -// The query parameter is the SQL statement being executed. -// The args parameter contains the query arguments that will be interpolated. +// The operation parameter describes the type of SQL operation (e.g., +// "Execute", "Query"). The query parameter is the SQL statement being +// executed. The args parameter contains the query arguments that will be +// interpolated. func LogSQL(operation, query string, args ...any) { if strings.Contains(os.Getenv("GODEBUG"), "vaultik") { log.Debug( diff --git a/internal/database/database_test.go b/internal/database/database_test.go index 0e5d2d8..9e1c214 100644 --- a/internal/database/database_test.go +++ b/internal/database/database_test.go @@ -1,3 +1,4 @@ +//nolint:testpackage // exercises unexported migration internals package database import ( @@ -9,6 +10,8 @@ import ( ) func TestDatabase(t *testing.T) { + t.Parallel() + ctx := context.Background() dbPath := filepath.Join(t.TempDir(), "test.db") @@ -39,7 +42,9 @@ func TestDatabase(t *testing.T) { for _, table := range tables { var name string - err := db.conn.QueryRow("SELECT name FROM sqlite_master WHERE type='table' AND name=?", table).Scan(&name) + err := db.conn.QueryRowContext(ctx, + "SELECT name FROM sqlite_master WHERE type='table' AND name=?", table, + ).Scan(&name) if err != nil { t.Errorf("table %s does not exist: %v", table, err) } @@ -47,6 +52,8 @@ func TestDatabase(t *testing.T) { } func TestDatabaseInvalidPath(t *testing.T) { + t.Parallel() + ctx := context.Background() // Test with invalid path @@ -57,6 +64,8 @@ func TestDatabaseInvalidPath(t *testing.T) { } func TestDatabaseConcurrentAccess(t *testing.T) { + t.Parallel() + ctx := context.Background() dbPath := filepath.Join(t.TempDir(), "test.db") @@ -81,7 +90,8 @@ func TestDatabaseConcurrentAccess(t *testing.T) { for i := range 10 { go func(i int) { - _, err := db.ExecWithLog(ctx, "INSERT INTO chunks (chunk_hash, size) VALUES (?, ?)", + _, err := db.ExecWithLog(ctx, + "INSERT INTO chunks (chunk_hash, size) VALUES (?, ?)", fmt.Sprintf("hash%d", i), i*1024) results <- result{index: i, err: err} }(i) @@ -109,6 +119,8 @@ func TestDatabaseConcurrentAccess(t *testing.T) { } func TestParseMigrationVersion(t *testing.T) { + t.Parallel() + tests := []struct { name string filename string @@ -118,8 +130,14 @@ func TestParseMigrationVersion(t *testing.T) { {name: "valid 000.sql", filename: "000.sql", wantVer: 0, wantError: false}, {name: "valid 001.sql", filename: "001.sql", wantVer: 1, wantError: false}, {name: "valid 099.sql", filename: "099.sql", wantVer: 99, wantError: false}, - {name: "valid with description", filename: "001_initial_schema.sql", wantVer: 1, wantError: false}, - {name: "valid large version", filename: "123_big_migration.sql", wantVer: 123, wantError: false}, + { + name: "valid with description", filename: "001_initial_schema.sql", + wantVer: 1, wantError: false, + }, + { + name: "valid large version", filename: "123_big_migration.sql", + wantVer: 123, wantError: false, + }, {name: "invalid alpha version", filename: "abc.sql", wantVer: 0, wantError: true}, {name: "invalid mixed chars", filename: "12a.sql", wantVer: 0, wantError: true}, {name: "invalid no extension", filename: "schema.sql", wantVer: 0, wantError: true}, @@ -128,29 +146,36 @@ func TestParseMigrationVersion(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { + t.Parallel() + got, err := ParseMigrationVersion(tc.filename) if tc.wantError { if err == nil { - t.Errorf("ParseMigrationVersion(%q) = %d, nil; want error", tc.filename, got) + t.Errorf("ParseMigrationVersion(%q) = %d, nil; want error", + tc.filename, got) } return } if err != nil { - t.Errorf("ParseMigrationVersion(%q) unexpected error: %v", tc.filename, err) + t.Errorf("ParseMigrationVersion(%q) unexpected error: %v", + tc.filename, err) return } if got != tc.wantVer { - t.Errorf("ParseMigrationVersion(%q) = %d; want %d", tc.filename, got, tc.wantVer) + t.Errorf("ParseMigrationVersion(%q) = %d; want %d", + tc.filename, got, tc.wantVer) } }) } } func TestApplyMigrations_Idempotent(t *testing.T) { + t.Parallel() + ctx := context.Background() conn, err := sql.Open("sqlite", ":memory:?_foreign_keys=ON") @@ -176,7 +201,9 @@ func TestApplyMigrations_Idempotent(t *testing.T) { // Count rows in schema_migrations after first run. var countBefore int - err = conn.QueryRowContext(ctx, "SELECT COUNT(*) FROM schema_migrations").Scan(&countBefore) + err = conn.QueryRowContext(ctx, + "SELECT COUNT(*) FROM schema_migrations", + ).Scan(&countBefore) if err != nil { t.Fatalf("failed to count schema_migrations after first run: %v", err) } @@ -190,17 +217,22 @@ func TestApplyMigrations_Idempotent(t *testing.T) { // Count rows in schema_migrations after second run — must be unchanged. var countAfter int - err = conn.QueryRowContext(ctx, "SELECT COUNT(*) FROM schema_migrations").Scan(&countAfter) + err = conn.QueryRowContext(ctx, + "SELECT COUNT(*) FROM schema_migrations", + ).Scan(&countAfter) if err != nil { t.Fatalf("failed to count schema_migrations after second run: %v", err) } if countBefore != countAfter { - t.Errorf("schema_migrations row count changed: before=%d, after=%d", countBefore, countAfter) + t.Errorf("schema_migrations row count changed: before=%d, after=%d", + countBefore, countAfter) } } func TestBootstrapMigrationsTable_FreshDatabase(t *testing.T) { + t.Parallel() + ctx := context.Background() conn, err := sql.Open("sqlite", ":memory:?_foreign_keys=ON") @@ -248,7 +280,8 @@ func TestBootstrapMigrationsTable_FreshDatabase(t *testing.T) { } if tableAfter != 1 { - t.Fatalf("schema_migrations table should exist after bootstrap, got count=%d", tableAfter) + t.Fatalf("schema_migrations table should exist after bootstrap, got count=%d", + tableAfter) } // Verify version 0 row exists. diff --git a/internal/database/errors.go b/internal/database/errors.go index 59f1279..860c23f 100644 --- a/internal/database/errors.go +++ b/internal/database/errors.go @@ -6,8 +6,8 @@ import ( "os" ) -// Fatal prints an error message to stderr and exits with status 1 -func Fatal(format string, args ...any) { +// Fatalf prints an error message to stderr and exits with status 1 +func Fatalf(format string, args ...any) { fmt.Fprintf(os.Stderr, "FATAL: "+format+"\n", args...) os.Exit(1) } @@ -16,6 +16,6 @@ func Fatal(format string, args ...any) { func CloseRows(rows *sql.Rows) { err := rows.Close() if err != nil { - Fatal("failed to close rows: %v", err) + Fatalf("failed to close rows: %v", err) } } diff --git a/internal/database/file_chunks.go b/internal/database/file_chunks.go index c6b32f4..11711f7 100644 --- a/internal/database/file_chunks.go +++ b/internal/database/file_chunks.go @@ -9,15 +9,21 @@ import ( "sneak.berlin/go/vaultik/internal/types" ) +// FileChunkRepository provides access to the file_chunks table, which maps +// files to their ordered constituent chunks. type FileChunkRepository struct { db *DB } +// NewFileChunkRepository creates a FileChunkRepository backed by db. func NewFileChunkRepository(db *DB) *FileChunkRepository { return &FileChunkRepository{db: db} } -func (r *FileChunkRepository) Create(ctx context.Context, tx *sql.Tx, fc *FileChunk) error { +// Create inserts a file_chunks row (idempotently), using tx when non-nil. +func (r *FileChunkRepository) Create( + ctx context.Context, tx *sql.Tx, fc *FileChunk, +) error { query := ` INSERT INTO file_chunks (file_id, idx, chunk_hash) VALUES (?, ?, ?) @@ -28,7 +34,8 @@ func (r *FileChunkRepository) Create(ctx context.Context, tx *sql.Tx, fc *FileCh if tx != nil { _, err = tx.ExecContext(ctx, query, fc.FileID.String(), fc.Idx, fc.ChunkHash.String()) } else { - _, err = r.db.ExecWithLog(ctx, query, fc.FileID.String(), fc.Idx, fc.ChunkHash.String()) + _, err = r.db.ExecWithLog(ctx, query, + fc.FileID.String(), fc.Idx, fc.ChunkHash.String()) } if err != nil { @@ -38,7 +45,10 @@ func (r *FileChunkRepository) Create(ctx context.Context, tx *sql.Tx, fc *FileCh return nil } -func (r *FileChunkRepository) GetByPath(ctx context.Context, path string) ([]*FileChunk, error) { +// GetByPath returns the ordered chunks of the file at the given path. +func (r *FileChunkRepository) GetByPath( + ctx context.Context, path string, +) ([]*FileChunk, error) { query := ` SELECT fc.file_id, fc.idx, fc.chunk_hash FROM file_chunks fc @@ -57,7 +67,9 @@ func (r *FileChunkRepository) GetByPath(ctx context.Context, path string) ([]*Fi } // GetByFileID retrieves file chunks by file ID -func (r *FileChunkRepository) GetByFileID(ctx context.Context, fileID types.FileID) ([]*FileChunk, error) { +func (r *FileChunkRepository) GetByFileID( + ctx context.Context, fileID types.FileID, +) ([]*FileChunk, error) { query := ` SELECT file_id, idx, chunk_hash FROM file_chunks @@ -75,7 +87,9 @@ func (r *FileChunkRepository) GetByFileID(ctx context.Context, fileID types.File } // GetByPathTx retrieves file chunks within a transaction -func (r *FileChunkRepository) GetByPathTx(ctx context.Context, tx *sql.Tx, path string) ([]*FileChunk, error) { +func (r *FileChunkRepository) GetByPathTx( + ctx context.Context, tx *sql.Tx, path string, +) ([]*FileChunk, error) { query := ` SELECT fc.file_id, fc.idx, fc.chunk_hash FROM file_chunks fc @@ -98,6 +112,170 @@ func (r *FileChunkRepository) GetByPathTx(ctx context.Context, tx *sql.Tx, path return fileChunks, err } +// DeleteByPath deletes all file_chunks rows for the file at the given path. +func (r *FileChunkRepository) DeleteByPath( + ctx context.Context, tx *sql.Tx, path string, +) error { + query := ` + DELETE FROM file_chunks + WHERE file_id = (SELECT id FROM files WHERE path = ?) + ` + + var err error + if tx != nil { + _, err = tx.ExecContext(ctx, query, path) + } else { + _, err = r.db.ExecWithLog(ctx, query, path) + } + + if err != nil { + return fmt.Errorf("deleting file chunks: %w", err) + } + + return nil +} + +// DeleteByFileID deletes all chunks for a file by its UUID +func (r *FileChunkRepository) DeleteByFileID( + ctx context.Context, tx *sql.Tx, fileID types.FileID, +) error { + query := `DELETE FROM file_chunks WHERE file_id = ?` + + var err error + if tx != nil { + _, err = tx.ExecContext(ctx, query, fileID.String()) + } else { + _, err = r.db.ExecWithLog(ctx, query, fileID.String()) + } + + if err != nil { + return fmt.Errorf("deleting file chunks: %w", err) + } + + return nil +} + +// DeleteByFileIDs deletes all chunks for multiple files in a single statement. +// +//nolint:dupl // symmetric implementation for a parallel association table +func (r *FileChunkRepository) DeleteByFileIDs( + ctx context.Context, tx *sql.Tx, fileIDs []types.FileID, +) error { + if len(fileIDs) == 0 { + return nil + } + + // Batch at 500 to stay within SQLite's variable limit + const batchSize = 500 + + for i := 0; i < len(fileIDs); i += batchSize { + end := min(i+batchSize, len(fileIDs)) + + batch := fileIDs[i:end] + + //nolint:gosec // G202: concatenates constant SQL and "?" placeholders only + query := "DELETE FROM file_chunks WHERE file_id IN (?" + + repeatPlaceholder(len(batch)-1) + ")" + + args := make([]any, len(batch)) + for j, id := range batch { + args[j] = id.String() + } + + var err error + if tx != nil { + _, err = tx.ExecContext(ctx, query, args...) + } else { + _, err = r.db.ExecWithLog(ctx, query, args...) + } + + if err != nil { + return fmt.Errorf("batch deleting file_chunks: %w", err) + } + } + + return nil +} + +// CreateBatch inserts multiple file_chunks in a single statement for efficiency. +// Batches are automatically split to stay within SQLite's variable limit. +func (r *FileChunkRepository) CreateBatch( + ctx context.Context, tx *sql.Tx, fcs []FileChunk, +) error { + if len(fcs) == 0 { + return nil + } + + // Each file_chunks row binds this many SQL variables. + const fileChunkCols = 3 + + // SQLite has a limit on variables (typically 999 or 32766), so batch + // at 300 rows to be safe. + const batchSize = 300 + + for i := 0; i < len(fcs); i += batchSize { + end := min(i+batchSize, len(fcs)) + + batch := fcs[i:end] + + // Build the query with multiple value sets + query := "INSERT INTO file_chunks (file_id, idx, chunk_hash) VALUES " + + args := make([]any, 0, len(batch)*fileChunkCols) + + var querySb211 strings.Builder + + for j, fc := range batch { + if j > 0 { + querySb211.WriteString(", ") + } + + querySb211.WriteString("(?, ?, ?)") + + args = append(args, fc.FileID.String(), fc.Idx, fc.ChunkHash.String()) + } + + query += querySb211.String() //nolint:gosec // G202: appends "?" placeholders only + + query += " ON CONFLICT(file_id, idx) DO NOTHING" + + var err error + if tx != nil { + _, err = tx.ExecContext(ctx, query, args...) + } else { + _, err = r.db.ExecWithLog(ctx, query, args...) + } + + if err != nil { + return fmt.Errorf("batch inserting file_chunks: %w", err) + } + } + + return nil +} + +// GetByFile is an alias for GetByPath for compatibility +func (r *FileChunkRepository) GetByFile( + ctx context.Context, path string, +) ([]*FileChunk, error) { + LogSQL("GetByFile", "Starting", path) + result, err := r.GetByPath(ctx, path) + LogSQL("GetByFile", "Complete", path, "count", len(result)) + + return result, err +} + +// GetByFileTx retrieves file chunks within a transaction +func (r *FileChunkRepository) GetByFileTx( + ctx context.Context, tx *sql.Tx, path string, +) ([]*FileChunk, error) { + LogSQL("GetByFileTx", "Starting", path) + result, err := r.GetByPathTx(ctx, tx, path) + LogSQL("GetByFileTx", "Complete", path, "count", len(result)) + + return result, err +} + // scanFileChunks is a helper that scans file chunk rows func (r *FileChunkRepository) scanFileChunks(rows *sql.Rows) ([]*FileChunk, error) { var fileChunks []*FileChunk @@ -124,144 +302,3 @@ func (r *FileChunkRepository) scanFileChunks(rows *sql.Rows) ([]*FileChunk, erro return fileChunks, rows.Err() } - -func (r *FileChunkRepository) DeleteByPath(ctx context.Context, tx *sql.Tx, path string) error { - query := `DELETE FROM file_chunks WHERE file_id = (SELECT id FROM files WHERE path = ?)` - - var err error - if tx != nil { - _, err = tx.ExecContext(ctx, query, path) - } else { - _, err = r.db.ExecWithLog(ctx, query, path) - } - - if err != nil { - return fmt.Errorf("deleting file chunks: %w", err) - } - - return nil -} - -// DeleteByFileID deletes all chunks for a file by its UUID -func (r *FileChunkRepository) DeleteByFileID(ctx context.Context, tx *sql.Tx, fileID types.FileID) error { - query := `DELETE FROM file_chunks WHERE file_id = ?` - - var err error - if tx != nil { - _, err = tx.ExecContext(ctx, query, fileID.String()) - } else { - _, err = r.db.ExecWithLog(ctx, query, fileID.String()) - } - - if err != nil { - return fmt.Errorf("deleting file chunks: %w", err) - } - - return nil -} - -// DeleteByFileIDs deletes all chunks for multiple files in a single statement. -func (r *FileChunkRepository) DeleteByFileIDs(ctx context.Context, tx *sql.Tx, fileIDs []types.FileID) error { - if len(fileIDs) == 0 { - return nil - } - - // Batch at 500 to stay within SQLite's variable limit - const batchSize = 500 - - for i := 0; i < len(fileIDs); i += batchSize { - end := min(i+batchSize, len(fileIDs)) - - batch := fileIDs[i:end] - - query := "DELETE FROM file_chunks WHERE file_id IN (?" + repeatPlaceholder(len(batch)-1) + ")" - - args := make([]any, len(batch)) - for j, id := range batch { - args[j] = id.String() - } - - var err error - if tx != nil { - _, err = tx.ExecContext(ctx, query, args...) - } else { - _, err = r.db.ExecWithLog(ctx, query, args...) - } - - if err != nil { - return fmt.Errorf("batch deleting file_chunks: %w", err) - } - } - - return nil -} - -// CreateBatch inserts multiple file_chunks in a single statement for efficiency. -// Batches are automatically split to stay within SQLite's variable limit. -func (r *FileChunkRepository) CreateBatch(ctx context.Context, tx *sql.Tx, fcs []FileChunk) error { - if len(fcs) == 0 { - return nil - } - - // SQLite has a limit on variables (typically 999 or 32766). - // Each FileChunk has 3 values, so batch at 300 to be safe. - const batchSize = 300 - - for i := 0; i < len(fcs); i += batchSize { - end := min(i+batchSize, len(fcs)) - - batch := fcs[i:end] - - // Build the query with multiple value sets - query := "INSERT INTO file_chunks (file_id, idx, chunk_hash) VALUES " - - args := make([]any, 0, len(batch)*3) - - var querySb211 strings.Builder - - for j, fc := range batch { - if j > 0 { - querySb211.WriteString(", ") - } - - querySb211.WriteString("(?, ?, ?)") - - args = append(args, fc.FileID.String(), fc.Idx, fc.ChunkHash.String()) - } - - query += querySb211.String() - - query += " ON CONFLICT(file_id, idx) DO NOTHING" - - var err error - if tx != nil { - _, err = tx.ExecContext(ctx, query, args...) - } else { - _, err = r.db.ExecWithLog(ctx, query, args...) - } - - if err != nil { - return fmt.Errorf("batch inserting file_chunks: %w", err) - } - } - - return nil -} - -// GetByFile is an alias for GetByPath for compatibility -func (r *FileChunkRepository) GetByFile(ctx context.Context, path string) ([]*FileChunk, error) { - LogSQL("GetByFile", "Starting", path) - result, err := r.GetByPath(ctx, path) - LogSQL("GetByFile", "Complete", path, "count", len(result)) - - return result, err -} - -// GetByFileTx retrieves file chunks within a transaction -func (r *FileChunkRepository) GetByFileTx(ctx context.Context, tx *sql.Tx, path string) ([]*FileChunk, error) { - LogSQL("GetByFileTx", "Starting", path) - result, err := r.GetByPathTx(ctx, tx, path) - LogSQL("GetByFileTx", "Complete", path, "count", len(result)) - - return result, err -} diff --git a/internal/database/file_chunks_test.go b/internal/database/file_chunks_test.go index e1eb2b9..7fe58e2 100644 --- a/internal/database/file_chunks_test.go +++ b/internal/database/file_chunks_test.go @@ -1,4 +1,4 @@ -package database +package database_test import ( "context" @@ -6,21 +6,25 @@ import ( "testing" "time" + "sneak.berlin/go/vaultik/internal/database" "sneak.berlin/go/vaultik/internal/types" ) func TestFileChunkRepository(t *testing.T) { + t.Parallel() + db, cleanup := setupTestDB(t) defer cleanup() ctx := context.Background() - repo := NewFileChunkRepository(db) - fileRepo := NewFileRepository(db) + repo := database.NewFileChunkRepository(db) + fileRepo := database.NewFileRepository(db) + repos := database.NewRepositories(db) // Create test file first testTime := time.Now().Truncate(time.Second) - file := &File{ - Path: "/test/file.txt", + file := &database.File{ + Path: testFileTxt, MTime: testTime, Size: 3072, Mode: 0644, @@ -29,44 +33,26 @@ func TestFileChunkRepository(t *testing.T) { LinkTarget: "", } - err := fileRepo.Create(ctx, nil, file) - if err != nil { - t.Fatalf("failed to create file: %v", err) - } - - // Create chunks first - chunks := []types.ChunkHash{"chunk1", "chunk2", "chunk3"} - chunkRepo := NewChunkRepository(db) - - for _, chunkHash := range chunks { - chunk := &Chunk{ - ChunkHash: chunkHash, - Size: 1024, - } - - err = chunkRepo.Create(ctx, nil, chunk) - if err != nil { - t.Fatalf("failed to create chunk %s: %v", chunkHash, err) - } - } + mustCreateFile(t, fileRepo, file) + mustCreateChunks(t, repos, chunk1Hash, chunk2Hash, chunk3Hash) // Test Create - fc1 := &FileChunk{ + fc1 := &database.FileChunk{ FileID: file.ID, Idx: 0, - ChunkHash: types.ChunkHash("chunk1"), + ChunkHash: types.ChunkHash(chunk1Hash), } - err = repo.Create(ctx, nil, fc1) + err := repo.Create(ctx, nil, fc1) if err != nil { t.Fatalf("failed to create file chunk: %v", err) } // Add more chunks for the same file - fc2 := &FileChunk{ + fc2 := &database.FileChunk{ FileID: file.ID, Idx: 1, - ChunkHash: types.ChunkHash("chunk2"), + ChunkHash: types.ChunkHash(chunk2Hash), } err = repo.Create(ctx, nil, fc2) @@ -74,10 +60,10 @@ func TestFileChunkRepository(t *testing.T) { t.Fatalf("failed to create second file chunk: %v", err) } - fc3 := &FileChunk{ + fc3 := &database.FileChunk{ FileID: file.ID, Idx: 2, - ChunkHash: types.ChunkHash("chunk3"), + ChunkHash: types.ChunkHash(chunk3Hash), } err = repo.Create(ctx, nil, fc3) @@ -86,7 +72,7 @@ func TestFileChunkRepository(t *testing.T) { } // Test GetByFile - fileChunks, err := repo.GetByFile(ctx, "/test/file.txt") + fileChunks, err := repo.GetByFile(ctx, testFileTxt) if err != nil { t.Fatalf("failed to get file chunks: %v", err) } @@ -107,6 +93,41 @@ func TestFileChunkRepository(t *testing.T) { if err != nil { t.Fatalf("failed to create duplicate file chunk: %v", err) } +} + +func TestFileChunkRepositoryDeleteByFileID(t *testing.T) { + t.Parallel() + + db, cleanup := setupTestDB(t) + defer cleanup() + + ctx := context.Background() + repo := database.NewFileChunkRepository(db) + fileRepo := database.NewFileRepository(db) + repos := database.NewRepositories(db) + + file := &database.File{ + Path: testFileTxt, + MTime: time.Now().Truncate(time.Second), + Size: 1024, + Mode: 0644, + UID: 1000, + GID: 1000, + } + + mustCreateFile(t, fileRepo, file) + mustCreateChunks(t, repos, chunk1Hash) + + fc := &database.FileChunk{ + FileID: file.ID, + Idx: 0, + ChunkHash: types.ChunkHash(chunk1Hash), + } + + err := repo.Create(ctx, nil, fc) + if err != nil { + t.Fatalf("failed to create file chunk: %v", err) + } // Test DeleteByFileID err = repo.DeleteByFileID(ctx, nil, file.ID) @@ -114,7 +135,7 @@ func TestFileChunkRepository(t *testing.T) { t.Fatalf("failed to delete file chunks: %v", err) } - fileChunks, err = repo.GetByFileID(ctx, file.ID) + fileChunks, err := repo.GetByFileID(ctx, file.ID) if err != nil { t.Fatalf("failed to get deleted file chunks: %v", err) } @@ -125,20 +146,22 @@ func TestFileChunkRepository(t *testing.T) { } func TestFileChunkRepositoryMultipleFiles(t *testing.T) { + t.Parallel() + db, cleanup := setupTestDB(t) defer cleanup() ctx := context.Background() - repo := NewFileChunkRepository(db) - fileRepo := NewFileRepository(db) + repo := database.NewFileChunkRepository(db) + fileRepo := database.NewFileRepository(db) // Create test files testTime := time.Now().Truncate(time.Second) - filePaths := []string{"/file1.txt", "/file2.txt", "/file3.txt"} - files := make([]*File, len(filePaths)) + filePaths := []string{testFilePath1, testFilePath2, "/file3.txt"} + files := make([]*database.File, len(filePaths)) for i, path := range filePaths { - file := &File{ + file := &database.File{ Path: types.FilePath(path), MTime: testTime, Size: 2048, @@ -148,21 +171,18 @@ func TestFileChunkRepositoryMultipleFiles(t *testing.T) { LinkTarget: "", } - err := fileRepo.Create(ctx, nil, file) - if err != nil { - t.Fatalf("failed to create file %s: %v", path, err) - } + mustCreateFile(t, fileRepo, file) files[i] = file } // Create all chunks first - chunkRepo := NewChunkRepository(db) + chunkRepo := database.NewChunkRepository(db) for i := range files { for j := range 2 { chunkHash := types.ChunkHash(fmt.Sprintf("file%d_chunk%d", i, j)) - chunk := &Chunk{ + chunk := &database.Chunk{ ChunkHash: chunkHash, Size: 1024, } @@ -177,7 +197,7 @@ func TestFileChunkRepositoryMultipleFiles(t *testing.T) { // Create chunks for multiple files for i, file := range files { for j := range 2 { - fc := &FileChunk{ + fc := &database.FileChunk{ FileID: file.ID, Idx: j, ChunkHash: types.ChunkHash(fmt.Sprintf("file%d_chunk%d", i, j)), diff --git a/internal/database/files.go b/internal/database/files.go index 77ec1ec..4beb5d6 100644 --- a/internal/database/files.go +++ b/internal/database/files.go @@ -12,14 +12,20 @@ import ( "sneak.berlin/go/vaultik/internal/types" ) +// FileRepository provides access to the files table, which stores file +// metadata (path, times, permissions, ownership, symlink targets). type FileRepository struct { db *DB } +// NewFileRepository creates a FileRepository backed by db. func NewFileRepository(db *DB) *FileRepository { return &FileRepository{db: db} } +// Create inserts or updates a file row (upsert on path), using tx when +// non-nil. The file's ID is generated when zero and updated from the +// database's RETURNING clause. func (r *FileRepository) Create(ctx context.Context, tx *sql.Tx, file *File) error { // Generate UUID if not provided if file.ID.IsZero() { @@ -46,10 +52,19 @@ func (r *FileRepository) Create(ctx context.Context, tx *sql.Tx, file *File) err ) if tx != nil { - LogSQL("Execute", query, file.ID.String(), file.Path.String(), file.SourcePath.String(), file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID, file.LinkTarget.String()) - err = tx.QueryRowContext(ctx, query, file.ID.String(), file.Path.String(), file.SourcePath.String(), file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID, file.LinkTarget.String()).Scan(&idStr) + LogSQL("Execute", query, + file.ID.String(), file.Path.String(), file.SourcePath.String(), + file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID, + file.LinkTarget.String()) + err = tx.QueryRowContext(ctx, query, + file.ID.String(), file.Path.String(), file.SourcePath.String(), + file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID, + file.LinkTarget.String()).Scan(&idStr) } else { - err = r.db.QueryRowWithLog(ctx, query, file.ID.String(), file.Path.String(), file.SourcePath.String(), file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID, file.LinkTarget.String()).Scan(&idStr) + err = r.db.QueryRowWithLog(ctx, query, + file.ID.String(), file.Path.String(), file.SourcePath.String(), + file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID, + file.LinkTarget.String()).Scan(&idStr) } if err != nil { @@ -65,6 +80,8 @@ func (r *FileRepository) Create(ctx context.Context, tx *sql.Tx, file *File) err return nil } +// GetByPath returns the file at the given path, or nil if the path is not +// in the index. func (r *FileRepository) GetByPath(ctx context.Context, path string) (*File, error) { query := ` SELECT id, path, source_path, mtime, size, mode, uid, gid, link_target @@ -74,7 +91,7 @@ func (r *FileRepository) GetByPath(ctx context.Context, path string) (*File, err file, err := r.scanFile(r.db.conn.QueryRowContext(ctx, query, path)) if errors.Is(err, sql.ErrNoRows) { - return nil, nil + return nil, nil //nolint:nilnil // nil,nil signals not-found; callers check nil } if err != nil { @@ -94,7 +111,7 @@ func (r *FileRepository) GetByID(ctx context.Context, id types.FileID) (*File, e file, err := r.scanFile(r.db.conn.QueryRowContext(ctx, query, id.String())) if errors.Is(err, sql.ErrNoRows) { - return nil, nil + return nil, nil //nolint:nilnil // nil,nil signals not-found; callers check nil } if err != nil { @@ -104,7 +121,11 @@ func (r *FileRepository) GetByID(ctx context.Context, id types.FileID) (*File, e return file, nil } -func (r *FileRepository) GetByPathTx(ctx context.Context, tx *sql.Tx, path string) (*File, error) { +// GetByPathTx returns the file at the given path within a transaction, or +// nil if the path is not in the index. +func (r *FileRepository) GetByPathTx( + ctx context.Context, tx *sql.Tx, path string, +) (*File, error) { query := ` SELECT id, path, source_path, mtime, size, mode, uid, gid, link_target FROM files @@ -116,7 +137,7 @@ func (r *FileRepository) GetByPathTx(ctx context.Context, tx *sql.Tx, path strin LogSQL("GetByPathTx Scan complete", query, path) if errors.Is(err, sql.ErrNoRows) { - return nil, nil + return nil, nil //nolint:nilnil // nil,nil signals not-found; callers check nil } if err != nil { @@ -126,87 +147,16 @@ func (r *FileRepository) GetByPathTx(ctx context.Context, tx *sql.Tx, path strin return file, nil } -// scanFile is a helper that scans a single file row -func (r *FileRepository) scanFile(row *sql.Row) (*File, error) { - var ( - file File - idStr, pathStr, sourcePathStr string - mtimeUnix int64 - linkTarget sql.NullString - ) - - err := row.Scan( - &idStr, - &pathStr, - &sourcePathStr, - &mtimeUnix, - &file.Size, - &file.Mode, - &file.UID, - &file.GID, - &linkTarget, - ) - if err != nil { - return nil, err - } - - file.ID, err = types.ParseFileID(idStr) - if err != nil { - return nil, fmt.Errorf("parsing file ID: %w", err) - } - - file.Path = types.FilePath(pathStr) - file.SourcePath = types.SourcePath(sourcePathStr) - - file.MTime = time.Unix(mtimeUnix, 0).UTC() - if linkTarget.Valid { - file.LinkTarget = types.FilePath(linkTarget.String) - } - - return &file, nil +// fileRowScanner abstracts *sql.Row and *sql.Rows for scanning a file row. +type fileRowScanner interface { + Scan(dest ...any) error } -// scanFileRows is a helper that scans a file row from rows iterator -func (r *FileRepository) scanFileRows(rows *sql.Rows) (*File, error) { - var ( - file File - idStr, pathStr, sourcePathStr string - mtimeUnix int64 - linkTarget sql.NullString - ) - - err := rows.Scan( - &idStr, - &pathStr, - &sourcePathStr, - &mtimeUnix, - &file.Size, - &file.Mode, - &file.UID, - &file.GID, - &linkTarget, - ) - if err != nil { - return nil, err - } - - file.ID, err = types.ParseFileID(idStr) - if err != nil { - return nil, fmt.Errorf("parsing file ID: %w", err) - } - - file.Path = types.FilePath(pathStr) - file.SourcePath = types.SourcePath(sourcePathStr) - - file.MTime = time.Unix(mtimeUnix, 0).UTC() - if linkTarget.Valid { - file.LinkTarget = types.FilePath(linkTarget.String) - } - - return &file, nil -} - -func (r *FileRepository) ListModifiedSince(ctx context.Context, since time.Time) ([]*File, error) { +// ListModifiedSince returns all files whose recorded mtime is at or after +// since, ordered by path. +func (r *FileRepository) ListModifiedSince( + ctx context.Context, since time.Time, +) ([]*File, error) { query := ` SELECT id, path, source_path, mtime, size, mode, uid, gid, link_target FROM files @@ -234,6 +184,7 @@ func (r *FileRepository) ListModifiedSince(ctx context.Context, since time.Time) return files, rows.Err() } +// Delete removes the file row at the given path, using tx when non-nil. func (r *FileRepository) Delete(ctx context.Context, tx *sql.Tx, path string) error { query := `DELETE FROM files WHERE path = ?` @@ -252,7 +203,9 @@ func (r *FileRepository) Delete(ctx context.Context, tx *sql.Tx, path string) er } // DeleteByID deletes a file by its UUID -func (r *FileRepository) DeleteByID(ctx context.Context, tx *sql.Tx, id types.FileID) error { +func (r *FileRepository) DeleteByID( + ctx context.Context, tx *sql.Tx, id types.FileID, +) error { query := `DELETE FROM files WHERE id = ?` var err error @@ -269,7 +222,11 @@ func (r *FileRepository) DeleteByID(ctx context.Context, tx *sql.Tx, id types.Fi return nil } -func (r *FileRepository) ListByPrefix(ctx context.Context, prefix string) ([]*File, error) { +// ListByPrefix returns all files whose path starts with prefix, ordered by +// path. +func (r *FileRepository) ListByPrefix( + ctx context.Context, prefix string, +) ([]*File, error) { query := ` SELECT id, path, source_path, mtime, size, mode, uid, gid, link_target FROM files @@ -327,12 +284,17 @@ func (r *FileRepository) ListAll(ctx context.Context) ([]*File, error) { // CreateBatch inserts or updates multiple files in a single statement for efficiency. // File IDs must be pre-generated before calling this method. -func (r *FileRepository) CreateBatch(ctx context.Context, tx *sql.Tx, files []*File) error { +func (r *FileRepository) CreateBatch( + ctx context.Context, tx *sql.Tx, files []*File, +) error { if len(files) == 0 { return nil } - // Each File has 9 values, so batch at 100 to be safe with SQLite's variable limit + // Each files row binds this many SQL variables. + const fileCols = 9 + + // Batch at 100 rows to be safe with SQLite's variable limit. const batchSize = 100 for i := 0; i < len(files); i += batchSize { @@ -340,9 +302,11 @@ func (r *FileRepository) CreateBatch(ctx context.Context, tx *sql.Tx, files []*F batch := files[i:end] - query := `INSERT INTO files (id, path, source_path, mtime, size, mode, uid, gid, link_target) VALUES ` + query := `INSERT INTO files + (id, path, source_path, mtime, size, mode, uid, gid, link_target) + VALUES ` - args := make([]any, 0, len(batch)*9) + args := make([]any, 0, len(batch)*fileCols) var querySb325 strings.Builder @@ -353,10 +317,13 @@ func (r *FileRepository) CreateBatch(ctx context.Context, tx *sql.Tx, files []*F querySb325.WriteString("(?, ?, ?, ?, ?, ?, ?, ?, ?)") - args = append(args, f.ID.String(), f.Path.String(), f.SourcePath.String(), f.MTime.Unix(), f.Size, f.Mode, f.UID, f.GID, f.LinkTarget.String()) + args = append(args, + f.ID.String(), f.Path.String(), f.SourcePath.String(), + f.MTime.Unix(), f.Size, f.Mode, f.UID, f.GID, + f.LinkTarget.String()) } - query += querySb325.String() + query += querySb325.String() //nolint:gosec // G202: appends "?" placeholders only query += ` ON CONFLICT(path) DO UPDATE SET source_path = excluded.source_path, @@ -404,3 +371,53 @@ func (r *FileRepository) DeleteOrphaned(ctx context.Context) error { return nil } + +// scanFile is a helper that scans a single file row +func (r *FileRepository) scanFile(row *sql.Row) (*File, error) { + return r.scanFileFrom(row) +} + +// scanFileRows is a helper that scans a file row from rows iterator +func (r *FileRepository) scanFileRows(rows *sql.Rows) (*File, error) { + return r.scanFileFrom(rows) +} + +// scanFileFrom scans one file row from any row scanner. +func (r *FileRepository) scanFileFrom(row fileRowScanner) (*File, error) { + var ( + file File + idStr, pathStr, sourcePathStr string + mtimeUnix int64 + linkTarget sql.NullString + ) + + err := row.Scan( + &idStr, + &pathStr, + &sourcePathStr, + &mtimeUnix, + &file.Size, + &file.Mode, + &file.UID, + &file.GID, + &linkTarget, + ) + if err != nil { + return nil, err + } + + file.ID, err = types.ParseFileID(idStr) + if err != nil { + return nil, fmt.Errorf("parsing file ID: %w", err) + } + + file.Path = types.FilePath(pathStr) + file.SourcePath = types.SourcePath(sourcePathStr) + + file.MTime = time.Unix(mtimeUnix, 0).UTC() + if linkTarget.Valid { + file.LinkTarget = types.FilePath(linkTarget.String) + } + + return &file, nil +} diff --git a/internal/database/files_test.go b/internal/database/files_test.go index 7f4b472..2d4823b 100644 --- a/internal/database/files_test.go +++ b/internal/database/files_test.go @@ -1,44 +1,32 @@ -package database +package database_test import ( "context" "database/sql" "errors" "os" - "path/filepath" "testing" "time" + + "sneak.berlin/go/vaultik/internal/database" ) -func setupTestDB(t *testing.T) (*DB, func()) { - ctx := context.Background() - dbPath := filepath.Join(t.TempDir(), "test.db") - - db, err := New(ctx, dbPath) - if err != nil { - t.Fatalf("failed to create database: %v", err) - } - - cleanup := func() { - err := db.Close() - if err != nil { - t.Errorf("failed to close database: %v", err) - } - } - - return db, cleanup -} +// errTestRollback is the sentinel returned from transaction bodies to +// force a rollback in tests. +var errTestRollback = errors.New("test rollback") func TestFileRepository(t *testing.T) { + t.Parallel() + db, cleanup := setupTestDB(t) defer cleanup() ctx := context.Background() - repo := NewFileRepository(db) + repo := database.NewFileRepository(db) // Test Create - file := &File{ - Path: "/test/file.txt", + file := &database.File{ + Path: testFileTxt, MTime: time.Now().Truncate(time.Second), Size: 1024, Mode: 0644, @@ -95,6 +83,30 @@ func TestFileRepository(t *testing.T) { if retrieved.Size != 2048 { t.Errorf("size not updated: got %d, want %d", retrieved.Size, 2048) } +} + +func TestFileRepositoryListDelete(t *testing.T) { + t.Parallel() + + db, cleanup := setupTestDB(t) + defer cleanup() + + ctx := context.Background() + repo := database.NewFileRepository(db) + + file := &database.File{ + Path: testFileTxt, + MTime: time.Now().Truncate(time.Second), + Size: 1024, + Mode: 0644, + UID: 1000, + GID: 1000, + } + + err := repo.Create(ctx, nil, file) + if err != nil { + t.Fatalf("failed to create file: %v", err) + } // Test ListModifiedSince files, err := repo.ListModifiedSince(ctx, time.Now().Add(-1*time.Hour)) @@ -112,7 +124,7 @@ func TestFileRepository(t *testing.T) { t.Fatalf("failed to delete file: %v", err) } - retrieved, err = repo.GetByPath(ctx, file.Path.String()) + retrieved, err := repo.GetByPath(ctx, file.Path.String()) if err != nil { t.Fatalf("error getting deleted file: %v", err) } @@ -123,14 +135,16 @@ func TestFileRepository(t *testing.T) { } func TestFileRepositorySymlink(t *testing.T) { + t.Parallel() + db, cleanup := setupTestDB(t) defer cleanup() ctx := context.Background() - repo := NewFileRepository(db) + repo := database.NewFileRepository(db) // Test symlink - symlink := &File{ + symlink := &database.File{ Path: "/test/link", MTime: time.Now().Truncate(time.Second), Size: 0, @@ -155,21 +169,24 @@ func TestFileRepositorySymlink(t *testing.T) { } if retrieved.LinkTarget != symlink.LinkTarget { - t.Errorf("link target mismatch: got %s, want %s", retrieved.LinkTarget, symlink.LinkTarget) + t.Errorf("link target mismatch: got %s, want %s", + retrieved.LinkTarget, symlink.LinkTarget) } } func TestFileRepositoryTransaction(t *testing.T) { + t.Parallel() + db, cleanup := setupTestDB(t) defer cleanup() ctx := context.Background() - repos := NewRepositories(db) + repos := database.NewRepositories(db) // Test transaction rollback err := repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { - file := &File{ - Path: "/test/tx_file.txt", + file := &database.File{ + Path: testTxFile, MTime: time.Now().Truncate(time.Second), Size: 1024, Mode: 0644, @@ -183,15 +200,14 @@ func TestFileRepositoryTransaction(t *testing.T) { } // Return error to trigger rollback - return errors.New("test rollback") + return errTestRollback }) - - if err == nil || err.Error() != "test rollback" { + if !errors.Is(err, errTestRollback) { t.Fatalf("expected rollback error, got: %v", err) } // Verify file was not created - retrieved, err := repos.Files.GetByPath(ctx, "/test/tx_file.txt") + retrieved, err := repos.Files.GetByPath(ctx, testTxFile) if err != nil { t.Fatalf("error checking for file: %v", err) } diff --git a/internal/database/helpers_internal_test.go b/internal/database/helpers_internal_test.go new file mode 100644 index 0000000..8b27f67 --- /dev/null +++ b/internal/database/helpers_internal_test.go @@ -0,0 +1,81 @@ +package database + +import ( + "context" + "path/filepath" + "testing" + + "sneak.berlin/go/vaultik/internal/types" +) + +// Common fixture values shared by the internal repository tests. +const ( + internalTestHost = "test-host" + internalTestSnapshotID = "test-snapshot" + internalTestFilePath = "/test.txt" + internalTestFile1 = "/file1.txt" + internalTestFile2 = "/file2.txt" + + // countFilesQuery counts the rows of the files table. + countFilesQuery = "SELECT COUNT(*) FROM files" +) + +// mustCreateFileRow inserts the file row, failing the test on error. +func mustCreateFileRow(t *testing.T, repos *Repositories, file *File) { + t.Helper() + + err := repos.Files.Create(context.Background(), nil, file) + if err != nil { + t.Fatalf("failed to create file %s: %v", file.Path, err) + } +} + +// mustAddFileToSnapshot associates a file with a snapshot, failing the +// test on error. +func mustAddFileToSnapshot( + t *testing.T, repos *Repositories, snapshotID string, fileID types.FileID, +) { + t.Helper() + + err := repos.Snapshots.AddFileByID(context.Background(), nil, snapshotID, fileID) + if err != nil { + t.Fatal(err) + } +} + +// setupTestDB creates an on-disk test database in a per-test temp +// directory and returns it along with a cleanup func that closes it. +func setupTestDB(t *testing.T) (*DB, func()) { + t.Helper() + + ctx := context.Background() + dbPath := filepath.Join(t.TempDir(), "test.db") + + db, err := New(ctx, dbPath) + if err != nil { + t.Fatalf("failed to create database: %v", err) + } + + cleanup := func() { + err := db.Close() + if err != nil { + t.Errorf("failed to close database: %v", err) + } + } + + return db, cleanup +} + +// countRow runs a single-integer COUNT-style query and returns the value. +func countRow(t *testing.T, db *DB, query string, args ...any) int { + t.Helper() + + var count int + + err := db.conn.QueryRowContext(context.Background(), query, args...).Scan(&count) + if err != nil { + t.Fatal(err) + } + + return count +} diff --git a/internal/database/helpers_test.go b/internal/database/helpers_test.go new file mode 100644 index 0000000..8a2b705 --- /dev/null +++ b/internal/database/helpers_test.go @@ -0,0 +1,52 @@ +package database_test + +import ( + "context" + "path/filepath" + "testing" + + "sneak.berlin/go/vaultik/internal/database" +) + +// Common fixture values shared by the repository tests. +const ( + testFilePath1 = "/file1.txt" + testFilePath2 = "/file2.txt" + testFileTxt = "/test/file.txt" + testTxFile = "/test/tx_file.txt" + testHostname = "test-host" + testVersion = "1.0.0" +) + +// mustCreateFile inserts the given file row, failing the test on error. +func mustCreateFile(t *testing.T, repo *database.FileRepository, file *database.File) { + t.Helper() + + err := repo.Create(context.Background(), nil, file) + if err != nil { + t.Fatalf("failed to create file %s: %v", file.Path, err) + } +} + +// setupTestDB creates an on-disk test database in a per-test temp +// directory and returns it along with a cleanup func that closes it. +func setupTestDB(t *testing.T) (*database.DB, func()) { + t.Helper() + + ctx := context.Background() + dbPath := filepath.Join(t.TempDir(), "test.db") + + db, err := database.New(ctx, dbPath) + if err != nil { + t.Fatalf("failed to create database: %v", err) + } + + cleanup := func() { + err := db.Close() + if err != nil { + t.Errorf("failed to close database: %v", err) + } + } + + return db, cleanup +} diff --git a/internal/database/local_meta.go b/internal/database/local_meta.go index 83c2048..27e23bb 100644 --- a/internal/database/local_meta.go +++ b/internal/database/local_meta.go @@ -18,6 +18,7 @@ type LocalMetaRepository struct { db *DB } +// NewLocalMetaRepository creates a LocalMetaRepository backed by db. func NewLocalMetaRepository(db *DB) *LocalMetaRepository { return &LocalMetaRepository{db: db} } diff --git a/internal/database/local_meta_test.go b/internal/database/local_meta_test.go index d9b1d64..52df419 100644 --- a/internal/database/local_meta_test.go +++ b/internal/database/local_meta_test.go @@ -9,6 +9,8 @@ import ( ) func TestLocalMetaEmptyOnFresh(t *testing.T) { + t.Parallel() + db, err := database.NewTestDB() require.NoError(t, err) @@ -22,6 +24,8 @@ func TestLocalMetaEmptyOnFresh(t *testing.T) { } func TestLocalMetaSetGetRoundTrip(t *testing.T) { + t.Parallel() + db, err := database.NewTestDB() require.NoError(t, err) @@ -30,7 +34,8 @@ func TestLocalMetaSetGetRoundTrip(t *testing.T) { repos := database.NewRepositories(db) ctx := context.Background() - require.NoError(t, repos.LocalMeta.Set(ctx, database.LocalMetaKeyStorageURL, "file:///mnt/backups")) + require.NoError(t, repos.LocalMeta.Set( + ctx, database.LocalMetaKeyStorageURL, "file:///mnt/backups")) got, err := repos.LocalMeta.Get(ctx, database.LocalMetaKeyStorageURL) require.NoError(t, err) @@ -38,6 +43,8 @@ func TestLocalMetaSetGetRoundTrip(t *testing.T) { } func TestLocalMetaSetOverwrites(t *testing.T) { + t.Parallel() + db, err := database.NewTestDB() require.NoError(t, err) @@ -46,8 +53,10 @@ func TestLocalMetaSetOverwrites(t *testing.T) { repos := database.NewRepositories(db) ctx := context.Background() - require.NoError(t, repos.LocalMeta.Set(ctx, database.LocalMetaKeyStorageURL, "s3://old")) - require.NoError(t, repos.LocalMeta.Set(ctx, database.LocalMetaKeyStorageURL, "s3://new")) + require.NoError(t, repos.LocalMeta.Set( + ctx, database.LocalMetaKeyStorageURL, "s3://old")) + require.NoError(t, repos.LocalMeta.Set( + ctx, database.LocalMetaKeyStorageURL, "s3://new")) got, err := repos.LocalMeta.Get(ctx, database.LocalMetaKeyStorageURL) require.NoError(t, err) diff --git a/internal/database/models.go b/internal/database/models.go index f95831b..69f8730 100644 --- a/internal/database/models.go +++ b/internal/database/models.go @@ -1,5 +1,3 @@ -// Package database provides data models and repository interfaces for the Vaultik backup system. -// It includes types for files, chunks, blobs, snapshots, and their relationships. package database import ( @@ -13,9 +11,12 @@ import ( // and symlink targets. This information is used to restore files with their // original attributes. type File struct { - ID types.FileID // UUID primary key - Path types.FilePath // Absolute path of the file - SourcePath types.SourcePath // The source directory this file came from (for restore path stripping) + ID types.FileID // UUID primary key + Path types.FilePath // Absolute path of the file + + // SourcePath is the source directory this file came from (used for + // restore path stripping). + SourcePath types.SourcePath MTime time.Time Size int64 Mode uint32 @@ -55,13 +56,16 @@ type Chunk struct { // The blob creation process is: chunks are accumulated -> compressed with zstd // -> encrypted with age -> hashed -> uploaded to S3 with the hash as filename. type Blob struct { - ID types.BlobID // UUID assigned when blob creation starts - Hash types.BlobHash // SHA256 of final compressed+encrypted content (empty until finalized) - CreatedTS time.Time // When blob creation started - FinishedTS *time.Time // When blob was finalized (nil if still packing) - UncompressedSize int64 // Total size of raw chunks before compression - CompressedSize int64 // Size after compression and encryption - UploadedTS *time.Time // When blob was uploaded to S3 (nil if not uploaded) + ID types.BlobID // UUID assigned when blob creation starts + + // Hash is the SHA256 of the final compressed+encrypted content + // (empty until finalized). + Hash types.BlobHash + CreatedTS time.Time // When blob creation started + FinishedTS *time.Time // When blob was finalized (nil if still packing) + UncompressedSize int64 // Total size of raw chunks before compression + CompressedSize int64 // Size after compression and encryption + UploadedTS *time.Time // When blob was uploaded to S3 (nil if not uploaded) } // BlobChunk represents the mapping between blobs and the chunks they contain. @@ -75,9 +79,10 @@ type BlobChunk struct { Length int64 } -// ChunkFile represents the reverse mapping showing which files contain a specific chunk. -// This is used during deduplication to identify all files that share a chunk, -// which is important for garbage collection and integrity verification. +// ChunkFile represents the reverse mapping showing which files contain a +// specific chunk. This is used during deduplication to identify all files +// that share a chunk, which is important for garbage collection and +// integrity verification. type ChunkFile struct { ChunkHash types.ChunkHash FileID types.FileID @@ -87,17 +92,20 @@ type ChunkFile struct { // Snapshot represents a snapshot record in the database type Snapshot struct { - ID types.SnapshotID - Hostname types.Hostname - VaultikVersion types.Version - VaultikGitRevision types.GitRevision - StartedAt time.Time - CompletedAt *time.Time // nil if still in progress - FileCount int64 - ChunkCount int64 - BlobCount int64 - TotalSize int64 // Total size of all referenced files - BlobSize int64 // Total size of all referenced blobs (compressed and encrypted) + ID types.SnapshotID + Hostname types.Hostname + VaultikVersion types.Version + VaultikGitRevision types.GitRevision + StartedAt time.Time + CompletedAt *time.Time // nil if still in progress + FileCount int64 + ChunkCount int64 + BlobCount int64 + TotalSize int64 // Total size of all referenced files + + // BlobSize is the total size of all referenced blobs (compressed and + // encrypted). + BlobSize int64 BlobUncompressedSize int64 // Total uncompressed size of all referenced blobs CompressionRatio float64 // Compression ratio (BlobSize / BlobUncompressedSize) CompressionLevel int // Compression level used for this snapshot diff --git a/internal/database/module.go b/internal/database/module.go index e4af208..ec8fa47 100644 --- a/internal/database/module.go +++ b/internal/database/module.go @@ -11,7 +11,13 @@ import ( "sneak.berlin/go/vaultik/internal/log" ) +// indexDirPerm restricts the local index directory to the owning user; +// the index describes the backed-up file tree and must stay private. +const indexDirPerm = 0o700 + // Module provides database dependencies +// +//nolint:gochecknoglobals // fx module definitions are package globals by convention var Module = fx.Module("database", fx.Provide( provideDatabase, @@ -22,7 +28,9 @@ var Module = fx.Module("database", func provideDatabase(lc fx.Lifecycle, cfg *config.Config) (*DB, error) { // Ensure the index directory exists indexDir := filepath.Dir(cfg.IndexPath) - if err := os.MkdirAll(indexDir, 0700); err != nil { + + err := os.MkdirAll(indexDir, indexDirPerm) + if err != nil { return nil, fmt.Errorf("creating index directory: %w", err) } @@ -32,7 +40,7 @@ func provideDatabase(lc fx.Lifecycle, cfg *config.Config) (*DB, error) { } lc.Append(fx.Hook{ - OnStop: func(ctx context.Context) error { + OnStop: func(_ context.Context) error { log.Debug("Database module OnStop hook called") err := db.Close() diff --git a/internal/database/repositories.go b/internal/database/repositories.go index 07a1bb0..64a3f66 100644 --- a/internal/database/repositories.go +++ b/internal/database/repositories.go @@ -62,14 +62,14 @@ func (r *Repositories) WithTx(ctx context.Context, fn TxFunc) error { if p := recover(); p != nil { rollbackErr := tx.Rollback() if rollbackErr != nil { - Fatal("failed to rollback transaction: %v", rollbackErr) + Fatalf("failed to rollback transaction: %v", rollbackErr) } panic(p) } else if err != nil { rollbackErr := tx.Rollback() if rollbackErr != nil { - Fatal("failed to rollback transaction: %v", rollbackErr) + Fatalf("failed to rollback transaction: %v", rollbackErr) } } }() @@ -105,14 +105,14 @@ func (r *Repositories) WithReadTx(ctx context.Context, fn TxFunc) error { if p := recover(); p != nil { rollbackErr := tx.Rollback() if rollbackErr != nil { - Fatal("failed to rollback transaction: %v", rollbackErr) + Fatalf("failed to rollback transaction: %v", rollbackErr) } panic(p) } else if err != nil { rollbackErr := tx.Rollback() if rollbackErr != nil { - Fatal("failed to rollback transaction: %v", rollbackErr) + Fatalf("failed to rollback transaction: %v", rollbackErr) } } }() diff --git a/internal/database/repositories_test.go b/internal/database/repositories_test.go index 485ba61..f4e2199 100644 --- a/internal/database/repositories_test.go +++ b/internal/database/repositories_test.go @@ -1,4 +1,4 @@ -package database +package database_test import ( "context" @@ -7,21 +7,21 @@ import ( "testing" "time" + "sneak.berlin/go/vaultik/internal/database" "sneak.berlin/go/vaultik/internal/types" ) -func TestRepositoriesTransaction(t *testing.T) { - db, cleanup := setupTestDB(t) - defer cleanup() +// errIntentionalRollback forces a transaction rollback in tests. +var errIntentionalRollback = errors.New("intentional rollback") - ctx := context.Background() - repos := NewRepositories(db) - - // Test successful transaction with multiple operations - err := repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { - // Create a file - file := &File{ - Path: "/test/tx_file.txt", +// createTxTestData returns a transaction body that creates a file with +// two chunks packed into one blob. +func createTxTestData( + repos *database.Repositories, +) func(context.Context, *sql.Tx) error { + return func(ctx context.Context, tx *sql.Tx) error { + file := &database.File{ + Path: testTxFile, MTime: time.Now().Truncate(time.Second), Size: 1024, Mode: 0644, @@ -34,95 +34,116 @@ func TestRepositoriesTransaction(t *testing.T) { return err } - // Create chunks - chunk1 := &Chunk{ - ChunkHash: types.ChunkHash("tx_chunk1"), - Size: 512, - } - - err = repos.Chunks.Create(ctx, tx, chunk1) + err = createTxFileChunks(ctx, tx, repos, file.ID) if err != nil { return err } - chunk2 := &Chunk{ - ChunkHash: types.ChunkHash("tx_chunk2"), - Size: 512, - } + return createTxBlob(ctx, tx, repos) + } +} - err = repos.Chunks.Create(ctx, tx, chunk2) - if err != nil { - return err - } +// createTxFileChunks creates the two test chunks and maps them to the file. +func createTxFileChunks( + ctx context.Context, tx *sql.Tx, + repos *database.Repositories, fileID types.FileID, +) error { + // Create chunks + chunk1 := &database.Chunk{ + ChunkHash: types.ChunkHash("tx_chunk1"), + Size: 512, + } - // Map chunks to file - fc1 := &FileChunk{ - FileID: file.ID, - Idx: 0, - ChunkHash: chunk1.ChunkHash, - } + err := repos.Chunks.Create(ctx, tx, chunk1) + if err != nil { + return err + } - err = repos.FileChunks.Create(ctx, tx, fc1) - if err != nil { - return err - } + chunk2 := &database.Chunk{ + ChunkHash: types.ChunkHash("tx_chunk2"), + Size: 512, + } - fc2 := &FileChunk{ - FileID: file.ID, - Idx: 1, - ChunkHash: chunk2.ChunkHash, - } + err = repos.Chunks.Create(ctx, tx, chunk2) + if err != nil { + return err + } - err = repos.FileChunks.Create(ctx, tx, fc2) - if err != nil { - return err - } + // Map chunks to file + fc1 := &database.FileChunk{ + FileID: fileID, + Idx: 0, + ChunkHash: chunk1.ChunkHash, + } - // Create blob - blob := &Blob{ - ID: types.NewBlobID(), - Hash: types.BlobHash("tx_blob1"), - CreatedTS: time.Now().Truncate(time.Second), - } + err = repos.FileChunks.Create(ctx, tx, fc1) + if err != nil { + return err + } - err = repos.Blobs.Create(ctx, tx, blob) - if err != nil { - return err - } + fc2 := &database.FileChunk{ + FileID: fileID, + Idx: 1, + ChunkHash: chunk2.ChunkHash, + } - // Map chunks to blob - bc1 := &BlobChunk{ - BlobID: blob.ID, - ChunkHash: chunk1.ChunkHash, - Offset: 0, - Length: 512, - } + return repos.FileChunks.Create(ctx, tx, fc2) +} - err = repos.BlobChunks.Create(ctx, tx, bc1) - if err != nil { - return err - } +// createTxBlob creates the test blob and maps both chunks into it. +func createTxBlob( + ctx context.Context, tx *sql.Tx, repos *database.Repositories, +) error { + blob := &database.Blob{ + ID: types.NewBlobID(), + Hash: types.BlobHash("tx_blob1"), + CreatedTS: time.Now().Truncate(time.Second), + } - bc2 := &BlobChunk{ - BlobID: blob.ID, - ChunkHash: chunk2.ChunkHash, - Offset: 512, - Length: 512, - } + err := repos.Blobs.Create(ctx, tx, blob) + if err != nil { + return err + } - err = repos.BlobChunks.Create(ctx, tx, bc2) - if err != nil { - return err - } + // Map chunks to blob + bc1 := &database.BlobChunk{ + BlobID: blob.ID, + ChunkHash: types.ChunkHash("tx_chunk1"), + Offset: 0, + Length: 512, + } - return nil - }) + err = repos.BlobChunks.Create(ctx, tx, bc1) + if err != nil { + return err + } + + bc2 := &database.BlobChunk{ + BlobID: blob.ID, + ChunkHash: types.ChunkHash("tx_chunk2"), + Offset: 512, + Length: 512, + } + + return repos.BlobChunks.Create(ctx, tx, bc2) +} + +func TestRepositoriesTransaction(t *testing.T) { + t.Parallel() + + db, cleanup := setupTestDB(t) + defer cleanup() + + ctx := context.Background() + repos := database.NewRepositories(db) + + err := repos.WithTx(ctx, createTxTestData(repos)) if err != nil { t.Fatalf("transaction failed: %v", err) } // Verify all data was committed - file, err := repos.Files.GetByPath(ctx, "/test/tx_file.txt") + file, err := repos.Files.GetByPath(ctx, testTxFile) if err != nil { t.Fatalf("failed to get file: %v", err) } @@ -131,7 +152,7 @@ func TestRepositoriesTransaction(t *testing.T) { t.Error("expected file after transaction") } - chunks, err := repos.FileChunks.GetByFile(ctx, "/test/tx_file.txt") + chunks, err := repos.FileChunks.GetByFile(ctx, testTxFile) if err != nil { t.Fatalf("failed to get file chunks: %v", err) } @@ -151,16 +172,18 @@ func TestRepositoriesTransaction(t *testing.T) { } func TestRepositoriesTransactionRollback(t *testing.T) { + t.Parallel() + db, cleanup := setupTestDB(t) defer cleanup() ctx := context.Background() - repos := NewRepositories(db) + repos := database.NewRepositories(db) // Test transaction rollback err := repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { // Create a file - file := &File{ + file := &database.File{ Path: "/test/rollback_file.txt", MTime: time.Now().Truncate(time.Second), Size: 1024, @@ -175,7 +198,7 @@ func TestRepositoriesTransactionRollback(t *testing.T) { } // Create a chunk - chunk := &Chunk{ + chunk := &database.Chunk{ ChunkHash: types.ChunkHash("rollback_chunk"), Size: 1024, } @@ -186,10 +209,9 @@ func TestRepositoriesTransactionRollback(t *testing.T) { } // Return error to trigger rollback - return errors.New("intentional rollback") + return errIntentionalRollback }) - - if err == nil || err.Error() != "intentional rollback" { + if !errors.Is(err, errIntentionalRollback) { t.Fatalf("expected rollback error, got: %v", err) } @@ -214,14 +236,16 @@ func TestRepositoriesTransactionRollback(t *testing.T) { } func TestRepositoriesReadTransaction(t *testing.T) { + t.Parallel() + db, cleanup := setupTestDB(t) defer cleanup() ctx := context.Background() - repos := NewRepositories(db) + repos := database.NewRepositories(db) // First, create some data - file := &File{ + file := &database.File{ Path: "/test/read_file.txt", MTime: time.Now().Truncate(time.Second), Size: 1024, @@ -236,7 +260,7 @@ func TestRepositoriesReadTransaction(t *testing.T) { } // Test read-only transaction - var retrievedFile *File + var retrievedFile *database.File err = repos.WithReadTx(ctx, func(ctx context.Context, tx *sql.Tx) error { var err error @@ -247,7 +271,7 @@ func TestRepositoriesReadTransaction(t *testing.T) { } // Try to write in read-only transaction (should fail) - _ = repos.Files.Create(ctx, tx, &File{ + _ = repos.Files.Create(ctx, tx, &database.File{ Path: "/test/should_fail.txt", MTime: time.Now(), Size: 0, diff --git a/internal/database/repository_comprehensive_test.go b/internal/database/repository_comprehensive_test.go index f3fce80..4799458 100644 --- a/internal/database/repository_comprehensive_test.go +++ b/internal/database/repository_comprehensive_test.go @@ -1,3 +1,4 @@ +//nolint:testpackage // inspects the unexported database connection package database import ( @@ -11,8 +12,13 @@ import ( "sneak.berlin/go/vaultik/internal/types" ) +// errTxIntentionalRollback forces a transaction rollback in tests. +var errTxIntentionalRollback = errors.New("intentional rollback") + // TestFileRepositoryUUIDGeneration tests that files get unique UUIDs func TestFileRepositoryUUIDGeneration(t *testing.T) { + t.Parallel() + db, cleanup := setupTestDB(t) defer cleanup() @@ -22,7 +28,7 @@ func TestFileRepositoryUUIDGeneration(t *testing.T) { // Create multiple files files := []*File{ { - Path: "/file1.txt", + Path: internalTestFile1, MTime: time.Now().Truncate(time.Second), Size: 1024, Mode: 0644, @@ -30,7 +36,7 @@ func TestFileRepositoryUUIDGeneration(t *testing.T) { GID: 1000, }, { - Path: "/file2.txt", + Path: internalTestFile2, MTime: time.Now().Truncate(time.Second), Size: 2048, Mode: 0644, @@ -63,6 +69,8 @@ func TestFileRepositoryUUIDGeneration(t *testing.T) { // TestFileRepositoryGetByID tests retrieving files by UUID func TestFileRepositoryGetByID(t *testing.T) { + t.Parallel() + db, cleanup := setupTestDB(t) defer cleanup() @@ -71,7 +79,7 @@ func TestFileRepositoryGetByID(t *testing.T) { // Create a file file := &File{ - Path: "/test.txt", + Path: internalTestFilePath, MTime: time.Now().Truncate(time.Second), Size: 1024, Mode: 0644, @@ -98,8 +106,9 @@ func TestFileRepositoryGetByID(t *testing.T) { t.Errorf("Path mismatch: expected %s, got %s", file.Path, retrieved.Path) } - // Test non-existent ID - nonExistentID := types.NewFileID() // Generate a new UUID that won't exist in the database + // Test non-existent ID: generate a new UUID that won't exist in the + // database. + nonExistentID := types.NewFileID() nonExistent, err := repo.GetByID(ctx, nonExistentID) if err != nil { @@ -113,6 +122,8 @@ func TestFileRepositoryGetByID(t *testing.T) { // TestOrphanedFileCleanup tests the cleanup of orphaned files func TestOrphanedFileCleanup(t *testing.T) { + t.Parallel() + db, cleanup := setupTestDB(t) defer cleanup() @@ -149,8 +160,8 @@ func TestOrphanedFileCleanup(t *testing.T) { // Create a snapshot and reference only file2 snapshot := &Snapshot{ - ID: "test-snapshot", - Hostname: "test-host", + ID: internalTestSnapshotID, + Hostname: internalTestHost, StartedAt: time.Now(), } @@ -160,10 +171,7 @@ func TestOrphanedFileCleanup(t *testing.T) { } // Add file2 to snapshot - err = repos.Snapshots.AddFileByID(ctx, nil, snapshot.ID.String(), file2.ID) - if err != nil { - t.Fatalf("failed to add file to snapshot: %v", err) - } + mustAddFileToSnapshot(t, repos, snapshot.ID.String(), file2.ID) // Run orphaned cleanup err = repos.Files.DeleteOrphaned(ctx) @@ -194,6 +202,8 @@ func TestOrphanedFileCleanup(t *testing.T) { // TestOrphanedChunkCleanup tests the cleanup of orphaned chunks func TestOrphanedChunkCleanup(t *testing.T) { + t.Parallel() + db, cleanup := setupTestDB(t) defer cleanup() @@ -222,7 +232,7 @@ func TestOrphanedChunkCleanup(t *testing.T) { // Create a file and reference only chunk2 file := &File{ - Path: "/test.txt", + Path: internalTestFilePath, MTime: time.Now().Truncate(time.Second), Size: 1024, Mode: 0644, @@ -276,6 +286,8 @@ func TestOrphanedChunkCleanup(t *testing.T) { // TestOrphanedBlobCleanup tests the cleanup of orphaned blobs func TestOrphanedBlobCleanup(t *testing.T) { + t.Parallel() + db, cleanup := setupTestDB(t) defer cleanup() @@ -306,8 +318,8 @@ func TestOrphanedBlobCleanup(t *testing.T) { // Create a snapshot and reference only blob2 snapshot := &Snapshot{ - ID: "test-snapshot", - Hostname: "test-host", + ID: internalTestSnapshotID, + Hostname: internalTestHost, StartedAt: time.Now(), } @@ -351,6 +363,8 @@ func TestOrphanedBlobCleanup(t *testing.T) { // TestFileChunkRepositoryWithUUIDs tests file-chunk relationships with UUIDs func TestFileChunkRepositoryWithUUIDs(t *testing.T) { + t.Parallel() + db, cleanup := setupTestDB(t) defer cleanup() @@ -359,7 +373,7 @@ func TestFileChunkRepositoryWithUUIDs(t *testing.T) { // Create a file file := &File{ - Path: "/test.txt", + Path: internalTestFilePath, MTime: time.Now().Truncate(time.Second), Size: 3072, Mode: 0644, @@ -367,10 +381,7 @@ func TestFileChunkRepositoryWithUUIDs(t *testing.T) { GID: 1000, } - err := repos.Files.Create(ctx, nil, file) - if err != nil { - t.Fatalf("failed to create file: %v", err) - } + mustCreateFileRow(t, repos, file) // Create chunks chunks := []types.ChunkHash{"chunk1", "chunk2", "chunk3"} @@ -380,7 +391,7 @@ func TestFileChunkRepositoryWithUUIDs(t *testing.T) { Size: 1024, } - err = repos.Chunks.Create(ctx, nil, chunk) + err := repos.Chunks.Create(ctx, nil, chunk) if err != nil { t.Fatalf("failed to create chunk: %v", err) } @@ -426,6 +437,8 @@ func TestFileChunkRepositoryWithUUIDs(t *testing.T) { // TestChunkFileRepositoryWithUUIDs tests chunk-file relationships with UUIDs func TestChunkFileRepositoryWithUUIDs(t *testing.T) { + t.Parallel() + db, cleanup := setupTestDB(t) defer cleanup() @@ -434,7 +447,7 @@ func TestChunkFileRepositoryWithUUIDs(t *testing.T) { // Create files file1 := &File{ - Path: "/file1.txt", + Path: internalTestFile1, MTime: time.Now().Truncate(time.Second), Size: 1024, Mode: 0644, @@ -442,7 +455,7 @@ func TestChunkFileRepositoryWithUUIDs(t *testing.T) { GID: 1000, } file2 := &File{ - Path: "/file2.txt", + Path: internalTestFile2, MTime: time.Now().Truncate(time.Second), Size: 1024, Mode: 0644, @@ -450,15 +463,8 @@ func TestChunkFileRepositoryWithUUIDs(t *testing.T) { GID: 1000, } - err := repos.Files.Create(ctx, nil, file1) - if err != nil { - t.Fatalf("failed to create file1: %v", err) - } - - err = repos.Files.Create(ctx, nil, file2) - if err != nil { - t.Fatalf("failed to create file2: %v", err) - } + mustCreateFileRow(t, repos, file1) + mustCreateFileRow(t, repos, file2) // Create a chunk that appears in both files (deduplication) chunk := &Chunk{ @@ -466,7 +472,7 @@ func TestChunkFileRepositoryWithUUIDs(t *testing.T) { Size: 1024, } - err = repos.Chunks.Create(ctx, nil, chunk) + err := repos.Chunks.Create(ctx, nil, chunk) if err != nil { t.Fatalf("failed to create chunk: %v", err) } @@ -518,6 +524,8 @@ func TestChunkFileRepositoryWithUUIDs(t *testing.T) { // TestSnapshotRepositoryExtendedFields tests snapshot with version and git revision func TestSnapshotRepositoryExtendedFields(t *testing.T) { + t.Parallel() + db, cleanup := setupTestDB(t) defer cleanup() @@ -527,7 +535,7 @@ func TestSnapshotRepositoryExtendedFields(t *testing.T) { // Create snapshot with extended fields snapshot := &Snapshot{ ID: "test-20250722-120000Z", - Hostname: "test-host", + Hostname: internalTestHost, VaultikVersion: "0.0.1", VaultikGitRevision: "abc123def456", StartedAt: time.Now(), @@ -555,35 +563,39 @@ func TestSnapshotRepositoryExtendedFields(t *testing.T) { } if retrieved.VaultikVersion != snapshot.VaultikVersion { - t.Errorf("version mismatch: expected %s, got %s", snapshot.VaultikVersion, retrieved.VaultikVersion) + t.Errorf("version mismatch: expected %s, got %s", + snapshot.VaultikVersion, retrieved.VaultikVersion) } if retrieved.VaultikGitRevision != snapshot.VaultikGitRevision { - t.Errorf("git revision mismatch: expected %s, got %s", snapshot.VaultikGitRevision, retrieved.VaultikGitRevision) + t.Errorf("git revision mismatch: expected %s, got %s", + snapshot.VaultikGitRevision, retrieved.VaultikGitRevision) } if retrieved.CompressionLevel != snapshot.CompressionLevel { - t.Errorf("compression level mismatch: expected %d, got %d", snapshot.CompressionLevel, retrieved.CompressionLevel) + t.Errorf("compression level mismatch: expected %d, got %d", + snapshot.CompressionLevel, retrieved.CompressionLevel) } if retrieved.BlobUncompressedSize != snapshot.BlobUncompressedSize { - t.Errorf("uncompressed size mismatch: expected %d, got %d", snapshot.BlobUncompressedSize, retrieved.BlobUncompressedSize) + t.Errorf("uncompressed size mismatch: expected %d, got %d", + snapshot.BlobUncompressedSize, retrieved.BlobUncompressedSize) } if retrieved.UploadDurationMs != snapshot.UploadDurationMs { - t.Errorf("upload duration mismatch: expected %d, got %d", snapshot.UploadDurationMs, retrieved.UploadDurationMs) + t.Errorf("upload duration mismatch: expected %d, got %d", + snapshot.UploadDurationMs, retrieved.UploadDurationMs) } } // TestComplexOrphanedDataScenario tests a complex scenario with multiple relationships -func TestComplexOrphanedDataScenario(t *testing.T) { - db, cleanup := setupTestDB(t) - defer cleanup() +// createOrphanScenarioFixtures creates two snapshots and three files for +// the orphaned-data cleanup scenario. +func createOrphanScenarioFixtures( + ctx context.Context, t *testing.T, repos *Repositories, +) (*Snapshot, *Snapshot, []*File) { + t.Helper() - ctx := context.Background() - repos := NewRepositories(db) - - // Create snapshots snapshot1 := &Snapshot{ ID: "snapshot1", Hostname: "host1", @@ -623,34 +635,33 @@ func TestComplexOrphanedDataScenario(t *testing.T) { } } + return snapshot1, snapshot2, files +} + +func TestComplexOrphanedDataScenario(t *testing.T) { + t.Parallel() + + db, cleanup := setupTestDB(t) + defer cleanup() + + ctx := context.Background() + repos := NewRepositories(db) + + snapshot1, snapshot2, files := createOrphanScenarioFixtures(ctx, t, repos) + // Add files to snapshots // Snapshot1: file0, file1 // Snapshot2: file1, file2 // file0: only in snapshot1 // file1: in both snapshots // file2: only in snapshot2 - err = repos.Snapshots.AddFileByID(ctx, nil, snapshot1.ID.String(), files[0].ID) - if err != nil { - t.Fatal(err) - } - - err = repos.Snapshots.AddFileByID(ctx, nil, snapshot1.ID.String(), files[1].ID) - if err != nil { - t.Fatal(err) - } - - err = repos.Snapshots.AddFileByID(ctx, nil, snapshot2.ID.String(), files[1].ID) - if err != nil { - t.Fatal(err) - } - - err = repos.Snapshots.AddFileByID(ctx, nil, snapshot2.ID.String(), files[2].ID) - if err != nil { - t.Fatal(err) - } + mustAddFileToSnapshot(t, repos, snapshot1.ID.String(), files[0].ID) + mustAddFileToSnapshot(t, repos, snapshot1.ID.String(), files[1].ID) + mustAddFileToSnapshot(t, repos, snapshot2.ID.String(), files[1].ID) + mustAddFileToSnapshot(t, repos, snapshot2.ID.String(), files[2].ID) // Delete snapshot1 - err = repos.Snapshots.DeleteSnapshotFiles(ctx, snapshot1.ID.String()) + err := repos.Snapshots.DeleteSnapshotFiles(ctx, snapshot1.ID.String()) if err != nil { t.Fatal(err) } @@ -700,6 +711,8 @@ func TestComplexOrphanedDataScenario(t *testing.T) { // TestCascadeDelete tests that cascade deletes work properly func TestCascadeDelete(t *testing.T) { + t.Parallel() + db, cleanup := setupTestDB(t) defer cleanup() @@ -774,6 +787,8 @@ func TestCascadeDelete(t *testing.T) { // TestTransactionIsolation tests that transactions properly isolate changes func TestTransactionIsolation(t *testing.T) { + t.Parallel() + db, cleanup := setupTestDB(t) defer cleanup() @@ -802,7 +817,7 @@ func TestTransactionIsolation(t *testing.T) { // For now, we'll just test that rollback works // Return an error to trigger rollback - return errors.New("intentional rollback") + return errTxIntentionalRollback }) if err == nil { t.Fatal("expected error from transaction") @@ -819,32 +834,15 @@ func TestTransactionIsolation(t *testing.T) { } } -// TestConcurrentOrphanedCleanup tests that concurrent cleanup operations don't interfere -func TestConcurrentOrphanedCleanup(t *testing.T) { - db, cleanup := setupTestDB(t) - defer cleanup() +// TestConcurrentOrphanedCleanup tests that concurrent cleanup operations +// don't interfere. +// createConcurrentCleanupFiles creates 20 files and associates the +// even-numbered ones with the snapshot, leaving the rest orphaned. +func createConcurrentCleanupFiles( + ctx context.Context, t *testing.T, repos *Repositories, snapshotID string, +) { + t.Helper() - ctx := context.Background() - repos := NewRepositories(db) - - // Set a 5-second busy timeout to handle concurrent operations - if _, err := db.conn.Exec("PRAGMA busy_timeout = 5000"); err != nil { - t.Fatalf("failed to set busy timeout: %v", err) - } - - // Create a snapshot - snapshot := &Snapshot{ - ID: "concurrent-test", - Hostname: "test-host", - StartedAt: time.Now(), - } - - err := repos.Snapshots.Create(ctx, nil, snapshot) - if err != nil { - t.Fatal(err) - } - - // Create many files, some orphaned for i := range 20 { file := &File{ Path: types.FilePath(fmt.Sprintf("/concurrent-%d.txt", i)), @@ -855,19 +853,49 @@ func TestConcurrentOrphanedCleanup(t *testing.T) { GID: 1000, } - err = repos.Files.Create(ctx, nil, file) + err := repos.Files.Create(ctx, nil, file) if err != nil { t.Fatal(err) } // Add even-numbered files to snapshot if i%2 == 0 { - err = repos.Snapshots.AddFileByID(ctx, nil, snapshot.ID.String(), file.ID) + err = repos.Snapshots.AddFileByID(ctx, nil, snapshotID, file.ID) if err != nil { t.Fatal(err) } } } +} + +func TestConcurrentOrphanedCleanup(t *testing.T) { + t.Parallel() + + db, cleanup := setupTestDB(t) + defer cleanup() + + ctx := context.Background() + repos := NewRepositories(db) + + // Set a 5-second busy timeout to handle concurrent operations + _, err := db.conn.ExecContext(ctx, "PRAGMA busy_timeout = 5000") + if err != nil { + t.Fatalf("failed to set busy timeout: %v", err) + } + + // Create a snapshot + snapshot := &Snapshot{ + ID: "concurrent-test", + Hostname: internalTestHost, + StartedAt: time.Now(), + } + + err = repos.Snapshots.Create(ctx, nil, snapshot) + if err != nil { + t.Fatal(err) + } + + createConcurrentCleanupFiles(ctx, t, repos, snapshot.ID.String()) // Run multiple cleanup operations concurrently // Note: SQLite has limited support for concurrent writes, so we expect some to fail diff --git a/internal/database/repository_debug_test.go b/internal/database/repository_debug_test.go index 9deedaa..db541ed 100644 --- a/internal/database/repository_debug_test.go +++ b/internal/database/repository_debug_test.go @@ -1,3 +1,4 @@ +//nolint:testpackage // inspects the unexported database connection package database import ( @@ -6,15 +7,50 @@ import ( "time" ) -// TestOrphanedFileCleanupDebug tests orphaned file cleanup with debug output -func TestOrphanedFileCleanupDebug(t *testing.T) { - db, cleanup := setupTestDB(t) - defer cleanup() +// logSnapshotFileIDs logs every file_id present in snapshot_files. +func logSnapshotFileIDs(t *testing.T, db *DB) { + t.Helper() ctx := context.Background() - repos := NewRepositories(db) - // Create files + rows, err := db.conn.QueryContext(ctx, "SELECT file_id FROM snapshot_files") + if err != nil { + t.Fatal(err) + } + defer func() { + err := rows.Close() + if err != nil { + t.Logf("failed to close rows: %v", err) + } + }() + + t.Log("Files in snapshot_files:") + + for rows.Next() { + var fileID string + + err := rows.Scan(&fileID) + if err != nil { + t.Fatal(err) + } + + t.Logf(" - %s", fileID) + } + + err = rows.Err() + if err != nil { + t.Fatal(err) + } +} + +// TestOrphanedFileCleanupDebug tests orphaned file cleanup with debug output +// createOrphanDebugFixtures creates one orphaned file, one referenced +// file, and the snapshot that will reference the latter. +func createOrphanDebugFixtures( + ctx context.Context, t *testing.T, repos *Repositories, +) (*File, *File, *Snapshot) { + t.Helper() + file1 := &File{ Path: "/orphaned.txt", MTime: time.Now().Truncate(time.Second), @@ -48,8 +84,8 @@ func TestOrphanedFileCleanupDebug(t *testing.T) { // Create a snapshot and reference only file2 snapshot := &Snapshot{ - ID: "test-snapshot", - Hostname: "test-host", + ID: internalTestSnapshotID, + Hostname: internalTestHost, StartedAt: time.Now(), } @@ -60,18 +96,26 @@ func TestOrphanedFileCleanupDebug(t *testing.T) { t.Logf("Created snapshot: %s", snapshot.ID) + return file1, file2, snapshot +} + +func TestOrphanedFileCleanupDebug(t *testing.T) { + t.Parallel() + + db, cleanup := setupTestDB(t) + defer cleanup() + + ctx := context.Background() + repos := NewRepositories(db) + + file1, file2, snapshot := createOrphanDebugFixtures(ctx, t, repos) + // Check snapshot_files before adding - var count int - - err = db.conn.QueryRow("SELECT COUNT(*) FROM snapshot_files").Scan(&count) - if err != nil { - t.Fatal(err) - } - + count := countRow(t, db, "SELECT COUNT(*) FROM snapshot_files") t.Logf("snapshot_files count before add: %d", count) // Add file2 to snapshot - err = repos.Snapshots.AddFileByID(ctx, nil, snapshot.ID.String(), file2.ID) + err := repos.Snapshots.AddFileByID(ctx, nil, snapshot.ID.String(), file2.ID) if err != nil { t.Fatalf("failed to add file to snapshot: %v", err) } @@ -79,44 +123,14 @@ func TestOrphanedFileCleanupDebug(t *testing.T) { t.Logf("Added file2 to snapshot") // Check snapshot_files after adding - err = db.conn.QueryRow("SELECT COUNT(*) FROM snapshot_files").Scan(&count) - if err != nil { - t.Fatal(err) - } - + count = countRow(t, db, "SELECT COUNT(*) FROM snapshot_files") t.Logf("snapshot_files count after add: %d", count) // Check which files are referenced - rows, err := db.conn.Query("SELECT file_id FROM snapshot_files") - if err != nil { - t.Fatal(err) - } - defer func() { - err := rows.Close() - if err != nil { - t.Logf("failed to close rows: %v", err) - } - }() - - t.Log("Files in snapshot_files:") - - for rows.Next() { - var fileID string - - err := rows.Scan(&fileID) - if err != nil { - t.Fatal(err) - } - - t.Logf(" - %s", fileID) - } + logSnapshotFileIDs(t, db) // Check files before cleanup - err = db.conn.QueryRow("SELECT COUNT(*) FROM files").Scan(&count) - if err != nil { - t.Fatal(err) - } - + count = countRow(t, db, countFilesQuery) t.Logf("Files count before cleanup: %d", count) // Run orphaned cleanup @@ -128,11 +142,7 @@ func TestOrphanedFileCleanupDebug(t *testing.T) { t.Log("Ran orphaned cleanup") // Check files after cleanup - err = db.conn.QueryRow("SELECT COUNT(*) FROM files").Scan(&count) - if err != nil { - t.Fatal(err) - } - + count = countRow(t, db, countFilesQuery) t.Logf("Files count after cleanup: %d", count) // List remaining files @@ -156,18 +166,12 @@ func TestOrphanedFileCleanupDebug(t *testing.T) { if orphanedFile != nil { t.Error("orphaned file should have been deleted") // Let's check why it wasn't deleted - var exists bool - - err = db.conn.QueryRow(` + stillReferenced := countRow(t, db, ` SELECT EXISTS( - SELECT 1 FROM snapshot_files + SELECT 1 FROM snapshot_files WHERE file_id = ? - )`, file1.ID).Scan(&exists) - if err != nil { - t.Fatal(err) - } - - t.Logf("File1 exists in snapshot_files: %v", exists) + )`, file1.ID) + t.Logf("File1 exists in snapshot_files: %v", stillReferenced != 0) } else { t.Log("Orphaned file was correctly deleted") } diff --git a/internal/database/repository_edge_cases_test.go b/internal/database/repository_edge_cases_test.go index 666f33f..3e8cfb2 100644 --- a/internal/database/repository_edge_cases_test.go +++ b/internal/database/repository_edge_cases_test.go @@ -1,3 +1,4 @@ +//nolint:testpackage // inspects the unexported database connection package database import ( @@ -10,20 +11,17 @@ import ( "sneak.berlin/go/vaultik/internal/types" ) -// TestFileRepositoryEdgeCases tests edge cases for file repository -func TestFileRepositoryEdgeCases(t *testing.T) { - db, cleanup := setupTestDB(t) - defer cleanup() +// fileEdgeCase describes one Create edge-case scenario. +type fileEdgeCase struct { + name string + file *File + wantErr bool + errMsg string +} - ctx := context.Background() - repo := NewFileRepository(db) - - tests := []struct { - name string - file *File - wantErr bool - errMsg string - }{ +// fileEdgeCases returns the Create edge-case table. +func fileEdgeCases() []fileEdgeCase { + return []fileEdgeCase{ { name: "empty path", file: &File{ @@ -51,6 +49,7 @@ func TestFileRepositoryEdgeCases(t *testing.T) { { name: "path with special characters", file: &File{ + //nolint:gosmopolitan // non-ASCII path is deliberate test data Path: "/test/file with spaces and 特殊文字.txt", MTime: time.Now(), Size: 1024, @@ -86,12 +85,26 @@ func TestFileRepositoryEdgeCases(t *testing.T) { wantErr: false, }, } +} - for i, tt := range tests { +// TestFileRepositoryEdgeCases tests edge cases for file repository +func TestFileRepositoryEdgeCases(t *testing.T) { + t.Parallel() + + db, cleanup := setupTestDB(t) + t.Cleanup(cleanup) + + ctx := context.Background() + repo := NewFileRepository(db) + + for i, tt := range fileEdgeCases() { t.Run(tt.name, func(t *testing.T) { + t.Parallel() + // Add a unique suffix to paths to avoid UNIQUE constraint violations if tt.file.Path != "" { - tt.file.Path = types.FilePath(fmt.Sprintf("%s_%d_%d", tt.file.Path, i, time.Now().UnixNano())) + tt.file.Path = types.FilePath(fmt.Sprintf("%s_%d_%d", + tt.file.Path, i, time.Now().UnixNano())) } err := repo.Create(ctx, nil, tt.file) @@ -106,65 +119,128 @@ func TestFileRepositoryEdgeCases(t *testing.T) { } } +// testDuplicateFilePaths exercises the UPSERT behavior for duplicate paths. +func testDuplicateFilePaths(t *testing.T, repos *Repositories) { + t.Helper() + + ctx := context.Background() + + file1 := &File{ + Path: "/duplicate.txt", + MTime: time.Now(), + Size: 1024, + Mode: 0644, + UID: 1000, + GID: 1000, + } + file2 := &File{ + Path: "/duplicate.txt", // Same path + MTime: time.Now().Add(time.Hour), + Size: 2048, + Mode: 0644, + UID: 1000, + GID: 1000, + } + + err := repos.Files.Create(ctx, nil, file1) + if err != nil { + t.Fatalf("failed to create file1: %v", err) + } + + originalID := file1.ID + + // Create with same path should update the existing record (UPSERT behavior) + err = repos.Files.Create(ctx, nil, file2) + if err != nil { + t.Fatalf("failed to create file2: %v", err) + } + + // Verify the file was updated, not duplicated + retrievedFile, err := repos.Files.GetByPath(ctx, "/duplicate.txt") + if err != nil { + t.Fatalf("failed to retrieve file: %v", err) + } + + // The file should have been updated with file2's data + if retrievedFile.Size != 2048 { + t.Errorf("expected size 2048, got %d", retrievedFile.Size) + } + + // ID might be different due to the UPSERT + if retrievedFile.ID != file2.ID { + t.Logf("File ID changed from %s to %s during upsert", + originalID, retrievedFile.ID) + } +} + +// testDuplicateFileChunks exercises idempotent file-chunk mapping creation. +func testDuplicateFileChunks(t *testing.T, repos *Repositories) { + t.Helper() + + ctx := context.Background() + + file := &File{ + Path: "/test-dup-fc.txt", + MTime: time.Now(), + Size: 1024, + Mode: 0644, + UID: 1000, + GID: 1000, + } + + err := repos.Files.Create(ctx, nil, file) + if err != nil { + t.Fatal(err) + } + + chunk := &Chunk{ + ChunkHash: types.ChunkHash("test-chunk-dup"), + Size: 1024, + } + + err = repos.Chunks.Create(ctx, nil, chunk) + if err != nil { + t.Fatal(err) + } + + fc := &FileChunk{ + FileID: file.ID, + Idx: 0, + ChunkHash: chunk.ChunkHash, + } + + err = repos.FileChunks.Create(ctx, nil, fc) + if err != nil { + t.Fatal(err) + } + + // Creating the same mapping again should be idempotent + err = repos.FileChunks.Create(ctx, nil, fc) + if err != nil { + t.Error("file-chunk creation should be idempotent") + } +} + // TestDuplicateHandling tests handling of duplicate entries func TestDuplicateHandling(t *testing.T) { + t.Parallel() + db, cleanup := setupTestDB(t) - defer cleanup() + t.Cleanup(cleanup) ctx := context.Background() repos := NewRepositories(db) // Test duplicate file paths - Create uses UPSERT logic t.Run("duplicate file paths", func(t *testing.T) { - file1 := &File{ - Path: "/duplicate.txt", - MTime: time.Now(), - Size: 1024, - Mode: 0644, - UID: 1000, - GID: 1000, - } - file2 := &File{ - Path: "/duplicate.txt", // Same path - MTime: time.Now().Add(time.Hour), - Size: 2048, - Mode: 0644, - UID: 1000, - GID: 1000, - } - - err := repos.Files.Create(ctx, nil, file1) - if err != nil { - t.Fatalf("failed to create file1: %v", err) - } - - originalID := file1.ID - - // Create with same path should update the existing record (UPSERT behavior) - err = repos.Files.Create(ctx, nil, file2) - if err != nil { - t.Fatalf("failed to create file2: %v", err) - } - - // Verify the file was updated, not duplicated - retrievedFile, err := repos.Files.GetByPath(ctx, "/duplicate.txt") - if err != nil { - t.Fatalf("failed to retrieve file: %v", err) - } - - // The file should have been updated with file2's data - if retrievedFile.Size != 2048 { - t.Errorf("expected size 2048, got %d", retrievedFile.Size) - } - - // ID might be different due to the UPSERT - if retrievedFile.ID != file2.ID { - t.Logf("File ID changed from %s to %s during upsert", originalID, retrievedFile.ID) - } + t.Parallel() + testDuplicateFilePaths(t, repos) }) // Test duplicate chunk hashes t.Run("duplicate chunk hashes", func(t *testing.T) { + t.Parallel() + chunk := &Chunk{ ChunkHash: types.ChunkHash("duplicate-chunk"), Size: 1024, @@ -184,59 +260,25 @@ func TestDuplicateHandling(t *testing.T) { // Test duplicate file-chunk mappings t.Run("duplicate file-chunk mappings", func(t *testing.T) { - file := &File{ - Path: "/test-dup-fc.txt", - MTime: time.Now(), - Size: 1024, - Mode: 0644, - UID: 1000, - GID: 1000, - } - - err := repos.Files.Create(ctx, nil, file) - if err != nil { - t.Fatal(err) - } - - chunk := &Chunk{ - ChunkHash: types.ChunkHash("test-chunk-dup"), - Size: 1024, - } - - err = repos.Chunks.Create(ctx, nil, chunk) - if err != nil { - t.Fatal(err) - } - - fc := &FileChunk{ - FileID: file.ID, - Idx: 0, - ChunkHash: chunk.ChunkHash, - } - - err = repos.FileChunks.Create(ctx, nil, fc) - if err != nil { - t.Fatal(err) - } - - // Creating the same mapping again should be idempotent - err = repos.FileChunks.Create(ctx, nil, fc) - if err != nil { - t.Error("file-chunk creation should be idempotent") - } + t.Parallel() + testDuplicateFileChunks(t, repos) }) } // TestNullHandling tests handling of NULL values func TestNullHandling(t *testing.T) { + t.Parallel() + db, cleanup := setupTestDB(t) - defer cleanup() + t.Cleanup(cleanup) ctx := context.Background() repos := NewRepositories(db) // Test file with no link target t.Run("file without link target", func(t *testing.T) { + t.Parallel() + file := &File{ Path: "/regular.txt", MTime: time.Now(), @@ -264,9 +306,11 @@ func TestNullHandling(t *testing.T) { // Test snapshot with NULL completed_at t.Run("incomplete snapshot", func(t *testing.T) { + t.Parallel() + snapshot := &Snapshot{ ID: "incomplete-test", - Hostname: "test-host", + Hostname: internalTestHost, StartedAt: time.Now(), CompletedAt: nil, // Should remain NULL until completed } @@ -288,31 +332,86 @@ func TestNullHandling(t *testing.T) { // Test blob with NULL uploaded_ts t.Run("blob not uploaded", func(t *testing.T) { - blob := &Blob{ - ID: types.NewBlobID(), - Hash: types.BlobHash("test-hash"), - CreatedTS: time.Now(), - UploadedTS: nil, // Not uploaded yet - } - - err := repos.Blobs.Create(ctx, nil, blob) - if err != nil { - t.Fatal(err) - } - - retrieved, err := repos.Blobs.GetByID(ctx, blob.ID.String()) - if err != nil { - t.Fatal(err) - } - - if retrieved.UploadedTS != nil { - t.Error("expected nil UploadedTS for non-uploaded blob") - } + t.Parallel() + verifyBlobNullUploadTS(ctx, t, repos) }) } +// verifyBlobNullUploadTS checks that a blob created without an upload +// timestamp round-trips with UploadedTS nil. +func verifyBlobNullUploadTS( + ctx context.Context, t *testing.T, repos *Repositories, +) { + t.Helper() + + blob := &Blob{ + ID: types.NewBlobID(), + Hash: types.BlobHash("test-hash"), + CreatedTS: time.Now(), + UploadedTS: nil, // Not uploaded yet + } + + err := repos.Blobs.Create(ctx, nil, blob) + if err != nil { + t.Fatal(err) + } + + retrieved, err := repos.Blobs.GetByID(ctx, blob.ID.String()) + if err != nil { + t.Fatal(err) + } + + if retrieved.UploadedTS != nil { + t.Error("expected nil UploadedTS for non-uploaded blob") + } +} + +// createLargeDatasetFiles creates fileCount files and adds every other +// one to the snapshot. +func createLargeDatasetFiles( + t *testing.T, + repos *Repositories, + snapshotID string, + fileCount int, +) { + t.Helper() + + ctx := context.Background() + start := time.Now() + + for i := range fileCount { + file := &File{ + Path: types.FilePath(fmt.Sprintf("/large/file%05d.txt", i)), + MTime: time.Now(), + Size: int64(i * 1024), + Mode: 0644, + UID: uint32(1000 + (i % 10)), + GID: uint32(1000 + (i % 10)), + } + + err := repos.Files.Create(ctx, nil, file) + if err != nil { + t.Fatalf("failed to create file %d: %v", i, err) + } + + // Add half to snapshot + if i%2 == 0 { + err = repos.Snapshots.AddFileByID(ctx, nil, snapshotID, file.ID) + if err != nil { + t.Fatal(err) + } + } + } + + t.Logf("Created %d files in %v", fileCount, time.Since(start)) +} + // TestLargeDatasets tests operations with large amounts of data +// +//nolint:tparallel // subtests share one database and are order-dependent func TestLargeDatasets(t *testing.T) { + t.Parallel() + if testing.Short() { t.Skip("skipping large dataset test in short mode") } @@ -326,7 +425,7 @@ func TestLargeDatasets(t *testing.T) { // Create a snapshot snapshot := &Snapshot{ ID: "large-dataset-test", - Hostname: "test-host", + Hostname: internalTestHost, StartedAt: time.Now(), } @@ -338,40 +437,13 @@ func TestLargeDatasets(t *testing.T) { // Create many files const fileCount = 1000 - fileIDs := make([]types.FileID, fileCount) - + //nolint:paralleltest // phases share one database and are order-dependent t.Run("create many files", func(t *testing.T) { - start := time.Now() - for i := range fileCount { - file := &File{ - Path: types.FilePath(fmt.Sprintf("/large/file%05d.txt", i)), - MTime: time.Now(), - Size: int64(i * 1024), - Mode: 0644, - UID: uint32(1000 + (i % 10)), - GID: uint32(1000 + (i % 10)), - } - - err := repos.Files.Create(ctx, nil, file) - if err != nil { - t.Fatalf("failed to create file %d: %v", i, err) - } - - fileIDs[i] = file.ID - - // Add half to snapshot - if i%2 == 0 { - err = repos.Snapshots.AddFileByID(ctx, nil, snapshot.ID.String(), file.ID) - if err != nil { - t.Fatal(err) - } - } - } - - t.Logf("Created %d files in %v", fileCount, time.Since(start)) + createLargeDatasetFiles(t, repos, snapshot.ID.String(), fileCount) }) // Test ListByPrefix performance + //nolint:paralleltest // phases share one database and are order-dependent t.Run("list by prefix performance", func(t *testing.T) { start := time.Now() @@ -388,6 +460,7 @@ func TestLargeDatasets(t *testing.T) { }) // Test orphaned cleanup performance + //nolint:paralleltest // phases share one database and are order-dependent t.Run("orphaned cleanup performance", func(t *testing.T) { start := time.Now() @@ -405,21 +478,26 @@ func TestLargeDatasets(t *testing.T) { } if len(files) != fileCount/2 { - t.Errorf("expected %d files after cleanup, got %d", fileCount/2, len(files)) + t.Errorf("expected %d files after cleanup, got %d", + fileCount/2, len(files)) } }) } // TestErrorPropagation tests that errors are properly propagated func TestErrorPropagation(t *testing.T) { + t.Parallel() + db, cleanup := setupTestDB(t) - defer cleanup() + t.Cleanup(cleanup) ctx := context.Background() repos := NewRepositories(db) // Test GetByID with non-existent ID t.Run("GetByID non-existent", func(t *testing.T) { + t.Parallel() + file, err := repos.Files.GetByID(ctx, types.NewFileID()) if err != nil { t.Errorf("GetByID should not return error for non-existent ID, got: %v", err) @@ -432,9 +510,12 @@ func TestErrorPropagation(t *testing.T) { // Test GetByPath with non-existent path t.Run("GetByPath non-existent", func(t *testing.T) { + t.Parallel() + file, err := repos.Files.GetByPath(ctx, "/non/existent/path.txt") if err != nil { - t.Errorf("GetByPath should not return error for non-existent path, got: %v", err) + t.Errorf("GetByPath should not return error for non-existent path, got: %v", + err) } if file != nil { @@ -444,6 +525,8 @@ func TestErrorPropagation(t *testing.T) { // Test invalid foreign key reference t.Run("invalid foreign key", func(t *testing.T) { + t.Parallel() + fc := &FileChunk{ FileID: types.NewFileID(), Idx: 0, @@ -463,8 +546,10 @@ func TestErrorPropagation(t *testing.T) { // TestQueryInjection tests that the system is safe from SQL injection func TestQueryInjection(t *testing.T) { + t.Parallel() + db, cleanup := setupTestDB(t) - defer cleanup() + t.Cleanup(cleanup) ctx := context.Background() repos := NewRepositories(db) @@ -479,6 +564,8 @@ func TestQueryInjection(t *testing.T) { for _, injection := range injectionTests { t.Run("injection attempt", func(t *testing.T) { + t.Parallel() + // Try injection in file path file := &File{ Path: types.FilePath(injection), @@ -495,7 +582,7 @@ func TestQueryInjection(t *testing.T) { // Verify tables still exist var count int - err := db.conn.QueryRow("SELECT COUNT(*) FROM files").Scan(&count) + err := db.conn.QueryRowContext(ctx, countFilesQuery).Scan(&count) if err != nil { t.Fatal("files table was damaged by injection") } @@ -505,6 +592,8 @@ func TestQueryInjection(t *testing.T) { // TestTimezoneHandling tests that times are properly handled in UTC func TestTimezoneHandling(t *testing.T) { + t.Parallel() + db, cleanup := setupTestDB(t) defer cleanup() diff --git a/internal/database/snapshots.go b/internal/database/snapshots.go index 3677edf..7bc550a 100644 --- a/internal/database/snapshots.go +++ b/internal/database/snapshots.go @@ -11,19 +11,27 @@ import ( "sneak.berlin/go/vaultik/internal/types" ) +// SnapshotRepository provides access to the snapshots table and its +// snapshot_files / snapshot_blobs association tables. type SnapshotRepository struct { db *DB } +// NewSnapshotRepository creates a SnapshotRepository backed by db. func NewSnapshotRepository(db *DB) *SnapshotRepository { return &SnapshotRepository{db: db} } -func (r *SnapshotRepository) Create(ctx context.Context, tx *sql.Tx, snapshot *Snapshot) error { +// Create inserts a snapshot row, using tx when non-nil. +func (r *SnapshotRepository) Create( + ctx context.Context, tx *sql.Tx, snapshot *Snapshot, +) error { query := ` - INSERT INTO snapshots (id, hostname, vaultik_version, vaultik_git_revision, started_at, completed_at, - file_count, chunk_count, blob_count, total_size, blob_size, blob_uncompressed_size, - compression_ratio, compression_level, upload_bytes, upload_duration_ms) + INSERT INTO snapshots (id, hostname, vaultik_version, + vaultik_git_revision, started_at, completed_at, + file_count, chunk_count, blob_count, total_size, blob_size, + blob_uncompressed_size, compression_ratio, compression_level, + upload_bytes, upload_duration_ms) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ` @@ -34,15 +42,21 @@ func (r *SnapshotRepository) Create(ctx context.Context, tx *sql.Tx, snapshot *S completedAt = &ts } + args := []any{ + snapshot.ID, snapshot.Hostname, snapshot.VaultikVersion, + snapshot.VaultikGitRevision, snapshot.StartedAt.Unix(), + completedAt, snapshot.FileCount, snapshot.ChunkCount, + snapshot.BlobCount, snapshot.TotalSize, snapshot.BlobSize, + snapshot.BlobUncompressedSize, snapshot.CompressionRatio, + snapshot.CompressionLevel, snapshot.UploadBytes, + snapshot.UploadDurationMs, + } + var err error if tx != nil { - _, err = tx.ExecContext(ctx, query, snapshot.ID, snapshot.Hostname, snapshot.VaultikVersion, snapshot.VaultikGitRevision, snapshot.StartedAt.Unix(), - completedAt, snapshot.FileCount, snapshot.ChunkCount, snapshot.BlobCount, snapshot.TotalSize, snapshot.BlobSize, snapshot.BlobUncompressedSize, - snapshot.CompressionRatio, snapshot.CompressionLevel, snapshot.UploadBytes, snapshot.UploadDurationMs) + _, err = tx.ExecContext(ctx, query, args...) } else { - _, err = r.db.ExecWithLog(ctx, query, snapshot.ID, snapshot.Hostname, snapshot.VaultikVersion, snapshot.VaultikGitRevision, snapshot.StartedAt.Unix(), - completedAt, snapshot.FileCount, snapshot.ChunkCount, snapshot.BlobCount, snapshot.TotalSize, snapshot.BlobSize, snapshot.BlobUncompressedSize, - snapshot.CompressionRatio, snapshot.CompressionLevel, snapshot.UploadBytes, snapshot.UploadDurationMs) + _, err = r.db.ExecWithLog(ctx, query, args...) } if err != nil { @@ -52,7 +66,14 @@ func (r *SnapshotRepository) Create(ctx context.Context, tx *sql.Tx, snapshot *S return nil } -func (r *SnapshotRepository) UpdateCounts(ctx context.Context, tx *sql.Tx, snapshotID string, fileCount, chunkCount, blobCount, totalSize, blobSize int64) error { +// UpdateCounts updates a snapshot's file/chunk/blob counters and sizes, +// recomputing the compression ratio, using tx when non-nil. +func (r *SnapshotRepository) UpdateCounts( + ctx context.Context, + tx *sql.Tx, + snapshotID string, + fileCount, chunkCount, blobCount, totalSize, blobSize int64, +) error { compressionRatio := 1.0 if totalSize > 0 { compressionRatio = float64(blobSize) / float64(totalSize) @@ -71,9 +92,13 @@ func (r *SnapshotRepository) UpdateCounts(ctx context.Context, tx *sql.Tx, snaps var err error if tx != nil { - _, err = tx.ExecContext(ctx, query, fileCount, chunkCount, blobCount, totalSize, blobSize, compressionRatio, snapshotID) + _, err = tx.ExecContext(ctx, query, + fileCount, chunkCount, blobCount, totalSize, blobSize, + compressionRatio, snapshotID) } else { - _, err = r.db.ExecWithLog(ctx, query, fileCount, chunkCount, blobCount, totalSize, blobSize, compressionRatio, snapshotID) + _, err = r.db.ExecWithLog(ctx, query, + fileCount, chunkCount, blobCount, totalSize, blobSize, + compressionRatio, snapshotID) } if err != nil { @@ -84,34 +109,23 @@ func (r *SnapshotRepository) UpdateCounts(ctx context.Context, tx *sql.Tx, snaps } // UpdateExtendedStats updates extended statistics for a snapshot -func (r *SnapshotRepository) UpdateExtendedStats(ctx context.Context, tx *sql.Tx, snapshotID string, blobUncompressedSize int64, compressionLevel int, uploadDurationMs int64) error { - // Calculate compression ratio based on uncompressed vs compressed sizes - var compressionRatio float64 - - if blobUncompressedSize > 0 { - // Get current blob_size from DB to calculate ratio - var blobSize int64 - - queryGet := `SELECT blob_size FROM snapshots WHERE id = ?` - if tx != nil { - err := tx.QueryRowContext(ctx, queryGet, snapshotID).Scan(&blobSize) - if err != nil { - return fmt.Errorf("getting blob size: %w", err) - } - } else { - err := r.db.conn.QueryRowContext(ctx, queryGet, snapshotID).Scan(&blobSize) - if err != nil { - return fmt.Errorf("getting blob size: %w", err) - } - } - - compressionRatio = float64(blobSize) / float64(blobUncompressedSize) - } else { - compressionRatio = 1.0 +func (r *SnapshotRepository) UpdateExtendedStats( + ctx context.Context, + tx *sql.Tx, + snapshotID string, + blobUncompressedSize int64, + compressionLevel int, + uploadDurationMs int64, +) error { + compressionRatio, err := r.extendedCompressionRatio( + ctx, tx, snapshotID, blobUncompressedSize, + ) + if err != nil { + return err } query := ` - UPDATE snapshots + UPDATE snapshots SET blob_uncompressed_size = ?, compression_ratio = ?, compression_level = ?, @@ -120,11 +134,14 @@ func (r *SnapshotRepository) UpdateExtendedStats(ctx context.Context, tx *sql.Tx WHERE id = ? ` - var err error if tx != nil { - _, err = tx.ExecContext(ctx, query, blobUncompressedSize, compressionRatio, compressionLevel, uploadDurationMs, snapshotID) + _, err = tx.ExecContext(ctx, query, + blobUncompressedSize, compressionRatio, compressionLevel, + uploadDurationMs, snapshotID) } else { - _, err = r.db.ExecWithLog(ctx, query, blobUncompressedSize, compressionRatio, compressionLevel, uploadDurationMs, snapshotID) + _, err = r.db.ExecWithLog(ctx, query, + blobUncompressedSize, compressionRatio, compressionLevel, + uploadDurationMs, snapshotID) } if err != nil { @@ -134,7 +151,11 @@ func (r *SnapshotRepository) UpdateExtendedStats(ctx context.Context, tx *sql.Tx return nil } -func (r *SnapshotRepository) GetByID(ctx context.Context, snapshotID string) (*Snapshot, error) { +// GetByID returns the snapshot with the given ID, or nil if no such +// snapshot exists. +func (r *SnapshotRepository) GetByID( + ctx context.Context, snapshotID string, +) (*Snapshot, error) { query := ` SELECT id, hostname, vaultik_version, vaultik_git_revision, started_at, completed_at, file_count, chunk_count, blob_count, total_size, blob_size, blob_uncompressed_size, @@ -169,7 +190,7 @@ func (r *SnapshotRepository) GetByID(ctx context.Context, snapshotID string) (*S ) if errors.Is(err, sql.ErrNoRows) { - return nil, nil + return nil, nil //nolint:nilnil // nil,nil signals not-found; callers check nil } if err != nil { @@ -185,9 +206,14 @@ func (r *SnapshotRepository) GetByID(ctx context.Context, snapshotID string) (*S return &snapshot, nil } -func (r *SnapshotRepository) ListRecent(ctx context.Context, limit int) ([]*Snapshot, error) { +// ListRecent returns up to limit snapshots, most recently started first. +func (r *SnapshotRepository) ListRecent( + ctx context.Context, limit int, +) ([]*Snapshot, error) { query := ` - SELECT id, hostname, vaultik_version, vaultik_git_revision, started_at, completed_at, file_count, chunk_count, blob_count, total_size, blob_size, compression_ratio + SELECT id, hostname, vaultik_version, vaultik_git_revision, + started_at, completed_at, file_count, chunk_count, blob_count, + total_size, blob_size, compression_ratio FROM snapshots ORDER BY started_at DESC LIMIT ? @@ -199,47 +225,13 @@ func (r *SnapshotRepository) ListRecent(ctx context.Context, limit int) ([]*Snap } defer CloseRows(rows) - var snapshots []*Snapshot - - for rows.Next() { - var ( - snapshot Snapshot - startedAtUnix int64 - completedAtUnix *int64 - ) - - err := rows.Scan( - &snapshot.ID, - &snapshot.Hostname, - &snapshot.VaultikVersion, - &snapshot.VaultikGitRevision, - &startedAtUnix, - &completedAtUnix, - &snapshot.FileCount, - &snapshot.ChunkCount, - &snapshot.BlobCount, - &snapshot.TotalSize, - &snapshot.BlobSize, - &snapshot.CompressionRatio, - ) - if err != nil { - return nil, fmt.Errorf("scanning snapshot: %w", err) - } - - snapshot.StartedAt = time.Unix(startedAtUnix, 0) - if completedAtUnix != nil { - t := time.Unix(*completedAtUnix, 0) - snapshot.CompletedAt = &t - } - - snapshots = append(snapshots, &snapshot) - } - - return snapshots, rows.Err() + return r.scanSnapshotRows(rows) } // MarkComplete marks a snapshot as completed with the current timestamp -func (r *SnapshotRepository) MarkComplete(ctx context.Context, tx *sql.Tx, snapshotID string) error { +func (r *SnapshotRepository) MarkComplete( + ctx context.Context, tx *sql.Tx, snapshotID string, +) error { query := ` UPDATE snapshots SET completed_at = ? @@ -263,7 +255,9 @@ func (r *SnapshotRepository) MarkComplete(ctx context.Context, tx *sql.Tx, snaps } // AddFile adds a file to a snapshot -func (r *SnapshotRepository) AddFile(ctx context.Context, tx *sql.Tx, snapshotID string, filePath string) error { +func (r *SnapshotRepository) AddFile( + ctx context.Context, tx *sql.Tx, snapshotID string, filePath string, +) error { query := ` INSERT OR IGNORE INTO snapshot_files (snapshot_id, file_id) SELECT ?, id FROM files WHERE path = ? @@ -284,7 +278,9 @@ func (r *SnapshotRepository) AddFile(ctx context.Context, tx *sql.Tx, snapshotID } // AddFileByID adds a file to a snapshot by file ID -func (r *SnapshotRepository) AddFileByID(ctx context.Context, tx *sql.Tx, snapshotID string, fileID types.FileID) error { +func (r *SnapshotRepository) AddFileByID( + ctx context.Context, tx *sql.Tx, snapshotID string, fileID types.FileID, +) error { query := ` INSERT OR IGNORE INTO snapshot_files (snapshot_id, file_id) VALUES (?, ?) @@ -305,12 +301,17 @@ func (r *SnapshotRepository) AddFileByID(ctx context.Context, tx *sql.Tx, snapsh } // AddFilesByIDBatch adds multiple files to a snapshot in batched inserts -func (r *SnapshotRepository) AddFilesByIDBatch(ctx context.Context, tx *sql.Tx, snapshotID string, fileIDs []types.FileID) error { +func (r *SnapshotRepository) AddFilesByIDBatch( + ctx context.Context, tx *sql.Tx, snapshotID string, fileIDs []types.FileID, +) error { if len(fileIDs) == 0 { return nil } - // Each entry has 2 values, so batch at 400 to be safe + // Each snapshot_files row binds this many SQL variables. + const snapshotFileCols = 2 + + // Batch at 400 rows to be safe with SQLite's variable limit. const batchSize = 400 for i := 0; i < len(fileIDs); i += batchSize { @@ -320,7 +321,7 @@ func (r *SnapshotRepository) AddFilesByIDBatch(ctx context.Context, tx *sql.Tx, query := "INSERT OR IGNORE INTO snapshot_files (snapshot_id, file_id) VALUES " - args := make([]any, 0, len(batch)*2) + args := make([]any, 0, len(batch)*snapshotFileCols) var querySb312 strings.Builder @@ -334,7 +335,7 @@ func (r *SnapshotRepository) AddFilesByIDBatch(ctx context.Context, tx *sql.Tx, args = append(args, snapshotID, fileID.String()) } - query += querySb312.String() + query += querySb312.String() //nolint:gosec // G202: appends "?" placeholders only var err error if tx != nil { @@ -361,7 +362,9 @@ func (r *SnapshotRepository) AddFilesByIDBatch(ctx context.Context, tx *sql.Tx, // Returns the number of rows inserted (i.e. blobs that were previously // referenced indirectly via file_chunks but not yet recorded in // snapshot_blobs for this snapshot). -func (r *SnapshotRepository) PopulateReferencedBlobs(ctx context.Context, tx *sql.Tx, snapshotID string) (int64, error) { +func (r *SnapshotRepository) PopulateReferencedBlobs( + ctx context.Context, tx *sql.Tx, snapshotID string, +) (int64, error) { query := ` INSERT OR IGNORE INTO snapshot_blobs (snapshot_id, blob_id, blob_hash) SELECT DISTINCT ?, blobs.id, blobs.blob_hash @@ -393,7 +396,13 @@ func (r *SnapshotRepository) PopulateReferencedBlobs(ctx context.Context, tx *sq } // AddBlob adds a blob to a snapshot -func (r *SnapshotRepository) AddBlob(ctx context.Context, tx *sql.Tx, snapshotID string, blobID types.BlobID, blobHash types.BlobHash) error { +func (r *SnapshotRepository) AddBlob( + ctx context.Context, + tx *sql.Tx, + snapshotID string, + blobID types.BlobID, + blobHash types.BlobHash, +) error { query := ` INSERT OR IGNORE INTO snapshot_blobs (snapshot_id, blob_id, blob_hash) VALUES (?, ?, ?) @@ -414,7 +423,9 @@ func (r *SnapshotRepository) AddBlob(ctx context.Context, tx *sql.Tx, snapshotID } // GetBlobHashes returns all blob hashes for a snapshot -func (r *SnapshotRepository) GetBlobHashes(ctx context.Context, snapshotID string) ([]string, error) { +func (r *SnapshotRepository) GetBlobHashes( + ctx context.Context, snapshotID string, +) ([]string, error) { query := ` SELECT sb.blob_hash FROM snapshot_blobs sb @@ -444,8 +455,11 @@ func (r *SnapshotRepository) GetBlobHashes(ctx context.Context, snapshotID strin return blobs, rows.Err() } -// GetSnapshotTotalCompressedSize returns the total compressed size of all blobs referenced by a snapshot -func (r *SnapshotRepository) GetSnapshotTotalCompressedSize(ctx context.Context, snapshotID string) (int64, error) { +// GetSnapshotTotalCompressedSize returns the total compressed size of all +// blobs referenced by a snapshot. +func (r *SnapshotRepository) GetSnapshotTotalCompressedSize( + ctx context.Context, snapshotID string, +) (int64, error) { query := ` SELECT COALESCE(SUM(b.compressed_size), 0) FROM snapshot_blobs sb @@ -465,7 +479,9 @@ func (r *SnapshotRepository) GetSnapshotTotalCompressedSize(ctx context.Context, // GetSnapshotUncompressedChunkSize returns the sum of plaintext sizes of all unique // chunks referenced by a snapshot (via snapshot_files → file_chunks → chunks). -func (r *SnapshotRepository) GetSnapshotUncompressedChunkSize(ctx context.Context, snapshotID string) (int64, error) { +func (r *SnapshotRepository) GetSnapshotUncompressedChunkSize( + ctx context.Context, snapshotID string, +) (int64, error) { query := ` SELECT COALESCE(SUM(c.size), 0) FROM ( @@ -491,7 +507,9 @@ func (r *SnapshotRepository) GetSnapshotUncompressedChunkSize(ctx context.Contex // referenced by this snapshot but not by any earlier completed snapshot known to // the local database. The result is the marginal uncompressed data this snapshot // added to the dedup pool — i.e., the delta from prior snapshots. -func (r *SnapshotRepository) GetSnapshotNewChunkSize(ctx context.Context, snapshotID string) (int64, error) { +func (r *SnapshotRepository) GetSnapshotNewChunkSize( + ctx context.Context, snapshotID string, +) (int64, error) { query := ` WITH this_snap_chunks AS ( SELECT DISTINCT fc.chunk_hash @@ -516,7 +534,9 @@ func (r *SnapshotRepository) GetSnapshotNewChunkSize(ctx context.Context, snapsh var totalSize int64 - err := r.db.conn.QueryRowContext(ctx, query, snapshotID, snapshotID, snapshotID).Scan(&totalSize) + err := r.db.conn.QueryRowContext( + ctx, query, snapshotID, snapshotID, snapshotID, + ).Scan(&totalSize) if err != nil { return 0, fmt.Errorf("querying new chunk size: %w", err) } @@ -525,9 +545,13 @@ func (r *SnapshotRepository) GetSnapshotNewChunkSize(ctx context.Context, snapsh } // GetIncompleteSnapshots returns all snapshots that haven't been completed -func (r *SnapshotRepository) GetIncompleteSnapshots(ctx context.Context) ([]*Snapshot, error) { +func (r *SnapshotRepository) GetIncompleteSnapshots( + ctx context.Context, +) ([]*Snapshot, error) { query := ` - SELECT id, hostname, vaultik_version, vaultik_git_revision, started_at, completed_at, file_count, chunk_count, blob_count, total_size, blob_size, compression_ratio + SELECT id, hostname, vaultik_version, vaultik_git_revision, + started_at, completed_at, file_count, chunk_count, blob_count, + total_size, blob_size, compression_ratio FROM snapshots WHERE completed_at IS NULL ORDER BY started_at DESC @@ -539,49 +563,17 @@ func (r *SnapshotRepository) GetIncompleteSnapshots(ctx context.Context) ([]*Sna } defer CloseRows(rows) - var snapshots []*Snapshot - - for rows.Next() { - var ( - snapshot Snapshot - startedAtUnix int64 - completedAtUnix *int64 - ) - - err := rows.Scan( - &snapshot.ID, - &snapshot.Hostname, - &snapshot.VaultikVersion, - &snapshot.VaultikGitRevision, - &startedAtUnix, - &completedAtUnix, - &snapshot.FileCount, - &snapshot.ChunkCount, - &snapshot.BlobCount, - &snapshot.TotalSize, - &snapshot.BlobSize, - &snapshot.CompressionRatio, - ) - if err != nil { - return nil, fmt.Errorf("scanning snapshot: %w", err) - } - - snapshot.StartedAt = time.Unix(startedAtUnix, 0) - if completedAtUnix != nil { - t := time.Unix(*completedAtUnix, 0) - snapshot.CompletedAt = &t - } - - snapshots = append(snapshots, &snapshot) - } - - return snapshots, rows.Err() + return r.scanSnapshotRows(rows) } // GetIncompleteByHostname returns all incomplete snapshots for a specific hostname -func (r *SnapshotRepository) GetIncompleteByHostname(ctx context.Context, hostname string) ([]*Snapshot, error) { +func (r *SnapshotRepository) GetIncompleteByHostname( + ctx context.Context, hostname string, +) ([]*Snapshot, error) { query := ` - SELECT id, hostname, vaultik_version, vaultik_git_revision, started_at, completed_at, file_count, chunk_count, blob_count, total_size, blob_size, compression_ratio + SELECT id, hostname, vaultik_version, vaultik_git_revision, + started_at, completed_at, file_count, chunk_count, blob_count, + total_size, blob_size, compression_ratio FROM snapshots WHERE completed_at IS NULL AND hostname = ? ORDER BY started_at DESC @@ -645,7 +637,9 @@ func (r *SnapshotRepository) Delete(ctx context.Context, snapshotID string) erro } // DeleteSnapshotFiles removes all snapshot_files entries for a snapshot -func (r *SnapshotRepository) DeleteSnapshotFiles(ctx context.Context, snapshotID string) error { +func (r *SnapshotRepository) DeleteSnapshotFiles( + ctx context.Context, snapshotID string, +) error { query := `DELETE FROM snapshot_files WHERE snapshot_id = ?` _, err := r.db.ExecWithLog(ctx, query, snapshotID) @@ -657,7 +651,9 @@ func (r *SnapshotRepository) DeleteSnapshotFiles(ctx context.Context, snapshotID } // DeleteSnapshotBlobs removes all snapshot_blobs entries for a snapshot -func (r *SnapshotRepository) DeleteSnapshotBlobs(ctx context.Context, snapshotID string) error { +func (r *SnapshotRepository) DeleteSnapshotBlobs( + ctx context.Context, snapshotID string, +) error { query := `DELETE FROM snapshot_blobs WHERE snapshot_id = ?` _, err := r.db.ExecWithLog(ctx, query, snapshotID) @@ -669,7 +665,9 @@ func (r *SnapshotRepository) DeleteSnapshotBlobs(ctx context.Context, snapshotID } // DeleteSnapshotUploads removes all uploads entries for a snapshot -func (r *SnapshotRepository) DeleteSnapshotUploads(ctx context.Context, snapshotID string) error { +func (r *SnapshotRepository) DeleteSnapshotUploads( + ctx context.Context, snapshotID string, +) error { query := `DELETE FROM uploads WHERE snapshot_id = ?` _, err := r.db.ExecWithLog(ctx, query, snapshotID) @@ -679,3 +677,77 @@ func (r *SnapshotRepository) DeleteSnapshotUploads(ctx context.Context, snapshot return nil } + +// extendedCompressionRatio computes the compression ratio for a snapshot +// from its stored blob_size and the given uncompressed size. Returns 1.0 +// when the uncompressed size is zero. +func (r *SnapshotRepository) extendedCompressionRatio( + ctx context.Context, + tx *sql.Tx, + snapshotID string, + blobUncompressedSize int64, +) (float64, error) { + if blobUncompressedSize <= 0 { + return 1.0, nil + } + + // Get current blob_size from DB to calculate ratio + var blobSize int64 + + queryGet := `SELECT blob_size FROM snapshots WHERE id = ?` + + var err error + if tx != nil { + err = tx.QueryRowContext(ctx, queryGet, snapshotID).Scan(&blobSize) + } else { + err = r.db.conn.QueryRowContext(ctx, queryGet, snapshotID).Scan(&blobSize) + } + + if err != nil { + return 0, fmt.Errorf("getting blob size: %w", err) + } + + return float64(blobSize) / float64(blobUncompressedSize), nil +} + +// scanSnapshotRows scans the standard snapshot column set from a rows +// iterator into Snapshot records. +func (r *SnapshotRepository) scanSnapshotRows(rows *sql.Rows) ([]*Snapshot, error) { + var snapshots []*Snapshot + + for rows.Next() { + var ( + snapshot Snapshot + startedAtUnix int64 + completedAtUnix *int64 + ) + + err := rows.Scan( + &snapshot.ID, + &snapshot.Hostname, + &snapshot.VaultikVersion, + &snapshot.VaultikGitRevision, + &startedAtUnix, + &completedAtUnix, + &snapshot.FileCount, + &snapshot.ChunkCount, + &snapshot.BlobCount, + &snapshot.TotalSize, + &snapshot.BlobSize, + &snapshot.CompressionRatio, + ) + if err != nil { + return nil, fmt.Errorf("scanning snapshot: %w", err) + } + + snapshot.StartedAt = time.Unix(startedAtUnix, 0) + if completedAtUnix != nil { + t := time.Unix(*completedAtUnix, 0) + snapshot.CompletedAt = &t + } + + snapshots = append(snapshots, &snapshot) + } + + return snapshots, rows.Err() +} diff --git a/internal/database/snapshots_test.go b/internal/database/snapshots_test.go index a1efaa1..5204a61 100644 --- a/internal/database/snapshots_test.go +++ b/internal/database/snapshots_test.go @@ -1,4 +1,4 @@ -package database +package database_test import ( "context" @@ -7,6 +7,7 @@ import ( "testing" "time" + "sneak.berlin/go/vaultik/internal/database" "sneak.berlin/go/vaultik/internal/types" ) @@ -21,17 +22,19 @@ const ( ) func TestSnapshotRepository(t *testing.T) { + t.Parallel() + db, cleanup := setupTestDB(t) defer cleanup() ctx := context.Background() - repo := NewSnapshotRepository(db) + repo := database.NewSnapshotRepository(db) // Test Create - snapshot := &Snapshot{ + snapshot := &database.Snapshot{ ID: "2024-01-01T12:00:00Z", - Hostname: "test-host", - VaultikVersion: "1.0.0", + Hostname: testHostname, + VaultikVersion: testVersion, StartedAt: time.Now().Truncate(time.Second), CompletedAt: nil, FileCount: 100, @@ -62,20 +65,52 @@ func TestSnapshotRepository(t *testing.T) { } if retrieved.Hostname != snapshot.Hostname { - t.Errorf("hostname mismatch: got %s, want %s", retrieved.Hostname, snapshot.Hostname) + t.Errorf("hostname mismatch: got %s, want %s", + retrieved.Hostname, snapshot.Hostname) } if retrieved.FileCount != snapshot.FileCount { - t.Errorf("file count mismatch: got %d, want %d", retrieved.FileCount, snapshot.FileCount) + t.Errorf("file count mismatch: got %d, want %d", + retrieved.FileCount, snapshot.FileCount) + } +} + +func TestSnapshotRepositoryUpdateCounts(t *testing.T) { + t.Parallel() + + db, cleanup := setupTestDB(t) + defer cleanup() + + ctx := context.Background() + repo := database.NewSnapshotRepository(db) + + snapshot := &database.Snapshot{ + ID: "2024-01-02T12:00:00Z", + Hostname: testHostname, + VaultikVersion: testVersion, + StartedAt: time.Now().Truncate(time.Second), + CompletedAt: nil, + FileCount: 100, + ChunkCount: 500, + BlobCount: 10, + TotalSize: oneHundredMebibytes, + BlobSize: fortyMebibytes, + CompressionRatio: compressionRatioPoint4, + } + + err := repo.Create(ctx, nil, snapshot) + if err != nil { + t.Fatalf("failed to create snapshot: %v", err) } // Test UpdateCounts - err = repo.UpdateCounts(ctx, nil, snapshot.ID.String(), 200, 1000, 20, twoHundredMebibytes, sixtyMebibytes) + err = repo.UpdateCounts(ctx, nil, snapshot.ID.String(), + 200, 1000, 20, twoHundredMebibytes, sixtyMebibytes) if err != nil { t.Fatalf("failed to update counts: %v", err) } - retrieved, err = repo.GetByID(ctx, snapshot.ID.String()) + retrieved, err := repo.GetByID(ctx, snapshot.ID.String()) if err != nil { t.Fatalf("failed to get updated snapshot: %v", err) } @@ -85,7 +120,8 @@ func TestSnapshotRepository(t *testing.T) { } if retrieved.ChunkCount != 1000 { - t.Errorf("chunk count not updated: got %d, want %d", retrieved.ChunkCount, 1000) + t.Errorf("chunk count not updated: got %d, want %d", + retrieved.ChunkCount, 1000) } if retrieved.BlobCount != 20 { @@ -93,25 +129,37 @@ func TestSnapshotRepository(t *testing.T) { } if retrieved.TotalSize != twoHundredMebibytes { - t.Errorf("total size not updated: got %d, want %d", retrieved.TotalSize, twoHundredMebibytes) + t.Errorf("total size not updated: got %d, want %d", + retrieved.TotalSize, twoHundredMebibytes) } if retrieved.BlobSize != sixtyMebibytes { - t.Errorf("blob size not updated: got %d, want %d", retrieved.BlobSize, sixtyMebibytes) + t.Errorf("blob size not updated: got %d, want %d", + retrieved.BlobSize, sixtyMebibytes) } expectedRatio := compressionRatioPoint3 // 0.3 if math.Abs(retrieved.CompressionRatio-expectedRatio) > 0.001 { - t.Errorf("compression ratio not updated: got %f, want %f", retrieved.CompressionRatio, expectedRatio) + t.Errorf("compression ratio not updated: got %f, want %f", + retrieved.CompressionRatio, expectedRatio) } +} - // Test ListRecent - // Add more snapshots - for i := 2; i <= 5; i++ { - s := &Snapshot{ +func TestSnapshotRepositoryListRecent(t *testing.T) { + t.Parallel() + + db, cleanup := setupTestDB(t) + defer cleanup() + + ctx := context.Background() + repo := database.NewSnapshotRepository(db) + + // Add snapshots + for i := 1; i <= 5; i++ { + s := &database.Snapshot{ ID: types.SnapshotID(fmt.Sprintf("2024-01-0%dT12:00:00Z", i)), - Hostname: "test-host", - VaultikVersion: "1.0.0", + Hostname: testHostname, + VaultikVersion: testVersion, StartedAt: time.Now().Add(time.Duration(i) * time.Hour).Truncate(time.Second), CompletedAt: nil, FileCount: int64(100 * i), @@ -144,11 +192,13 @@ func TestSnapshotRepository(t *testing.T) { } func TestSnapshotRepositoryNotFound(t *testing.T) { + t.Parallel() + db, cleanup := setupTestDB(t) defer cleanup() ctx := context.Background() - repo := NewSnapshotRepository(db) + repo := database.NewSnapshotRepository(db) // Test GetByID with non-existent ID snapshot, err := repo.GetByID(ctx, "nonexistent") @@ -161,7 +211,8 @@ func TestSnapshotRepositoryNotFound(t *testing.T) { } // Test UpdateCounts on non-existent snapshot - err = repo.UpdateCounts(ctx, nil, "nonexistent", 100, 200, 10, oneHundredMebibytes, fortyMebibytes) + err = repo.UpdateCounts(ctx, nil, "nonexistent", + 100, 200, 10, oneHundredMebibytes, fortyMebibytes) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -169,16 +220,18 @@ func TestSnapshotRepositoryNotFound(t *testing.T) { } func TestSnapshotRepositoryDuplicate(t *testing.T) { + t.Parallel() + db, cleanup := setupTestDB(t) defer cleanup() ctx := context.Background() - repo := NewSnapshotRepository(db) + repo := database.NewSnapshotRepository(db) - snapshot := &Snapshot{ + snapshot := &database.Snapshot{ ID: "2024-01-01T12:00:00Z", - Hostname: "test-host", - VaultikVersion: "1.0.0", + Hostname: testHostname, + VaultikVersion: testVersion, StartedAt: time.Now().Truncate(time.Second), CompletedAt: nil, FileCount: 100, diff --git a/internal/database/uploads.go b/internal/database/uploads.go index 4d64243..1070550 100644 --- a/internal/database/uploads.go +++ b/internal/database/uploads.go @@ -29,7 +29,9 @@ func NewUploadRepository(conn *sql.DB) *UploadRepository { } // Create inserts a new upload record -func (r *UploadRepository) Create(ctx context.Context, tx *sql.Tx, upload *Upload) error { +func (r *UploadRepository) Create( + ctx context.Context, tx *sql.Tx, upload *Upload, +) error { query := ` INSERT INTO uploads (blob_hash, snapshot_id, uploaded_at, size, duration_ms) VALUES (?, ?, ?, ?, ?) @@ -37,16 +39,22 @@ func (r *UploadRepository) Create(ctx context.Context, tx *sql.Tx, upload *Uploa var err error if tx != nil { - _, err = tx.ExecContext(ctx, query, upload.BlobHash, upload.SnapshotID, upload.UploadedAt, upload.Size, upload.DurationMs) + _, err = tx.ExecContext(ctx, query, + upload.BlobHash, upload.SnapshotID, upload.UploadedAt, + upload.Size, upload.DurationMs) } else { - _, err = r.conn.ExecContext(ctx, query, upload.BlobHash, upload.SnapshotID, upload.UploadedAt, upload.Size, upload.DurationMs) + _, err = r.conn.ExecContext(ctx, query, + upload.BlobHash, upload.SnapshotID, upload.UploadedAt, + upload.Size, upload.DurationMs) } return err } // GetByBlobHash retrieves an upload record by blob hash -func (r *UploadRepository) GetByBlobHash(ctx context.Context, blobHash string) (*Upload, error) { +func (r *UploadRepository) GetByBlobHash( + ctx context.Context, blobHash string, +) (*Upload, error) { query := ` SELECT blob_hash, uploaded_at, size, duration_ms FROM uploads @@ -63,7 +71,7 @@ func (r *UploadRepository) GetByBlobHash(ctx context.Context, blobHash string) ( ) if errors.Is(err, sql.ErrNoRows) { - return nil, nil + return nil, nil //nolint:nilnil // nil,nil signals not-found; callers check nil } if err != nil { @@ -74,7 +82,9 @@ func (r *UploadRepository) GetByBlobHash(ctx context.Context, blobHash string) ( } // GetRecentUploads retrieves recent uploads ordered by upload time -func (r *UploadRepository) GetRecentUploads(ctx context.Context, limit int) ([]*Upload, error) { +func (r *UploadRepository) GetRecentUploads( + ctx context.Context, limit int, +) ([]*Upload, error) { query := ` SELECT blob_hash, uploaded_at, size, duration_ms FROM uploads @@ -98,7 +108,9 @@ func (r *UploadRepository) GetRecentUploads(ctx context.Context, limit int) ([]* for rows.Next() { var upload Upload - err := rows.Scan(&upload.BlobHash, &upload.UploadedAt, &upload.Size, &upload.DurationMs) + err := rows.Scan( + &upload.BlobHash, &upload.UploadedAt, &upload.Size, &upload.DurationMs, + ) if err != nil { return nil, err } @@ -110,9 +122,11 @@ func (r *UploadRepository) GetRecentUploads(ctx context.Context, limit int) ([]* } // GetUploadStats returns aggregate statistics for uploads -func (r *UploadRepository) GetUploadStats(ctx context.Context, since time.Time) (*UploadStats, error) { +func (r *UploadRepository) GetUploadStats( + ctx context.Context, since time.Time, +) (*UploadStats, error) { query := ` - SELECT + SELECT COUNT(*) as count, COALESCE(SUM(size), 0) as total_size, COALESCE(AVG(duration_ms), 0) as avg_duration_ms, @@ -145,7 +159,9 @@ type UploadStats struct { } // GetCountBySnapshot returns the count of uploads for a specific snapshot -func (r *UploadRepository) GetCountBySnapshot(ctx context.Context, snapshotID string) (int64, error) { +func (r *UploadRepository) GetCountBySnapshot( + ctx context.Context, snapshotID string, +) (int64, error) { query := `SELECT COUNT(*) FROM uploads WHERE snapshot_id = ?` var count int64 diff --git a/internal/globals/globals.go b/internal/globals/globals.go index 21edab6..bbd2843 100644 --- a/internal/globals/globals.go +++ b/internal/globals/globals.go @@ -1,3 +1,5 @@ +// Package globals holds application-wide metadata (name, version, +// commit) that is populated at build time via linker flags. package globals import ( @@ -5,16 +7,16 @@ import ( ) // Appname is the application name, populated from main(). -var Appname string = "vaultik" +var Appname = "vaultik" //nolint:gochecknoglobals // set via -ldflags at build time // Version is the application version, populated from main(). -var Version string = "dev" +var Version = "dev" //nolint:gochecknoglobals // set via -ldflags at build time // Commit is the git commit hash, populated from main(). -var Commit string = "unknown" +var Commit = "unknown" //nolint:gochecknoglobals // set via -ldflags at build time // CommitDate is the ISO-8601 date of the commit, populated from main(). -var CommitDate string = "unknown" +var CommitDate = "unknown" //nolint:gochecknoglobals // set via -ldflags at build time // Author identifies the upstream author of vaultik. const Author = "Jeffrey Paul " @@ -34,7 +36,8 @@ type Globals struct { StartTime time.Time } -// New creates and returns a new Globals instance initialized with the package-level variables. +// New creates and returns a new Globals instance initialized with the +// package-level variables. func New() (*Globals, error) { return &Globals{ Appname: Appname, @@ -44,11 +47,14 @@ func New() (*Globals, error) { }, nil } +// shortCommitLen is the number of commit-hash characters ShortCommit keeps. +const shortCommitLen = 12 + // ShortCommit returns the first 12 chars of the commit hash, or the // whole string if it's shorter (e.g. "unknown"). func (g *Globals) ShortCommit() string { - if len(g.Commit) > 12 { - return g.Commit[:12] + if len(g.Commit) > shortCommitLen { + return g.Commit[:shortCommitLen] } return g.Commit diff --git a/internal/globals/globals_test.go b/internal/globals/globals_test.go index 164602c..2784428 100644 --- a/internal/globals/globals_test.go +++ b/internal/globals/globals_test.go @@ -1,12 +1,16 @@ -package globals +package globals_test import ( "testing" + + "sneak.berlin/go/vaultik/internal/globals" ) // TestGlobalsNew ensures the globals package initializes correctly func TestGlobalsNew(t *testing.T) { - g, err := New() + t.Parallel() + + g, err := globals.New() if err != nil { t.Fatalf("Failed to create Globals: %v", err) } diff --git a/internal/log/log.go b/internal/log/log.go index c9867ee..17025ca 100644 --- a/internal/log/log.go +++ b/internal/log/log.go @@ -1,3 +1,5 @@ +// Package log provides the application-wide structured logger: slog +// with a colorized TTY handler on terminals and JSON output otherwise. package log import ( @@ -12,12 +14,12 @@ import ( "golang.org/x/term" ) -// LogLevel represents the logging level. -type LogLevel int +// Level represents the logging level. +type Level int const ( // LevelFatal represents a fatal error level that will exit the program. - LevelFatal LogLevel = iota + LevelFatal Level = iota // LevelError represents an error level. LevelError // LevelWarn represents a warning level. @@ -38,6 +40,7 @@ type Config struct { Quiet bool } +//nolint:gochecknoglobals // package-level logger is the package's purpose var logger *slog.Logger // Initialize sets up the global logger based on the provided configuration. @@ -45,18 +48,19 @@ func Initialize(cfg Config) { // Determine log level based on configuration var level slog.Level - if cfg.Cron || cfg.Quiet { + switch { + case cfg.Cron || cfg.Quiet: // In cron/quiet mode keep warnings and errors visible — the // whole point of --cron is to stay silent only on total // success, so that anything cron emails to root is genuinely // "something went wrong, look at it." A backup with stuck // permission errors or skipped files should NOT be silent. level = slog.LevelWarn - } else if cfg.Debug || strings.Contains(os.Getenv("GODEBUG"), "vaultik") { + case cfg.Debug || strings.Contains(os.Getenv("GODEBUG"), "vaultik"): level = slog.LevelDebug - } else if cfg.Verbose { + case cfg.Verbose: level = slog.LevelInfo - } else { + default: level = slog.LevelWarn } @@ -78,9 +82,13 @@ func Initialize(cfg Config) { slog.SetDefault(logger) } +// callerSkipFrames is the number of stack frames between runtime.Caller +// and the code that invoked the package-level logging function. +const callerSkipFrames = 2 + // getCaller returns the caller information as a string -func getCaller(skip int) string { - _, file, line, ok := runtime.Caller(skip) +func getCaller() string { + _, file, line, ok := runtime.Caller(callerSkipFrames) if !ok { return "unknown" } @@ -92,7 +100,7 @@ func getCaller(skip int) string { func Fatal(msg string, args ...any) { if logger != nil { // Add caller info to args - args = append(args, "caller", getCaller(2)) + args = append(args, "caller", getCaller()) logger.Error(msg, args...) } @@ -107,7 +115,7 @@ func Fatalf(format string, args ...any) { // Error logs an error message. func Error(msg string, args ...any) { if logger != nil { - args = append(args, "caller", getCaller(2)) + args = append(args, "caller", getCaller()) logger.Error(msg, args...) } } @@ -120,7 +128,7 @@ func Errorf(format string, args ...any) { // Warn logs a warning message. func Warn(msg string, args ...any) { if logger != nil { - args = append(args, "caller", getCaller(2)) + args = append(args, "caller", getCaller()) logger.Warn(msg, args...) } } @@ -133,7 +141,7 @@ func Warnf(format string, args ...any) { // Notice logs a notice message (mapped to Info level). func Notice(msg string, args ...any) { if logger != nil { - args = append(args, "caller", getCaller(2)) + args = append(args, "caller", getCaller()) logger.Info(msg, args...) } } @@ -146,7 +154,7 @@ func Noticef(format string, args ...any) { // Info logs an informational message. func Info(msg string, args ...any) { if logger != nil { - args = append(args, "caller", getCaller(2)) + args = append(args, "caller", getCaller()) logger.Info(msg, args...) } } @@ -159,7 +167,7 @@ func Infof(format string, args ...any) { // Debug logs a debug message. func Debug(msg string, args ...any) { if logger != nil { - args = append(args, "caller", getCaller(2)) + args = append(args, "caller", getCaller()) logger.Debug(msg, args...) } } @@ -179,7 +187,7 @@ func With(args ...any) *slog.Logger { } // WithContext returns a logger with the provided context. -func WithContext(ctx context.Context) *slog.Logger { +func WithContext(_ context.Context) *slog.Logger { return logger } diff --git a/internal/log/module.go b/internal/log/module.go index 1b65128..525f969 100644 --- a/internal/log/module.go +++ b/internal/log/module.go @@ -5,6 +5,8 @@ import ( ) // Module exports logging functionality for dependency injection. +// +//nolint:gochecknoglobals // fx module definitions are package globals var Module = fx.Module("log", fx.Invoke(func(cfg Config) { Initialize(cfg) @@ -12,12 +14,12 @@ var Module = fx.Module("log", ) // New creates a new logger configuration from provided options. -func New(opts LogOptions) Config { +func New(opts Options) Config { return Config(opts) } -// LogOptions are provided by the CLI. -type LogOptions struct { +// Options are provided by the CLI. +type Options struct { Verbose bool Debug bool Cron bool diff --git a/internal/log/tty_handler.go b/internal/log/tty_handler.go index 81c1fe6..e787de8 100644 --- a/internal/log/tty_handler.go +++ b/internal/log/tty_handler.go @@ -94,6 +94,11 @@ func (h *TTYHandler) Handle(_ context.Context, r slog.Record) error { if a.Key == "bytes" { value = formatBytes(a.Value.Int64()) } + case slog.KindAny, slog.KindBool, slog.KindFloat64, slog.KindString, + slog.KindTime, slog.KindUint64, slog.KindGroup, slog.KindLogValuer: + // Plain string form above is already correct for these kinds. + default: + // Future kinds also use the plain string form. } _, _ = fmt.Fprintf(h.out, " %s%s%s=%s%s%s", @@ -109,26 +114,27 @@ func (h *TTYHandler) Handle(_ context.Context, r slog.Record) error { } // WithAttrs returns a new handler with the given attributes. -func (h *TTYHandler) WithAttrs(attrs []slog.Attr) slog.Handler { +func (h *TTYHandler) WithAttrs(_ []slog.Attr) slog.Handler { return h // Simplified for now } // WithGroup returns a new handler with the given group name. -func (h *TTYHandler) WithGroup(name string) slog.Handler { +func (h *TTYHandler) WithGroup(_ string) slog.Handler { return h // Simplified for now } // formatDuration formats a duration in a human-readable way func formatDuration(d time.Duration) string { - if d < time.Millisecond { + switch { + case d < time.Millisecond: return fmt.Sprintf("%dµs", d.Microseconds()) - } else if d < time.Second { + case d < time.Second: return fmt.Sprintf("%dms", d.Milliseconds()) - } else if d < time.Minute { + case d < time.Minute: return fmt.Sprintf("%.1fs", d.Seconds()) + default: + return d.String() } - - return d.String() } // formatBytes formats bytes in a human-readable way diff --git a/internal/models/models.go b/internal/models/models.go index f16e17a..819402a 100644 --- a/internal/models/models.go +++ b/internal/models/models.go @@ -1,3 +1,5 @@ +// Package models defines shared value types describing files, chunks, +// blobs, and snapshots as they move through the backup pipeline. package models import ( diff --git a/internal/models/models_test.go b/internal/models/models_test.go index 1f44ac9..b5c4cab 100644 --- a/internal/models/models_test.go +++ b/internal/models/models_test.go @@ -1,17 +1,21 @@ -package models +package models_test import ( "testing" "time" + + "sneak.berlin/go/vaultik/internal/models" ) // TestModelsCompilation ensures all model types can be instantiated func TestModelsCompilation(t *testing.T) { + t.Parallel() + // This test primarily serves as a compilation test // to ensure all types are properly defined // Test FileInfo - fi := &FileInfo{ + fi := &models.FileInfo{ Path: "/test/file.txt", MTime: time.Now(), Size: 1024, @@ -21,7 +25,7 @@ func TestModelsCompilation(t *testing.T) { } // Test ChunkInfo - ci := &ChunkInfo{ + ci := &models.ChunkInfo{ Hash: "abc123", Size: 512, Offset: 0, @@ -31,7 +35,7 @@ func TestModelsCompilation(t *testing.T) { } // Test BlobInfo - bi := &BlobInfo{ + bi := &models.BlobInfo{ Hash: "blob123", CreatedAt: time.Now(), Size: 1024, @@ -42,7 +46,7 @@ func TestModelsCompilation(t *testing.T) { } // Test Snapshot - s := &Snapshot{ + s := &models.Snapshot{ ID: "2024-01-01T00:00:00Z", Hostname: "test-host", Version: "1.0.0", diff --git a/internal/pidlock/pidlock.go b/internal/pidlock/pidlock.go index d8baa8c..ea15e77 100644 --- a/internal/pidlock/pidlock.go +++ b/internal/pidlock/pidlock.go @@ -21,6 +21,13 @@ type Lock struct { path string } +const ( + // lockDirPerm is the mode for the lock directory (owner-only). + lockDirPerm = 0o700 + // pidFilePerm is the mode for the PID file (owner-only). + pidFilePerm = 0o600 +) + // Acquire attempts to acquire a PID lock in the specified directory. // If the lock file exists and the process is still running, it returns // ErrAlreadyRunning with details about the existing process. @@ -28,7 +35,8 @@ type Lock struct { // a Lock that must be released with Release(). func Acquire(lockDir string) (*Lock, error) { // Ensure lock directory exists - if err := os.MkdirAll(lockDir, 0700); err != nil { + err := os.MkdirAll(lockDir, lockDirPerm) + if err != nil { return nil, fmt.Errorf("creating lock directory: %w", err) } @@ -46,7 +54,9 @@ func Acquire(lockDir string) (*Lock, error) { // Write our PID pid := os.Getpid() - if err := os.WriteFile(lockPath, []byte(strconv.Itoa(pid)), 0600); err != nil { + + err = os.WriteFile(lockPath, []byte(strconv.Itoa(pid)), pidFilePerm) + if err != nil { return nil, fmt.Errorf("writing PID file: %w", err) } @@ -64,7 +74,7 @@ func (l *Lock) Release() error { existingPID, err := readPIDFile(l.path) if err != nil { // File already gone or unreadable - that's fine - return nil + return nil //nolint:nilerr // unreadable lock file means nothing to release } if existingPID != os.Getpid() { @@ -84,7 +94,7 @@ func (l *Lock) Release() error { // readPIDFile reads and parses the PID from a lock file. func readPIDFile(path string) (int, error) { - data, err := os.ReadFile(path) + data, err := os.ReadFile(path) //nolint:gosec // G304: path is our own lock file if err != nil { return 0, err } diff --git a/internal/pidlock/pidlock_test.go b/internal/pidlock/pidlock_test.go index dce224d..3adc2e0 100644 --- a/internal/pidlock/pidlock_test.go +++ b/internal/pidlock/pidlock_test.go @@ -1,4 +1,4 @@ -package pidlock +package pidlock_test import ( "os" @@ -8,18 +8,22 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "sneak.berlin/go/vaultik/internal/pidlock" ) func TestAcquireAndRelease(t *testing.T) { + t.Parallel() + tmpDir := t.TempDir() // Acquire lock - lock, err := Acquire(tmpDir) + lock, err := pidlock.Acquire(tmpDir) require.NoError(t, err) require.NotNil(t, lock) // Verify PID file exists with our PID - data, err := os.ReadFile(filepath.Join(tmpDir, "vaultik.pid")) + pidPath := filepath.Join(tmpDir, "vaultik.pid") + data, err := os.ReadFile(pidPath) //nolint:gosec // G304: test's own temp file require.NoError(t, err) pid, err := strconv.Atoi(string(data)) require.NoError(t, err) @@ -30,27 +34,31 @@ func TestAcquireAndRelease(t *testing.T) { require.NoError(t, err) // Verify PID file is gone - _, err = os.Stat(filepath.Join(tmpDir, "vaultik.pid")) + _, err = os.Stat(pidPath) assert.True(t, os.IsNotExist(err)) } func TestAcquireBlocksSecondInstance(t *testing.T) { + t.Parallel() + tmpDir := t.TempDir() // Acquire first lock - lock1, err := Acquire(tmpDir) + lock1, err := pidlock.Acquire(tmpDir) require.NoError(t, err) require.NotNil(t, lock1) defer func() { _ = lock1.Release() }() // Try to acquire second lock - should fail - lock2, err := Acquire(tmpDir) - assert.ErrorIs(t, err, ErrAlreadyRunning) + lock2, err := pidlock.Acquire(tmpDir) + require.ErrorIs(t, err, pidlock.ErrAlreadyRunning) assert.Nil(t, lock2) } func TestAcquireWithStaleLock(t *testing.T) { + t.Parallel() + tmpDir := t.TempDir() // Write a stale PID file (PID that doesn't exist) @@ -60,14 +68,14 @@ func TestAcquireWithStaleLock(t *testing.T) { require.NoError(t, err) // Should be able to acquire lock (stale lock is cleaned up) - lock, err := Acquire(tmpDir) + lock, err := pidlock.Acquire(tmpDir) require.NoError(t, err) require.NotNil(t, lock) defer func() { _ = lock.Release() }() // Verify our PID is now in the file - data, err := os.ReadFile(pidPath) + data, err := os.ReadFile(pidPath) //nolint:gosec // G304: test's own temp file require.NoError(t, err) pid, err := strconv.Atoi(string(data)) require.NoError(t, err) @@ -75,9 +83,11 @@ func TestAcquireWithStaleLock(t *testing.T) { } func TestReleaseIsIdempotent(t *testing.T) { + t.Parallel() + tmpDir := t.TempDir() - lock, err := Acquire(tmpDir) + lock, err := pidlock.Acquire(tmpDir) require.NoError(t, err) // Release multiple times - should not error @@ -89,17 +99,21 @@ func TestReleaseIsIdempotent(t *testing.T) { } func TestReleaseNilLock(t *testing.T) { - var lock *Lock + t.Parallel() + + var lock *pidlock.Lock err := lock.Release() - assert.NoError(t, err) + require.NoError(t, err) } func TestAcquireCreatesDirectory(t *testing.T) { + t.Parallel() + tmpDir := t.TempDir() nestedDir := filepath.Join(tmpDir, "nested", "dir") - lock, err := Acquire(nestedDir) + lock, err := pidlock.Acquire(nestedDir) require.NoError(t, err) require.NotNil(t, lock) diff --git a/internal/s3/client.go b/internal/s3/client.go index 1079b8f..f043ba3 100644 --- a/internal/s3/client.go +++ b/internal/s3/client.go @@ -1,3 +1,5 @@ +// Package s3 wraps the AWS S3 SDK with a simplified client for vaultik's +// bucket-and-prefix scoped object operations. package s3 import ( @@ -42,7 +44,7 @@ type Config struct { // Used to suppress SDK warnings about checksums. type nopLogger struct{} -func (nopLogger) Logf(classification logging.Classification, format string, v ...any) {} +func (nopLogger) Logf(_ logging.Classification, _ string, _ ...any) {} // NewClient creates a new S3 client with the provided configuration. // It establishes a connection to the S3-compatible storage service and @@ -105,13 +107,18 @@ type ProgressCallback func(bytesUploaded int64) error // The size parameter must be the exact size of the data to upload. // The progress callback is called periodically with the number of bytes uploaded. // Returns an error if the upload fails. -func (c *Client) PutObjectWithProgress(ctx context.Context, key string, data io.Reader, size int64, progress ProgressCallback) error { +func (c *Client) PutObjectWithProgress( + ctx context.Context, key string, data io.Reader, + size int64, progress ProgressCallback, +) error { fullKey := c.prefix + key + // uploadPartSize is 10MB for better progress granularity. + const uploadPartSize = 10 * 1024 * 1024 + // Create an uploader with the S3 client uploader := manager.NewUploader(c.s3Client, func(u *manager.Uploader) { - // Set part size to 10MB for better progress granularity - u.PartSize = 10 * 1024 * 1024 + u.PartSize = uploadPartSize }) // Create a progress reader that tracks upload progress @@ -241,7 +248,9 @@ type ObjectInfo struct { // listing is complete or an error occurs. If an error occurs, it will be // sent as the last item with the Err field set. The recursive parameter // is currently unused but reserved for future use. -func (c *Client) ListObjectsStream(ctx context.Context, prefix string, recursive bool) <-chan ObjectInfo { +func (c *Client) ListObjectsStream( + ctx context.Context, prefix string, _ bool, +) <-chan ObjectInfo { ch := make(chan ObjectInfo) go func() { diff --git a/internal/s3/client_test.go b/internal/s3/client_test.go index 7ebab27..81f986f 100644 --- a/internal/s3/client_test.go +++ b/internal/s3/client_test.go @@ -9,6 +9,7 @@ import ( "sneak.berlin/go/vaultik/internal/s3" ) +//nolint:paralleltest // test servers share a fixed localhost port func TestClient(t *testing.T) { ts := NewTestServer(t) defer func() { @@ -33,11 +34,21 @@ func TestClient(t *testing.T) { t.Fatalf("failed to create client: %v", err) } - // Test PutObject testKey := "foo/bar.txt" testData := []byte("test data") - err = client.PutObject(ctx, testKey, bytes.NewReader(testData)) + verifyPutGetHead(ctx, t, client, testKey, testData) + verifyListAndDelete(ctx, t, client, testKey) +} + +// verifyPutGetHead uploads an object, reads it back, and checks existence. +func verifyPutGetHead( + ctx context.Context, t *testing.T, client *s3.Client, + testKey string, testData []byte, +) { + t.Helper() + + err := client.PutObject(ctx, testKey, bytes.NewReader(testData)) if err != nil { t.Fatalf("failed to put object: %v", err) } @@ -72,8 +83,15 @@ func TestClient(t *testing.T) { if !exists { t.Error("expected object to exist") } +} + +// verifyListAndDelete lists the object's prefix, deletes it, and checks +// it is gone. +func verifyListAndDelete( + ctx context.Context, t *testing.T, client *s3.Client, testKey string, +) { + t.Helper() - // Test ListObjects keys, err := client.ListObjects(ctx, "foo/") if err != nil { t.Fatalf("failed to list objects: %v", err) @@ -94,7 +112,7 @@ func TestClient(t *testing.T) { } // Verify deletion - exists, err = client.HeadObject(ctx, testKey) + exists, err := client.HeadObject(ctx, testKey) if err != nil { t.Fatalf("failed to head object after deletion: %v", err) } diff --git a/internal/s3/module.go b/internal/s3/module.go index 8839ec7..33b24ba 100644 --- a/internal/s3/module.go +++ b/internal/s3/module.go @@ -10,6 +10,8 @@ import ( // Module exports S3 functionality as an fx module. // It provides automatic dependency injection for the S3 client, // configuring it based on the application's configuration settings. +// +//nolint:gochecknoglobals // fx module definitions are package globals var Module = fx.Module("s3", fx.Provide( provideClient, @@ -32,7 +34,7 @@ func provideClient(lc fx.Lifecycle, cfg *config.Config) (*Client, error) { } lc.Append(fx.Hook{ - OnStop: func(ctx context.Context) error { + OnStop: func(_ context.Context) error { // S3 client doesn't need explicit cleanup return nil }, diff --git a/internal/s3/s3_test.go b/internal/s3/s3_test.go index d21e86a..38dc911 100644 --- a/internal/s3/s3_test.go +++ b/internal/s3/s3_test.go @@ -7,7 +7,6 @@ import ( "fmt" "io" "net/http" - "os" "path/filepath" "testing" "time" @@ -38,13 +37,16 @@ type TestServer struct { logBuf *bytes.Buffer } +// testServerReadHeaderTimeout bounds header reads on the in-process +// test server (gosec G112). +const testServerReadHeaderTimeout = 5 * time.Second + // NewTestServer creates and starts a new test server func NewTestServer(t *testing.T) *TestServer { + t.Helper() + // Create temp directory for any file operations - tempDir, err := os.MkdirTemp("", "vaultik-s3-test-*") - if err != nil { - t.Fatalf("failed to create temp dir: %v", err) - } + tempDir := t.TempDir() // Create in-memory backend backend := s3mem.New() @@ -52,8 +54,9 @@ func NewTestServer(t *testing.T) *TestServer { // Create HTTP server server := &http.Server{ - Addr: "localhost:9999", - Handler: faker.Server(), + Addr: "localhost:9999", + Handler: faker.Server(), + ReadHeaderTimeout: testServerReadHeaderTimeout, } // Start server in background @@ -71,6 +74,14 @@ func NewTestServer(t *testing.T) *TestServer { logBuf := &bytes.Buffer{} // Create S3 client with custom logger + logFn := func(classification logging.Classification, format string, v ...any) { + // Capture logs to buffer instead of stdout + fmt.Fprintf(logBuf, "SDK %s %s %s\n", + time.Now().Format("2006/01/02 15:04:05"), + string(classification), + fmt.Sprintf(format, v...)) + } + cfg, err := config.LoadDefaultConfig(context.Background(), config.WithRegion(testRegion), config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider( @@ -78,14 +89,9 @@ func NewTestServer(t *testing.T) *TestServer { testSecretKey, "", )), - config.WithClientLogMode(aws.LogRetries|aws.LogRequestWithBody|aws.LogResponseWithBody), - config.WithLogger(logging.LoggerFunc(func(classification logging.Classification, format string, v ...any) { - // Capture logs to buffer instead of stdout - fmt.Fprintf(logBuf, "SDK %s %s %s\n", - time.Now().Format("2006/01/02 15:04:05"), - string(classification), - fmt.Sprintf(format, v...)) - })), + config.WithClientLogMode( + aws.LogRetries|aws.LogRequestWithBody|aws.LogResponseWithBody), + config.WithLogger(logging.LoggerFunc(logFn)), ) if err != nil { t.Fatalf("failed to create AWS config: %v", err) @@ -122,17 +128,13 @@ func NewTestServer(t *testing.T) *TestServer { return ts } -// Cleanup shuts down the server and removes temp directory +// Cleanup shuts down the server. The temp directory is removed +// automatically by t.TempDir. func (ts *TestServer) Cleanup() error { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - err := ts.server.Shutdown(ctx) - if err != nil { - return err - } - - return os.RemoveAll(ts.tempDir) + return ts.server.Shutdown(ctx) } // Client returns the S3 client configured for the test server @@ -141,6 +143,8 @@ func (ts *TestServer) Client() *s3.Client { } // TestBasicS3Operations tests basic store and retrieve operations +// +//nolint:paralleltest // test servers share a fixed localhost port func TestBasicS3Operations(t *testing.T) { ts := NewTestServer(t) defer func() { @@ -194,6 +198,8 @@ func TestBasicS3Operations(t *testing.T) { } // TestBlobOperations tests blob storage patterns for vaultik +// +//nolint:paralleltest // test servers share a fixed localhost port func TestBlobOperations(t *testing.T) { ts := NewTestServer(t) defer func() { @@ -258,6 +264,8 @@ func TestBlobOperations(t *testing.T) { } // TestMetadataOperations tests metadata storage patterns +// +//nolint:paralleltest // test servers share a fixed localhost port func TestMetadataOperations(t *testing.T) { ts := NewTestServer(t) defer func() { @@ -287,7 +295,8 @@ func TestMetadataOperations(t *testing.T) { // Store manifest manifestKey := filepath.Join("metadata", snapshotID+".manifest.json.zst") - manifestData := []byte(`{"snapshot_id":"2024-01-01T12:00:00Z","blob_hashes":["hash1","hash2"]}`) + manifestData := []byte(`{"snapshot_id":"2024-01-01T12:00:00Z",` + + `"blob_hashes":["hash1","hash2"]}`) _, err = client.PutObject(ctx, &s3.PutObjectInput{ Bucket: aws.String(testBucket), diff --git a/internal/snapshot/backup_test.go b/internal/snapshot/backup_test.go index 44d49a3..a6e2a80 100644 --- a/internal/snapshot/backup_test.go +++ b/internal/snapshot/backup_test.go @@ -1,4 +1,4 @@ -package snapshot +package snapshot_test import ( "context" @@ -19,6 +19,12 @@ import ( "sneak.berlin/go/vaultik/internal/types" ) +// errBlobNotFound is returned by the mock S3 client for unknown hashes. +var errBlobNotFound = errors.New("blob not found") + +// testFile1Name is the shared fixture filename used across backup tests. +const testFile1Name = "file1.txt" + // MockS3Client is a mock implementation of S3 operations for testing type MockS3Client struct { storage map[string][]byte @@ -30,39 +36,149 @@ func NewMockS3Client() *MockS3Client { } } -func (m *MockS3Client) PutBlob(ctx context.Context, hash string, data []byte) error { +func (m *MockS3Client) PutBlob(_ context.Context, hash string, data []byte) error { m.storage[hash] = data return nil } -func (m *MockS3Client) GetBlob(ctx context.Context, hash string) ([]byte, error) { +func (m *MockS3Client) GetBlob(_ context.Context, hash string) ([]byte, error) { data, ok := m.storage[hash] if !ok { - return nil, fmt.Errorf("blob not found: %s", hash) + return nil, fmt.Errorf("%w: %s", errBlobNotFound, hash) } return data, nil } -func (m *MockS3Client) BlobExists(ctx context.Context, hash string) (bool, error) { +func (m *MockS3Client) BlobExists(_ context.Context, hash string) (bool, error) { _, ok := m.storage[hash] return ok, nil } -func (m *MockS3Client) CreateBucket(ctx context.Context, bucket string) error { +func (m *MockS3Client) CreateBucket(_ context.Context, _ string) error { return nil } +// verifyBackupFiles checks the file records created by a backup against +// the fixture filesystem. +func verifyBackupFiles( + ctx context.Context, + t *testing.T, + repos *database.Repositories, + testFS fstest.MapFS, +) { + t.Helper() + + files, err := repos.Files.ListByPrefix(ctx, "") + if err != nil { + t.Fatalf("Failed to list files: %v", err) + } + + expectedFiles := map[string]bool{ + testFile1Name: true, + "dir1/file2.txt": true, + "dir1/subdir/file3.txt": true, + "largefile.bin": true, + } + + if len(files) != len(expectedFiles) { + t.Errorf("Expected %d files, got %d", len(expectedFiles), len(files)) + } + + for _, file := range files { + if !expectedFiles[file.Path.String()] { + t.Errorf("Unexpected file in database: %s", file.Path) + } + + delete(expectedFiles, file.Path.String()) + + // Verify file metadata + fsFile := testFS[file.Path.String()] + if fsFile == nil { + t.Errorf("File %s not found in test filesystem", file.Path) + + continue + } + + if file.Size != int64(len(fsFile.Data)) { + t.Errorf("File %s: expected size %d, got %d", + file.Path, len(fsFile.Data), file.Size) + } + + if file.Mode != uint32(fsFile.Mode) { + t.Errorf("File %s: expected mode %o, got %o", + file.Path, fsFile.Mode, file.Mode) + } + } + + if len(expectedFiles) > 0 { + t.Errorf("Files not found in database: %v", expectedFiles) + } +} + +// verifyBackupChunksAndBlobs checks that chunking produced the expected +// records and every referenced blob exists in the mock S3 store. +func verifyBackupChunksAndBlobs( + ctx context.Context, + t *testing.T, + repos *database.Repositories, + s3Client *MockS3Client, + snapshotID string, +) { + t.Helper() + + chunks, err := repos.Chunks.List(ctx) + if err != nil { + t.Fatalf("Failed to list chunks: %v", err) + } + + if len(chunks) == 0 { + t.Error("No chunks found in database") + } + + // The large file should create 10 chunks (10MB / 1MB chunk size) + // Plus the small files + minExpectedChunks := 10 + 3 + if len(chunks) < minExpectedChunks { + t.Errorf("Expected at least %d chunks, got %d", minExpectedChunks, len(chunks)) + } + + // Verify at least one blob was created and uploaded + // We can't list blobs directly, but we can check via snapshot blobs + blobHashes, err := repos.Snapshots.GetBlobHashes(ctx, snapshotID) + if err != nil { + t.Fatalf("Failed to get blob hashes: %v", err) + } + + if len(blobHashes) == 0 { + t.Error("Expected at least one blob to be created") + } + + for _, blobHash := range blobHashes { + // Check blob exists in mock S3 + exists, err := s3Client.BlobExists(ctx, blobHash) + if err != nil { + t.Errorf("Failed to check blob %s: %v", blobHash, err) + } + + if !exists { + t.Errorf("Blob %s not found in S3", blobHash) + } + } +} + func TestBackupWithInMemoryFS(t *testing.T) { + t.Parallel() + // Create a temporary directory for the database tempDir := t.TempDir() dbPath := filepath.Join(tempDir, "test.db") // Create test filesystem testFS := fstest.MapFS{ - "file1.txt": &fstest.MapFile{ + testFile1Name: &fstest.MapFile{ Data: []byte("Hello, World!"), Mode: 0644, ModTime: time.Now(), @@ -129,100 +245,21 @@ func TestBackupWithInMemoryFS(t *testing.T) { t.Error("Expected snapshot to have files") } - // Verify files in database - files, err := repos.Files.ListByPrefix(ctx, "") - if err != nil { - t.Fatalf("Failed to list files: %v", err) - } - - expectedFiles := map[string]bool{ - "file1.txt": true, - "dir1/file2.txt": true, - "dir1/subdir/file3.txt": true, - "largefile.bin": true, - } - - if len(files) != len(expectedFiles) { - t.Errorf("Expected %d files, got %d", len(expectedFiles), len(files)) - } - - for _, file := range files { - if !expectedFiles[file.Path.String()] { - t.Errorf("Unexpected file in database: %s", file.Path) - } - - delete(expectedFiles, file.Path.String()) - - // Verify file metadata - fsFile := testFS[file.Path.String()] - if fsFile == nil { - t.Errorf("File %s not found in test filesystem", file.Path) - - continue - } - - if file.Size != int64(len(fsFile.Data)) { - t.Errorf("File %s: expected size %d, got %d", file.Path, len(fsFile.Data), file.Size) - } - - if file.Mode != uint32(fsFile.Mode) { - t.Errorf("File %s: expected mode %o, got %o", file.Path, fsFile.Mode, file.Mode) - } - } - - if len(expectedFiles) > 0 { - t.Errorf("Files not found in database: %v", expectedFiles) - } - - // Verify chunks - chunks, err := repos.Chunks.List(ctx) - if err != nil { - t.Fatalf("Failed to list chunks: %v", err) - } - - if len(chunks) == 0 { - t.Error("No chunks found in database") - } - - // The large file should create 10 chunks (10MB / 1MB chunk size) - // Plus the small files - minExpectedChunks := 10 + 3 - if len(chunks) < minExpectedChunks { - t.Errorf("Expected at least %d chunks, got %d", minExpectedChunks, len(chunks)) - } - - // Verify at least one blob was created and uploaded - // We can't list blobs directly, but we can check via snapshot blobs - blobHashes, err := repos.Snapshots.GetBlobHashes(ctx, snapshotID) - if err != nil { - t.Fatalf("Failed to get blob hashes: %v", err) - } - - if len(blobHashes) == 0 { - t.Error("Expected at least one blob to be created") - } - - for _, blobHash := range blobHashes { - // Check blob exists in mock S3 - exists, err := s3Client.BlobExists(ctx, blobHash) - if err != nil { - t.Errorf("Failed to check blob %s: %v", blobHash, err) - } - - if !exists { - t.Errorf("Blob %s not found in S3", blobHash) - } - } + // Verify files, chunks, and blob records + verifyBackupFiles(ctx, t, repos, testFS) + verifyBackupChunksAndBlobs(ctx, t, repos, s3Client, snapshotID) } func TestBackupDeduplication(t *testing.T) { + t.Parallel() + // Create a temporary directory for the database tempDir := t.TempDir() dbPath := filepath.Join(tempDir, "test.db") // Create test filesystem with duplicate content testFS := fstest.MapFS{ - "file1.txt": &fstest.MapFile{ + testFile1Name: &fstest.MapFile{ Data: []byte("Duplicate content"), Mode: 0644, ModTime: time.Now(), @@ -290,7 +327,8 @@ func TestBackupDeduplication(t *testing.T) { // The duplicate content chunk should be referenced by 2 files if chunk.Size == int64(len("Duplicate content")) && len(files) != 2 { - t.Errorf("Expected duplicate chunk to be referenced by 2 files, got %d", len(files)) + t.Errorf("Expected duplicate chunk to be referenced by 2 files, got %d", + len(files)) } } } @@ -304,8 +342,19 @@ type BackupEngine struct { } } +// backupCounters accumulates statistics across a test backup run. +type backupCounters struct { + fileCount int64 + chunkCount int64 + blobCount int64 + totalSize int64 + blobSize int64 +} + // Backup performs a backup of the given filesystem -func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (string, error) { +func (b *BackupEngine) Backup( + ctx context.Context, fsys fs.FS, root string, +) (string, error) { // Create a new snapshot hostname, _ := os.Hostname() snapshotID := time.Now().Format(time.RFC3339) @@ -325,8 +374,7 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str return "", err } - // Track counters - var fileCount, chunkCount, blobCount, totalSize, blobSize int64 + counters := &backupCounters{} // Track which chunks we've seen to handle deduplication processedChunks := make(map[string]bool) @@ -354,122 +402,170 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str return nil } - // Create file record in a short transaction - file := &database.File{ - Path: types.FilePath(path), - Size: info.Size(), - Mode: uint32(info.Mode()), - MTime: info.ModTime(), - UID: 1000, // Default UID for test - GID: 1000, // Default GID for test - } - - err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { - return b.repos.Files.Create(ctx, tx, file) - }) - if err != nil { - return err - } - - fileCount++ - totalSize += info.Size() - - // Read and process file in chunks - f, err := fsys.Open(path) - if err != nil { - return err - } - defer func() { - err := f.Close() - if err != nil { - // Log but don't fail since we're already in an error path potentially - fmt.Fprintf(os.Stderr, "Failed to close file: %v\n", err) - } - }() - - // Process file in chunks - chunkIndex := 0 - buffer := make([]byte, defaultChunkSize) - - for { - n, err := f.Read(buffer) - if err != nil && !errors.Is(err, io.EOF) { - return err - } - - if n == 0 { - break - } - - chunkData := buffer[:n] - chunkHash := calculateHash(chunkData) - - // Check if chunk already exists (outside of transaction) - existingChunk, _ := b.repos.Chunks.GetByHash(ctx, chunkHash) - if existingChunk == nil { - // Create new chunk in a short transaction - err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { - chunk := &database.Chunk{ - ChunkHash: types.ChunkHash(chunkHash), - Size: int64(n), - } - - return b.repos.Chunks.Create(ctx, tx, chunk) - }) - if err != nil { - return err - } - - processedChunks[chunkHash] = true - } - - // Create file-chunk mapping in a short transaction - err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { - fileChunk := &database.FileChunk{ - FileID: file.ID, - Idx: chunkIndex, - ChunkHash: types.ChunkHash(chunkHash), - } - - return b.repos.FileChunks.Create(ctx, tx, fileChunk) - }) - if err != nil { - return err - } - - // Create chunk-file mapping in a short transaction - err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { - chunkFile := &database.ChunkFile{ - ChunkHash: types.ChunkHash(chunkHash), - FileID: file.ID, - FileOffset: int64(chunkIndex * defaultChunkSize), - Length: int64(n), - } - - return b.repos.ChunkFiles.Create(ctx, tx, chunkFile) - }) - if err != nil { - return err - } - - chunkIndex++ - } - - return nil + return b.backupOneFile(ctx, fsys, path, info, processedChunks, counters) }) if err != nil { return "", err } // After all files are processed, create blobs for new chunks + err = b.createBlobsForChunks(ctx, snapshotID, processedChunks, counters) + if err != nil { + return "", err + } + + // Update snapshot with final counts + err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { + return b.repos.Snapshots.UpdateCounts(ctx, tx, snapshotID, + counters.fileCount, counters.chunkCount, counters.blobCount, + counters.totalSize, counters.blobSize) + }) + if err != nil { + return "", err + } + + return snapshotID, nil +} + +// backupOneFile records a single regular file and its chunks. +func (b *BackupEngine) backupOneFile( + ctx context.Context, + fsys fs.FS, + path string, + info fs.FileInfo, + processedChunks map[string]bool, + counters *backupCounters, +) error { + // Create file record in a short transaction + file := &database.File{ + Path: types.FilePath(path), + Size: info.Size(), + Mode: uint32(info.Mode()), + MTime: info.ModTime(), + UID: 1000, // Default UID for test + GID: 1000, // Default GID for test + } + + err := b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { + return b.repos.Files.Create(ctx, tx, file) + }) + if err != nil { + return err + } + + counters.fileCount++ + counters.totalSize += info.Size() + + // Read and process file in chunks + f, err := fsys.Open(path) + if err != nil { + return err + } + defer func() { + err := f.Close() + if err != nil { + // Log but don't fail since we're already in an error path potentially + fmt.Fprintf(os.Stderr, "Failed to close file: %v\n", err) + } + }() + + // Process file in chunks + chunkIndex := 0 + buffer := make([]byte, defaultChunkSize) + + for { + n, err := f.Read(buffer) + if err != nil && !errors.Is(err, io.EOF) { + return err + } + + if n == 0 { + break + } + + err = b.recordChunk(ctx, file, chunkIndex, buffer[:n], processedChunks) + if err != nil { + return err + } + + chunkIndex++ + } + + return nil +} + +// recordChunk creates the chunk record (if new) and its file associations. +func (b *BackupEngine) recordChunk( + ctx context.Context, + file *database.File, + chunkIndex int, + chunkData []byte, + processedChunks map[string]bool, +) error { + chunkHash := calculateHash(chunkData) + + // Check if chunk already exists (outside of transaction) + existingChunk, _ := b.repos.Chunks.GetByHash(ctx, chunkHash) + if existingChunk == nil { + // Create new chunk in a short transaction + err := b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { + chunk := &database.Chunk{ + ChunkHash: types.ChunkHash(chunkHash), + Size: int64(len(chunkData)), + } + + return b.repos.Chunks.Create(ctx, tx, chunk) + }) + if err != nil { + return err + } + + processedChunks[chunkHash] = true + } + + // Create file-chunk mapping in a short transaction + err := b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { + fileChunk := &database.FileChunk{ + FileID: file.ID, + Idx: chunkIndex, + ChunkHash: types.ChunkHash(chunkHash), + } + + return b.repos.FileChunks.Create(ctx, tx, fileChunk) + }) + if err != nil { + return err + } + + // Create chunk-file mapping in a short transaction + return b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { + chunkFile := &database.ChunkFile{ + ChunkHash: types.ChunkHash(chunkHash), + FileID: file.ID, + FileOffset: int64(chunkIndex * defaultChunkSize), + Length: int64(len(chunkData)), + } + + return b.repos.ChunkFiles.Create(ctx, tx, chunkFile) + }) +} + +// createBlobsForChunks uploads one blob per new chunk and records the blob +// metadata and snapshot association. +func (b *BackupEngine) createBlobsForChunks( + ctx context.Context, + snapshotID string, + processedChunks map[string]bool, + counters *backupCounters, +) error { for chunkHash := range processedChunks { // Get chunk data (outside of transaction) chunk, err := b.repos.Chunks.GetByHash(ctx, chunkHash) if err != nil { - return "", err + return err } - chunkCount++ + counters.chunkCount++ // In a real system, blobs would contain multiple chunks and be encrypted // For testing, we'll create a blob with a "blob-" prefix to differentiate @@ -481,7 +577,7 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str // Upload to S3 as a blob err = b.s3Client.PutBlob(ctx, blobHash, dummyData) if err != nil { - return "", err + return err } // Create blob entry in a short transaction @@ -497,11 +593,11 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str return b.repos.Blobs.Create(ctx, tx, blob) }) if err != nil { - return "", err + return err } - blobCount++ - blobSize += chunk.Size + counters.blobCount++ + counters.blobSize += chunk.Size // Create blob-chunk mapping in a short transaction err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { @@ -515,27 +611,20 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str return b.repos.BlobChunks.Create(ctx, tx, blobChunk) }) if err != nil { - return "", err + return err } // Add blob to snapshot in a short transaction err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { - return b.repos.Snapshots.AddBlob(ctx, tx, snapshotID, blobID, types.BlobHash(blobHash)) + return b.repos.Snapshots.AddBlob(ctx, tx, snapshotID, blobID, + types.BlobHash(blobHash)) }) if err != nil { - return "", err + return err } } - // Update snapshot with final counts - err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { - return b.repos.Snapshots.UpdateCounts(ctx, tx, snapshotID, fileCount, chunkCount, blobCount, totalSize, blobSize) - }) - if err != nil { - return "", err - } - - return snapshotID, nil + return nil } func calculateHash(data []byte) string { diff --git a/internal/snapshot/exclude_test.go b/internal/snapshot/exclude_test.go index f908250..9870018 100644 --- a/internal/snapshot/exclude_test.go +++ b/internal/snapshot/exclude_test.go @@ -10,16 +10,15 @@ import ( "github.com/spf13/afero" "github.com/stretchr/testify/require" "sneak.berlin/go/vaultik/internal/database" - "sneak.berlin/go/vaultik/internal/log" "sneak.berlin/go/vaultik/internal/snapshot" "sneak.berlin/go/vaultik/internal/types" ) -func setupExcludeTestFS(t *testing.T) afero.Fs { +func setupExcludeTestFS(t *testing.T) *afero.MemMapFs { t.Helper() // Create in-memory filesystem - fs := afero.NewMemMapFs() + fs := &afero.MemMapFs{} // Create test directory structure: // /backup/ @@ -77,12 +76,11 @@ func setupExcludeTestFS(t *testing.T) afero.Fs { return fs } -func createTestScanner(t *testing.T, fs afero.Fs, excludePatterns []string) (*snapshot.Scanner, *database.Repositories, func()) { +func createTestScanner( + t *testing.T, fs afero.Fs, excludePatterns []string, +) (*snapshot.Scanner, *database.Repositories, func()) { t.Helper() - // Initialize logger - log.Initialize(log.Config{}) - // Create test database db, err := database.NewTestDB() require.NoError(t, err) @@ -95,8 +93,9 @@ func createTestScanner(t *testing.T, fs afero.Fs, excludePatterns []string) (*sn Repositories: repos, MaxBlobSize: 1024 * 1024, CompressionLevel: 3, - AgeRecipients: []string{"age1ql3z7hjy54pw3hyww5ayyfg7zqgvc7w3j2elw8zmrj2kg5sfn9aqmcac8p"}, - Exclude: excludePatterns, + AgeRecipients: []string{ + "age1ql3z7hjy54pw3hyww5ayyfg7zqgvc7w3j2elw8zmrj2kg5sfn9aqmcac8p"}, + Exclude: excludePatterns, }) cleanup := func() { @@ -106,14 +105,16 @@ func createTestScanner(t *testing.T, fs afero.Fs, excludePatterns []string) (*sn return scanner, repos, cleanup } -func createSnapshotRecord(t *testing.T, ctx context.Context, repos *database.Repositories, snapshotID string) { +func createSnapshotRecord( + ctx context.Context, t *testing.T, repos *database.Repositories, snapshotID string, +) { t.Helper() err := repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { snap := &database.Snapshot{ ID: types.SnapshotID(snapshotID), - Hostname: "test-host", - VaultikVersion: "test", + Hostname: testHost, + VaultikVersion: testVersion, StartedAt: time.Now(), CompletedAt: nil, FileCount: 0, @@ -130,6 +131,8 @@ func createSnapshotRecord(t *testing.T, ctx context.Context, repos *database.Rep } func TestExcludePatterns_ExcludeGitDirectory(t *testing.T) { + t.Parallel() + fs := setupExcludeTestFS(t) scanner, repos, cleanup := createTestScanner(t, fs, []string{".git"}) @@ -138,13 +141,14 @@ func TestExcludePatterns_ExcludeGitDirectory(t *testing.T) { require.NotNil(t, scanner) ctx := context.Background() - createSnapshotRecord(t, ctx, repos, "test-snapshot") + createSnapshotRecord(ctx, t, repos, "test-snapshot") result, err := scanner.Scan(ctx, "/backup", "test-snapshot") require.NoError(t, err) // Should have scanned files but NOT .git directory contents - // Expected: file1.txt, file2.log, src/main.go, src/test.go, node_modules/package/index.js, + // Expected: file1.txt, file2.log, src/main.go, src/test.go, + // node_modules/package/index.js, // cache/temp.dat, build/output.bin, docs/readme.md, .DS_Store, thumbs.db, // src/.hidden, important.log.bak // Excluded: .git/config, .git/objects/pack/data.pack @@ -152,6 +156,8 @@ func TestExcludePatterns_ExcludeGitDirectory(t *testing.T) { } func TestExcludePatterns_ExcludeByExtension(t *testing.T) { + t.Parallel() + fs := setupExcludeTestFS(t) scanner, repos, cleanup := createTestScanner(t, fs, []string{"*.log"}) @@ -160,7 +166,7 @@ func TestExcludePatterns_ExcludeByExtension(t *testing.T) { require.NotNil(t, scanner) ctx := context.Background() - createSnapshotRecord(t, ctx, repos, "test-snapshot") + createSnapshotRecord(ctx, t, repos, "test-snapshot") result, err := scanner.Scan(ctx, "/backup", "test-snapshot") require.NoError(t, err) @@ -171,6 +177,8 @@ func TestExcludePatterns_ExcludeByExtension(t *testing.T) { } func TestExcludePatterns_ExcludeNodeModules(t *testing.T) { + t.Parallel() + fs := setupExcludeTestFS(t) scanner, repos, cleanup := createTestScanner(t, fs, []string{"node_modules"}) @@ -179,7 +187,7 @@ func TestExcludePatterns_ExcludeNodeModules(t *testing.T) { require.NotNil(t, scanner) ctx := context.Background() - createSnapshotRecord(t, ctx, repos, "test-snapshot") + createSnapshotRecord(ctx, t, repos, "test-snapshot") result, err := scanner.Scan(ctx, "/backup", "test-snapshot") require.NoError(t, err) @@ -190,25 +198,32 @@ func TestExcludePatterns_ExcludeNodeModules(t *testing.T) { } func TestExcludePatterns_MultiplePatterns(t *testing.T) { + t.Parallel() + fs := setupExcludeTestFS(t) - scanner, repos, cleanup := createTestScanner(t, fs, []string{".git", "node_modules", "*.log", ".DS_Store", "thumbs.db", "cache", "build"}) + scanner, repos, cleanup := createTestScanner(t, fs, []string{ + ".git", "node_modules", "*.log", ".DS_Store", "thumbs.db", "cache", "build"}) defer cleanup() require.NotNil(t, scanner) ctx := context.Background() - createSnapshotRecord(t, ctx, repos, "test-snapshot") + createSnapshotRecord(ctx, t, repos, "test-snapshot") result, err := scanner.Scan(ctx, "/backup", "test-snapshot") require.NoError(t, err) - // Should only have: file1.txt, src/main.go, src/test.go, docs/readme.md, src/.hidden, important.log.bak - // Excluded: .git/*, node_modules/*, *.log (file2.log), .DS_Store, thumbs.db, cache/*, build/* + // Should only have: file1.txt, src/main.go, src/test.go, docs/readme.md, + // src/.hidden, important.log.bak + // Excluded: .git/*, node_modules/*, *.log (file2.log), .DS_Store, + // thumbs.db, cache/*, build/* require.Equal(t, 6, result.FilesScanned, "Should exclude multiple patterns") } func TestExcludePatterns_NoExclusions(t *testing.T) { + t.Parallel() + fs := setupExcludeTestFS(t) scanner, repos, cleanup := createTestScanner(t, fs, []string{}) @@ -217,7 +232,7 @@ func TestExcludePatterns_NoExclusions(t *testing.T) { require.NotNil(t, scanner) ctx := context.Background() - createSnapshotRecord(t, ctx, repos, "test-snapshot") + createSnapshotRecord(ctx, t, repos, "test-snapshot") result, err := scanner.Scan(ctx, "/backup", "test-snapshot") require.NoError(t, err) @@ -227,6 +242,8 @@ func TestExcludePatterns_NoExclusions(t *testing.T) { } func TestExcludePatterns_ExcludeHiddenFiles(t *testing.T) { + t.Parallel() + fs := setupExcludeTestFS(t) scanner, repos, cleanup := createTestScanner(t, fs, []string{".*"}) @@ -235,17 +252,21 @@ func TestExcludePatterns_ExcludeHiddenFiles(t *testing.T) { require.NotNil(t, scanner) ctx := context.Background() - createSnapshotRecord(t, ctx, repos, "test-snapshot") + createSnapshotRecord(ctx, t, repos, "test-snapshot") result, err := scanner.Scan(ctx, "/backup", "test-snapshot") require.NoError(t, err) // Should exclude: .git/*, .DS_Store, src/.hidden - // Total files: 14, excluded: 4 (.git/config, .git/objects/pack/data.pack, .DS_Store, src/.hidden) - require.Equal(t, 10, result.FilesScanned, "Should exclude hidden files and directories") + // Total files: 14, excluded: 4 (.git/config, + // .git/objects/pack/data.pack, .DS_Store, src/.hidden) + require.Equal(t, 10, result.FilesScanned, + "Should exclude hidden files and directories") } func TestExcludePatterns_DoubleStarGlob(t *testing.T) { + t.Parallel() + fs := setupExcludeTestFS(t) scanner, repos, cleanup := createTestScanner(t, fs, []string{"**/*.pack"}) @@ -254,7 +275,7 @@ func TestExcludePatterns_DoubleStarGlob(t *testing.T) { require.NotNil(t, scanner) ctx := context.Background() - createSnapshotRecord(t, ctx, repos, "test-snapshot") + createSnapshotRecord(ctx, t, repos, "test-snapshot") result, err := scanner.Scan(ctx, "/backup", "test-snapshot") require.NoError(t, err) @@ -265,6 +286,8 @@ func TestExcludePatterns_DoubleStarGlob(t *testing.T) { } func TestExcludePatterns_ExactFileName(t *testing.T) { + t.Parallel() + fs := setupExcludeTestFS(t) scanner, repos, cleanup := createTestScanner(t, fs, []string{"thumbs.db", ".DS_Store"}) @@ -273,7 +296,7 @@ func TestExcludePatterns_ExactFileName(t *testing.T) { require.NotNil(t, scanner) ctx := context.Background() - createSnapshotRecord(t, ctx, repos, "test-snapshot") + createSnapshotRecord(ctx, t, repos, "test-snapshot") result, err := scanner.Scan(ctx, "/backup", "test-snapshot") require.NoError(t, err) @@ -284,6 +307,8 @@ func TestExcludePatterns_ExactFileName(t *testing.T) { } func TestExcludePatterns_CaseSensitive(t *testing.T) { + t.Parallel() + // Pattern matching should be case-sensitive fs := setupExcludeTestFS(t) @@ -293,7 +318,7 @@ func TestExcludePatterns_CaseSensitive(t *testing.T) { require.NotNil(t, scanner) ctx := context.Background() - createSnapshotRecord(t, ctx, repos, "test-snapshot") + createSnapshotRecord(ctx, t, repos, "test-snapshot") result, err := scanner.Scan(ctx, "/backup", "test-snapshot") require.NoError(t, err) @@ -304,6 +329,8 @@ func TestExcludePatterns_CaseSensitive(t *testing.T) { } func TestExcludePatterns_DirectoryWithTrailingSlash(t *testing.T) { + t.Parallel() + fs := setupExcludeTestFS(t) // Some users might add trailing slashes to directory patterns scanner, repos, cleanup := createTestScanner(t, fs, []string{"cache/", "build/"}) @@ -312,17 +339,20 @@ func TestExcludePatterns_DirectoryWithTrailingSlash(t *testing.T) { require.NotNil(t, scanner) ctx := context.Background() - createSnapshotRecord(t, ctx, repos, "test-snapshot") + createSnapshotRecord(ctx, t, repos, "test-snapshot") result, err := scanner.Scan(ctx, "/backup", "test-snapshot") require.NoError(t, err) // Should exclude cache/temp.dat and build/output.bin // Total files: 14, excluded: 2 - require.Equal(t, 12, result.FilesScanned, "Should handle directory patterns with trailing slashes") + require.Equal(t, 12, result.FilesScanned, + "Should handle directory patterns with trailing slashes") } func TestExcludePatterns_PatternInSubdirectory(t *testing.T) { + t.Parallel() + fs := setupExcludeTestFS(t) // Exclude .hidden file specifically in src directory scanner, repos, cleanup := createTestScanner(t, fs, []string{"src/.hidden"}) @@ -331,7 +361,7 @@ func TestExcludePatterns_PatternInSubdirectory(t *testing.T) { require.NotNil(t, scanner) ctx := context.Background() - createSnapshotRecord(t, ctx, repos, "test-snapshot") + createSnapshotRecord(ctx, t, repos, "test-snapshot") result, err := scanner.Scan(ctx, "/backup", "test-snapshot") require.NoError(t, err) @@ -350,13 +380,14 @@ func TestExcludePatterns_PatternInSubdirectory(t *testing.T) { // file.txt (should be excluded with /projectname) // otherproject/ // projectname/ -// file.txt (should NOT be excluded with /projectname, only with projectname) +// file.txt (should NOT be excluded with /projectname, +// only with projectname) // src/ // file.go -func setupAnchoredTestFS(t *testing.T) afero.Fs { +func setupAnchoredTestFS(t *testing.T) *afero.MemMapFs { t.Helper() - fs := afero.NewMemMapFs() + fs := &afero.MemMapFs{} files := map[string]string{ "/backup/projectname/file.txt": "root project file", @@ -381,6 +412,8 @@ func setupAnchoredTestFS(t *testing.T) afero.Fs { } func TestExcludePatterns_AnchoredPattern(t *testing.T) { + t.Parallel() + // Pattern starting with / should only match from root of source dir fs := setupAnchoredTestFS(t) @@ -390,7 +423,7 @@ func TestExcludePatterns_AnchoredPattern(t *testing.T) { require.NotNil(t, scanner) ctx := context.Background() - createSnapshotRecord(t, ctx, repos, "test-snapshot") + createSnapshotRecord(ctx, t, repos, "test-snapshot") result, err := scanner.Scan(ctx, "/backup", "test-snapshot") require.NoError(t, err) @@ -398,10 +431,13 @@ func TestExcludePatterns_AnchoredPattern(t *testing.T) { // /projectname should ONLY exclude /backup/projectname/file.txt (1 file) // /backup/otherproject/projectname/file.txt should NOT be excluded // Total files: 4, excluded: 1 - require.Equal(t, 3, result.FilesScanned, "Anchored pattern /projectname should only match at root of source dir") + require.Equal(t, 3, result.FilesScanned, + "Anchored pattern /projectname should only match at root of source dir") } func TestExcludePatterns_UnanchoredPattern(t *testing.T) { + t.Parallel() + // Pattern without leading / should match anywhere in path fs := setupAnchoredTestFS(t) @@ -411,7 +447,7 @@ func TestExcludePatterns_UnanchoredPattern(t *testing.T) { require.NotNil(t, scanner) ctx := context.Background() - createSnapshotRecord(t, ctx, repos, "test-snapshot") + createSnapshotRecord(ctx, t, repos, "test-snapshot") result, err := scanner.Scan(ctx, "/backup", "test-snapshot") require.NoError(t, err) @@ -420,10 +456,13 @@ func TestExcludePatterns_UnanchoredPattern(t *testing.T) { // - /backup/projectname/file.txt // - /backup/otherproject/projectname/file.txt // Total files: 4, excluded: 2 - require.Equal(t, 2, result.FilesScanned, "Unanchored pattern should match anywhere in path") + require.Equal(t, 2, result.FilesScanned, + "Unanchored pattern should match anywhere in path") } func TestExcludePatterns_AnchoredPatternWithGlob(t *testing.T) { + t.Parallel() + // Anchored pattern with glob fs := setupAnchoredTestFS(t) @@ -433,7 +472,7 @@ func TestExcludePatterns_AnchoredPatternWithGlob(t *testing.T) { require.NotNil(t, scanner) ctx := context.Background() - createSnapshotRecord(t, ctx, repos, "test-snapshot") + createSnapshotRecord(ctx, t, repos, "test-snapshot") result, err := scanner.Scan(ctx, "/backup", "test-snapshot") require.NoError(t, err) @@ -444,6 +483,8 @@ func TestExcludePatterns_AnchoredPatternWithGlob(t *testing.T) { } func TestExcludePatterns_AnchoredPatternFile(t *testing.T) { + t.Parallel() + // Anchored pattern for exact file at root fs := setupAnchoredTestFS(t) @@ -453,7 +494,7 @@ func TestExcludePatterns_AnchoredPatternFile(t *testing.T) { require.NotNil(t, scanner) ctx := context.Background() - createSnapshotRecord(t, ctx, repos, "test-snapshot") + createSnapshotRecord(ctx, t, repos, "test-snapshot") result, err := scanner.Scan(ctx, "/backup", "test-snapshot") require.NoError(t, err) @@ -461,10 +502,13 @@ func TestExcludePatterns_AnchoredPatternFile(t *testing.T) { // /file.txt should ONLY exclude /backup/file.txt // NOT /backup/projectname/file.txt or /backup/otherproject/projectname/file.txt // Total files: 4, excluded: 1 - require.Equal(t, 3, result.FilesScanned, "Anchored pattern for file should only match at root") + require.Equal(t, 3, result.FilesScanned, + "Anchored pattern for file should only match at root") } func TestExcludePatterns_UnanchoredPatternFile(t *testing.T) { + t.Parallel() + // Unanchored pattern for file should match anywhere fs := setupAnchoredTestFS(t) @@ -474,7 +518,7 @@ func TestExcludePatterns_UnanchoredPatternFile(t *testing.T) { require.NotNil(t, scanner) ctx := context.Background() - createSnapshotRecord(t, ctx, repos, "test-snapshot") + createSnapshotRecord(ctx, t, repos, "test-snapshot") result, err := scanner.Scan(ctx, "/backup", "test-snapshot") require.NoError(t, err) @@ -484,5 +528,6 @@ func TestExcludePatterns_UnanchoredPatternFile(t *testing.T) { // - /backup/projectname/file.txt // - /backup/otherproject/projectname/file.txt // Total files: 4, excluded: 3 - require.Equal(t, 1, result.FilesScanned, "Unanchored pattern for file should match anywhere") + require.Equal(t, 1, result.FilesScanned, + "Unanchored pattern for file should match anywhere") } diff --git a/internal/snapshot/file_change_test.go b/internal/snapshot/file_change_test.go index a918010..501b69b 100644 --- a/internal/snapshot/file_change_test.go +++ b/internal/snapshot/file_change_test.go @@ -2,7 +2,6 @@ package snapshot_test import ( "context" - "database/sql" "testing" "time" @@ -15,11 +14,55 @@ import ( "sneak.berlin/go/vaultik/internal/types" ) +// verifyChunkChange checks that after a content change the file references +// the new chunk, the old chunk still exists, and the old chunk no longer +// maps to the modified file. +func verifyChunkChange( + ctx context.Context, + t *testing.T, + repos *database.Repositories, + oldChunkHash, newChunkHash types.ChunkHash, +) { + t.Helper() + + // Verify the chunk hashes are different + assert.NotEqual(t, oldChunkHash, newChunkHash, + "Chunk hash should change when content changes") + + // Get chunk files from second scan + chunkFiles2, err := repos.ChunkFiles.GetByFilePath(ctx, "/test.txt") + require.NoError(t, err) + assert.Len(t, chunkFiles2, 1) + assert.Equal(t, newChunkHash, chunkFiles2[0].ChunkHash) + + // Verify old chunk still exists (it's still valid data) + oldChunk, err := repos.Chunks.GetByHash(ctx, oldChunkHash.String()) + require.NoError(t, err) + assert.NotNil(t, oldChunk) + + // Verify new chunk exists + newChunk, err := repos.Chunks.GetByHash(ctx, newChunkHash.String()) + require.NoError(t, err) + assert.NotNil(t, newChunk) + + // Verify that chunk_files for old chunk no longer references this file + oldChunkFiles, err := repos.ChunkFiles.GetByChunkHash(ctx, oldChunkHash) + require.NoError(t, err) + + for _, cf := range oldChunkFiles { + file, err := repos.Files.GetByID(ctx, cf.FileID) + require.NoError(t, err) + assert.NotEqual(t, "/data/test.txt", file.Path, + "Old chunk should not be associated with the modified file") + } +} + // TestFileContentChange verifies that when a file's content changes, // the old chunks are properly disassociated func TestFileContentChange(t *testing.T) { // Initialize logger for tests log.Initialize(log.Config{}) + t.Parallel() // Create in-memory filesystem fs := afero.NewMemMapFs() @@ -48,23 +91,13 @@ func TestFileContentChange(t *testing.T) { Repositories: repos, MaxBlobSize: int64(1024 * 1024), // 1MB blobs CompressionLevel: 3, - AgeRecipients: []string{"age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05gl0sjq9q9wjg"}, // Test public key + AgeRecipients: []string{testAgePublicKey}, }) // Create first snapshot ctx := context.Background() snapshotID1 := "snapshot1" - err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { - snapshot := &database.Snapshot{ - ID: types.SnapshotID(snapshotID1), - Hostname: "test-host", - VaultikVersion: "test", - StartedAt: time.Now(), - } - - return repos.Snapshots.Create(ctx, tx, snapshot) - }) - require.NoError(t, err) + createSnapshotRecord(ctx, t, repos, snapshotID1) // First scan - should create chunks for initial content result1, err := scanner.Scan(ctx, "/", snapshotID1) @@ -85,22 +118,13 @@ func TestFileContentChange(t *testing.T) { // Modify the file time.Sleep(10 * time.Millisecond) // Ensure mtime changes - err = afero.WriteFile(fs, "/test.txt", []byte("Modified content with different data"), 0644) + err = afero.WriteFile(fs, "/test.txt", + []byte("Modified content with different data"), 0644) require.NoError(t, err) // Create second snapshot snapshotID2 := "snapshot2" - err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { - snapshot := &database.Snapshot{ - ID: types.SnapshotID(snapshotID2), - Hostname: "test-host", - VaultikVersion: "test", - StartedAt: time.Now(), - } - - return repos.Snapshots.Create(ctx, tx, snapshot) - }) - require.NoError(t, err) + createSnapshotRecord(ctx, t, repos, snapshotID2) // Second scan - should create new chunks and remove old associations result2, err := scanner.Scan(ctx, "/", snapshotID2) @@ -113,40 +137,14 @@ func TestFileContentChange(t *testing.T) { assert.Len(t, fileChunks2, 1) // Still 1 chunk but different hash newChunkHash := fileChunks2[0].ChunkHash - // Verify the chunk hashes are different - assert.NotEqual(t, oldChunkHash, newChunkHash, "Chunk hash should change when content changes") - - // Get chunk files from second scan - chunkFiles2, err := repos.ChunkFiles.GetByFilePath(ctx, "/test.txt") - require.NoError(t, err) - assert.Len(t, chunkFiles2, 1) - assert.Equal(t, newChunkHash, chunkFiles2[0].ChunkHash) - - // Verify old chunk still exists (it's still valid data) - oldChunk, err := repos.Chunks.GetByHash(ctx, oldChunkHash.String()) - require.NoError(t, err) - assert.NotNil(t, oldChunk) - - // Verify new chunk exists - newChunk, err := repos.Chunks.GetByHash(ctx, newChunkHash.String()) - require.NoError(t, err) - assert.NotNil(t, newChunk) - - // Verify that chunk_files for old chunk no longer references this file - oldChunkFiles, err := repos.ChunkFiles.GetByChunkHash(ctx, oldChunkHash) - require.NoError(t, err) - - for _, cf := range oldChunkFiles { - file, err := repos.Files.GetByID(ctx, cf.FileID) - require.NoError(t, err) - assert.NotEqual(t, "/data/test.txt", file.Path, "Old chunk should not be associated with the modified file") - } + verifyChunkChange(ctx, t, repos, oldChunkHash, newChunkHash) } // TestMultipleFileChanges verifies handling of multiple file changes in one scan func TestMultipleFileChanges(t *testing.T) { // Initialize logger for tests log.Initialize(log.Config{}) + t.Parallel() // Create in-memory filesystem fs := afero.NewMemMapFs() @@ -183,23 +181,13 @@ func TestMultipleFileChanges(t *testing.T) { Repositories: repos, MaxBlobSize: int64(1024 * 1024), // 1MB blobs CompressionLevel: 3, - AgeRecipients: []string{"age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05gl0sjq9q9wjg"}, // Test public key + AgeRecipients: []string{testAgePublicKey}, }) // Create first snapshot ctx := context.Background() snapshotID1 := "snapshot1" - err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { - snapshot := &database.Snapshot{ - ID: types.SnapshotID(snapshotID1), - Hostname: "test-host", - VaultikVersion: "test", - StartedAt: time.Now(), - } - - return repos.Snapshots.Create(ctx, tx, snapshot) - }) - require.NoError(t, err) + createSnapshotRecord(ctx, t, repos, snapshotID1) // First scan result1, err := scanner.Scan(ctx, "/", snapshotID1) @@ -217,17 +205,7 @@ func TestMultipleFileChanges(t *testing.T) { // Create second snapshot snapshotID2 := "snapshot2" - err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { - snapshot := &database.Snapshot{ - ID: types.SnapshotID(snapshotID2), - Hostname: "test-host", - VaultikVersion: "test", - StartedAt: time.Now(), - } - - return repos.Snapshots.Create(ctx, tx, snapshot) - }) - require.NoError(t, err) + createSnapshotRecord(ctx, t, repos, snapshotID2) // Second scan result2, err := scanner.Scan(ctx, "/", snapshotID2) @@ -240,10 +218,12 @@ func TestMultipleFileChanges(t *testing.T) { for path := range files { fileChunks, err := repos.FileChunks.GetByPath(ctx, path) require.NoError(t, err) - assert.Len(t, fileChunks, 1, "File %s should have exactly 1 chunk association", path) + assert.Len(t, fileChunks, 1, + "File %s should have exactly 1 chunk association", path) chunkFiles, err := repos.ChunkFiles.GetByFilePath(ctx, path) require.NoError(t, err) - assert.Len(t, chunkFiles, 1, "File %s should have exactly 1 chunk-file association", path) + assert.Len(t, chunkFiles, 1, + "File %s should have exactly 1 chunk-file association", path) } } diff --git a/internal/snapshot/manifest.go b/internal/snapshot/manifest.go index d22b395..1f3f665 100644 --- a/internal/snapshot/manifest.go +++ b/internal/snapshot/manifest.go @@ -10,6 +10,8 @@ import ( ) // Manifest represents the structure of a snapshot's blob manifest +// +//nolint:tagliatelle // snake_case is the established on-disk manifest format type Manifest struct { SnapshotID string `json:"snapshot_id"` Timestamp string `json:"timestamp"` @@ -19,6 +21,8 @@ type Manifest struct { } // BlobInfo represents information about a single blob in the manifest +// +//nolint:tagliatelle // snake_case is the established on-disk manifest format type BlobInfo struct { Hash string `json:"hash"` CompressedSize int64 `json:"compressed_size"` @@ -55,7 +59,8 @@ func EncodeManifest(manifest *Manifest, compressionLevel int) ([]byte, error) { // Compress using zstd var compressedBuf bytes.Buffer - writer, err := zstd.NewWriter(&compressedBuf, zstd.WithEncoderLevel(zstd.EncoderLevelFromZstd(compressionLevel))) + writer, err := zstd.NewWriter(&compressedBuf, + zstd.WithEncoderLevel(zstd.EncoderLevelFromZstd(compressionLevel))) if err != nil { return nil, fmt.Errorf("creating zstd writer: %w", err) } diff --git a/internal/snapshot/module.go b/internal/snapshot/module.go index eca0b1d..fc0dfec 100644 --- a/internal/snapshot/module.go +++ b/internal/snapshot/module.go @@ -21,6 +21,8 @@ type ScannerParams struct { // Module exports backup functionality as an fx module. // It provides a ScannerFactory that can create Scanner instances // with custom parameters while sharing common dependencies. +// +//nolint:gochecknoglobals // fx module definitions are conventionally globals var Module = fx.Module("backup", fx.Provide( provideScannerFactory, @@ -31,7 +33,9 @@ var Module = fx.Module("backup", // ScannerFactory creates scanners with custom parameters type ScannerFactory func(params ScannerParams) *Scanner -func provideScannerFactory(cfg *config.Config, repos *database.Repositories, storer storage.Storer) ScannerFactory { +func provideScannerFactory( + cfg *config.Config, repos *database.Repositories, storer storage.Storer, +) ScannerFactory { return func(params ScannerParams) *Scanner { // Use provided excludes, or fall back to global config excludes excludes := params.Exclude diff --git a/internal/snapshot/permission_error_test.go b/internal/snapshot/permission_error_test.go index d339d3d..8dfad86 100644 --- a/internal/snapshot/permission_error_test.go +++ b/internal/snapshot/permission_error_test.go @@ -1,3 +1,4 @@ +//nolint:testpackage // needs access to unexported wrapPermissionError package snapshot import ( @@ -9,12 +10,15 @@ import ( "testing" ) -func TestWrapPermissionError(t *testing.T) { - // Non-permission errors pass through unchanged. - plain := errors.New("disk on fire") +// errDiskOnFire is a non-permission sentinel used to verify pass-through. +var errDiskOnFire = errors.New("disk on fire") - got := wrapPermissionError("/some/path", plain) - if !errors.Is(got, plain) { +func TestWrapPermissionError(t *testing.T) { + t.Parallel() + + // Non-permission errors pass through unchanged. + got := wrapPermissionError("/some/path", errDiskOnFire) + if !errors.Is(got, errDiskOnFire) { t.Errorf("non-permission error should pass through, got %v", got) } @@ -32,15 +36,16 @@ func TestWrapPermissionError(t *testing.T) { if runtime.GOOS == "darwin" { if !strings.Contains(wrapped.Error(), "Full Disk Access") { - t.Errorf("macOS permission error should mention Full Disk Access:\n%s", wrapped.Error()) + t.Errorf("macOS permission error should mention Full Disk Access:\n%s", + wrapped.Error()) } if !strings.Contains(wrapped.Error(), "System Settings") { - t.Errorf("macOS permission error should point at System Settings:\n%s", wrapped.Error()) - } - } else { - if !strings.Contains(wrapped.Error(), "--skip-errors") { - t.Errorf("non-macOS permission error should mention --skip-errors:\n%s", wrapped.Error()) + t.Errorf("macOS permission error should point at System Settings:\n%s", + wrapped.Error()) } + } else if !strings.Contains(wrapped.Error(), "--skip-errors") { + t.Errorf("non-macOS permission error should mention --skip-errors:\n%s", + wrapped.Error()) } } diff --git a/internal/snapshot/progress.go b/internal/snapshot/progress.go index 8795e92..71fbb98 100644 --- a/internal/snapshot/progress.go +++ b/internal/snapshot/progress.go @@ -19,14 +19,41 @@ const ( // These updates show current progress, ETA, and the file being processed. SummaryInterval = 10 * time.Second - // DetailInterval defines how often multi-line detailed status reports are printed. - // These reports include comprehensive statistics about files, chunks, blobs, and uploads. + // DetailInterval defines how often multi-line detailed status reports are + // printed. These reports include comprehensive statistics about files, + // chunks, blobs, and uploads. DetailInterval = 60 * time.Second // UploadProgressInterval defines how often upload progress messages are logged. UploadProgressInterval = 15 * time.Second ) +const ( + // bitsPerByte converts byte counts to bit counts for speed display. + bitsPerByte = 8 + + // percentScale converts a ratio to a percentage. + percentScale = 100 + + // currentFileMaxLen is the display width used for current-file paths. + currentFileMaxLen = 40 + + // secondsPerMinute and minutesPerHour are used for duration formatting. + secondsPerMinute = 60 + minutesPerHour = 60 + + // Bit-rate thresholds for human-readable upload speed formatting. + bitsPerGbit = 1e9 + bitsPerMbit = 1e6 + bitsPerKbit = 1e3 + + // ellipsis prefixes truncated paths and suffixes shortened hashes. + ellipsis = "..." + + // hashPrefixLen is how many hex characters of a blob hash to show in logs. + hashPrefixLen = 8 +) + // ProgressStats holds atomic counters for progress tracking type ProgressStats struct { FilesScanned atomic.Int64 // Total files seen during scan (includes skipped) @@ -64,7 +91,7 @@ type UploadInfo struct { // ProgressReporter handles periodic progress reporting type ProgressReporter struct { stats *ProgressStats - ctx context.Context + ctx context.Context //nolint:containedctx // bound at construction cancel context.CancelFunc wg sync.WaitGroup detailTicker *time.Ticker @@ -127,6 +154,161 @@ func (pr *ProgressReporter) SetTotalSize(size int64) { pr.stats.ProcessStartTime.Store(time.Now().UTC()) } +// Helper functions + +func formatDuration(d time.Duration) string { + if d < 0 { + return "unknown" + } + + if d < time.Minute { + return fmt.Sprintf("%ds", int(d.Seconds())) + } + + if d < time.Hour { + return fmt.Sprintf("%dm%ds", int(d.Minutes()), int(d.Seconds())%secondsPerMinute) + } + + return fmt.Sprintf("%dh%dm", int(d.Hours()), int(d.Minutes())%minutesPerHour) +} + +func formatPercent(numerator, denominator int64) string { + if denominator == 0 { + return "0.0%" + } + + return fmt.Sprintf("%.1f%%", float64(numerator)/float64(denominator)*percentScale) +} + +func formatRatio(compressed, uncompressed int64) string { + if uncompressed == 0 { + return "1.00" + } + + ratio := float64(compressed) / float64(uncompressed) + + return fmt.Sprintf("%.2f", ratio) +} + +func truncatePath(path string, maxLen int) string { + if len(path) <= maxLen { + return path + } + // Keep the last maxLen-len(ellipsis) characters and prepend the ellipsis. + return ellipsis + path[len(path)-(maxLen-len(ellipsis)):] +} + +// safeUint64 converts a non-negative int64 counter to uint64 for display, +// clamping negative values to zero. +func safeUint64(n int64) uint64 { + if n < 0 { + return 0 + } + + return uint64(n) +} + +// ReportUploadStart marks the beginning of a blob upload +func (pr *ProgressReporter) ReportUploadStart(blobHash string, size int64) { + info := &UploadInfo{ + BlobHash: blobHash, + Size: size, + StartTime: time.Now().UTC(), + } + pr.stats.CurrentUpload.Store(info) + + // Log the start of upload + log.Info("Starting blob upload", + "hash", blobHash[:hashPrefixLen]+ellipsis, + "size", humanize.Bytes(safeUint64(size))) +} + +// ReportUploadComplete marks the completion of a blob upload +func (pr *ProgressReporter) ReportUploadComplete( + blobHash string, size int64, duration time.Duration, +) { + // Clear current upload + pr.stats.CurrentUpload.Store((*UploadInfo)(nil)) + + // Add to total upload duration + pr.stats.UploadDurationMs.Add(duration.Milliseconds()) + + // Calculate speed + if duration < time.Millisecond { + duration = time.Millisecond + } + + bytesPerSec := float64(size) / duration.Seconds() + bitsPerSec := bytesPerSec * bitsPerByte + + // Format speed + var speedStr string + + switch { + case bitsPerSec >= bitsPerGbit: + speedStr = fmt.Sprintf("%.1fGbit/sec", bitsPerSec/bitsPerGbit) + case bitsPerSec >= bitsPerMbit: + speedStr = fmt.Sprintf("%.0fMbit/sec", bitsPerSec/bitsPerMbit) + case bitsPerSec >= bitsPerKbit: + speedStr = fmt.Sprintf("%.0fKbit/sec", bitsPerSec/bitsPerKbit) + default: + speedStr = fmt.Sprintf("%.0fbit/sec", bitsPerSec) + } + + log.Info("Blob upload completed", + "hash", blobHash[:hashPrefixLen]+ellipsis, + "size", humanize.Bytes(safeUint64(size)), + "duration", formatDuration(duration), + "speed", speedStr) +} + +// UpdateChunkingActivity updates the last chunking time +func (pr *ProgressReporter) UpdateChunkingActivity() { + pr.stats.mu.Lock() + pr.stats.lastChunkingTime = time.Now().UTC() + pr.stats.mu.Unlock() +} + +// ReportUploadProgress reports current upload progress with instantaneous speed +func (pr *ProgressReporter) ReportUploadProgress( + blobHash string, bytesUploaded, totalSize int64, instantSpeed float64, +) { + // Update the current upload info with progress + uploadInfo, ok := pr.stats.CurrentUpload.Load().(*UploadInfo) + if ok && uploadInfo != nil { + now := time.Now() + + // Only log at the configured interval + if now.Sub(uploadInfo.LastLogTime) >= UploadProgressInterval { + // Format speed in bits/second using humanize + bitsPerSec := instantSpeed * bitsPerByte + speedStr := humanize.SI(bitsPerSec, "bit/sec") + + percent := float64(bytesUploaded) / float64(totalSize) * percentScale + + // Calculate ETA based on current speed + etaStr := "unknown" + + if instantSpeed > 0 && bytesUploaded < totalSize { + remainingBytes := totalSize - bytesUploaded + remainingSeconds := float64(remainingBytes) / instantSpeed + eta := time.Duration(remainingSeconds * float64(time.Second)) + etaStr = formatDuration(eta) + } + + log.Info("Blob upload progress", + "hash", blobHash[:hashPrefixLen]+ellipsis, + "progress", fmt.Sprintf("%.1f%%", percent), + "uploaded", humanize.Bytes(safeUint64(bytesUploaded)), + "total", humanize.Bytes(safeUint64(totalSize)), + "speed", speedStr, + "eta", etaStr) + + uploadInfo.LastLogTime = now + } + } +} + // run is the main progress reporting loop func (pr *ProgressReporter) run() { defer pr.wg.Done() @@ -150,7 +332,8 @@ func (pr *ProgressReporter) run() { // printSummaryStatus prints a one-line status update func (pr *ProgressReporter) printSummaryStatus() { // Check if we're currently uploading - if uploadInfo, ok := pr.stats.CurrentUpload.Load().(*UploadInfo); ok && uploadInfo != nil { + uploadInfo, ok := pr.stats.CurrentUpload.Load().(*UploadInfo) + if ok && uploadInfo != nil { // Show upload progress instead pr.printUploadProgress(uploadInfo) @@ -172,7 +355,7 @@ func (pr *ProgressReporter) printSummaryStatus() { bytesSkipped := pr.stats.BytesSkipped.Load() bytesProcessed := pr.stats.BytesProcessed.Load() totalSize := pr.stats.TotalSize.Load() - currentFile := pr.stats.CurrentFile.Load().(string) + currentFile, _ := pr.stats.CurrentFile.Load().(string) // Calculate ETA if we have total size and are processing etaStr := "" @@ -201,15 +384,15 @@ func (pr *ProgressReporter) printSummaryStatus() { status := fmt.Sprintf("Snapshot progress: %d/%d files, %s/%s (%.1f%%), %s/s%s", filesProcessed, totalFiles, - humanize.Bytes(uint64(bytesProcessed)), - humanize.Bytes(uint64(totalSize)), - float64(bytesProcessed)/float64(totalSize)*100, + humanize.Bytes(safeUint64(bytesProcessed)), + humanize.Bytes(safeUint64(totalSize)), + float64(bytesProcessed)/float64(totalSize)*percentScale, humanize.Bytes(uint64(rate)), etaStr, ) if currentFile != "" { - status += " | Current: " + truncatePath(currentFile, 40) + status += " | Current: " + truncatePath(currentFile, currentFileMaxLen) } log.Info(status) @@ -232,7 +415,7 @@ func (pr *ProgressReporter) printDetailedStatus() { blobsCreated := pr.stats.BlobsCreated.Load() blobsUploaded := pr.stats.BlobsUploaded.Load() bytesUploaded := pr.stats.BytesUploaded.Load() - currentFile := pr.stats.CurrentFile.Load().(string) + currentFile, _ := pr.stats.CurrentFile.Load().(string) totalBytes := bytesScanned + bytesSkipped rate := float64(totalBytes) / elapsed.Seconds() @@ -251,11 +434,11 @@ func (pr *ProgressReporter) printDetailedStatus() { remainingBytes := totalSize - bytesProcessed remainingSeconds := float64(remainingBytes) / processRate eta := time.Duration(remainingSeconds * float64(time.Second)) - percentComplete := float64(bytesProcessed) / float64(totalSize) * 100 + percentComplete := float64(bytesProcessed) / float64(totalSize) * percentScale log.Info("Overall progress", "percent", fmt.Sprintf("%.1f%%", percentComplete), - "processed", humanize.Bytes(uint64(bytesProcessed)), - "total", humanize.Bytes(uint64(totalSize)), + "processed", humanize.Bytes(safeUint64(bytesProcessed)), + "total", humanize.Bytes(safeUint64(totalSize)), "rate", humanize.Bytes(uint64(processRate))+"/s", "eta", formatDuration(eta)) } @@ -268,9 +451,9 @@ func (pr *ProgressReporter) printDetailedStatus() { "total", filesScanned, "skip_rate", formatPercent(filesSkipped, filesScanned)) log.Info("Data scanned", - "new", humanize.Bytes(uint64(bytesScanned)), - "skipped", humanize.Bytes(uint64(bytesSkipped)), - "total", humanize.Bytes(uint64(totalBytes)), + "new", humanize.Bytes(safeUint64(bytesScanned)), + "skipped", humanize.Bytes(safeUint64(bytesSkipped)), + "total", humanize.Bytes(safeUint64(totalBytes)), "scan_rate", humanize.Bytes(uint64(rate))+"/s") log.Info("Chunks created", "count", chunksCreated) log.Info("Blobs status", @@ -278,7 +461,7 @@ func (pr *ProgressReporter) printDetailedStatus() { "uploaded", blobsUploaded, "pending", blobsCreated-blobsUploaded) log.Info("Total uploaded to remote", - "uploaded", humanize.Bytes(uint64(bytesUploaded)), + "uploaded", humanize.Bytes(safeUint64(bytesUploaded)), "compression_ratio", formatRatio(bytesUploaded, bytesScanned)) if currentFile != "" { @@ -288,146 +471,8 @@ func (pr *ProgressReporter) printDetailedStatus() { log.Notice("=============================") } -// Helper functions - -func formatDuration(d time.Duration) string { - if d < 0 { - return "unknown" - } - - if d < time.Minute { - return fmt.Sprintf("%ds", int(d.Seconds())) - } - - if d < time.Hour { - return fmt.Sprintf("%dm%ds", int(d.Minutes()), int(d.Seconds())%60) - } - - return fmt.Sprintf("%dh%dm", int(d.Hours()), int(d.Minutes())%60) -} - -func formatPercent(numerator, denominator int64) string { - if denominator == 0 { - return "0.0%" - } - - return fmt.Sprintf("%.1f%%", float64(numerator)/float64(denominator)*100) -} - -func formatRatio(compressed, uncompressed int64) string { - if uncompressed == 0 { - return "1.00" - } - - ratio := float64(compressed) / float64(uncompressed) - - return fmt.Sprintf("%.2f", ratio) -} - -func truncatePath(path string, maxLen int) string { - if len(path) <= maxLen { - return path - } - // Keep the last maxLen-3 characters and prepend "..." - return "..." + path[len(path)-(maxLen-3):] -} - // printUploadProgress prints upload progress -func (pr *ProgressReporter) printUploadProgress(info *UploadInfo) { +func (pr *ProgressReporter) printUploadProgress(_ *UploadInfo) { // This function is called repeatedly during upload, not just at start // Don't print anything here - the actual progress is shown by ReportUploadProgress } - -// ReportUploadStart marks the beginning of a blob upload -func (pr *ProgressReporter) ReportUploadStart(blobHash string, size int64) { - info := &UploadInfo{ - BlobHash: blobHash, - Size: size, - StartTime: time.Now().UTC(), - } - pr.stats.CurrentUpload.Store(info) - - // Log the start of upload - log.Info("Starting blob upload", - "hash", blobHash[:8]+"...", - "size", humanize.Bytes(uint64(size))) -} - -// ReportUploadComplete marks the completion of a blob upload -func (pr *ProgressReporter) ReportUploadComplete(blobHash string, size int64, duration time.Duration) { - // Clear current upload - pr.stats.CurrentUpload.Store((*UploadInfo)(nil)) - - // Add to total upload duration - pr.stats.UploadDurationMs.Add(duration.Milliseconds()) - - // Calculate speed - if duration < time.Millisecond { - duration = time.Millisecond - } - - bytesPerSec := float64(size) / duration.Seconds() - bitsPerSec := bytesPerSec * 8 - - // Format speed - var speedStr string - if bitsPerSec >= 1e9 { - speedStr = fmt.Sprintf("%.1fGbit/sec", bitsPerSec/1e9) - } else if bitsPerSec >= 1e6 { - speedStr = fmt.Sprintf("%.0fMbit/sec", bitsPerSec/1e6) - } else if bitsPerSec >= 1e3 { - speedStr = fmt.Sprintf("%.0fKbit/sec", bitsPerSec/1e3) - } else { - speedStr = fmt.Sprintf("%.0fbit/sec", bitsPerSec) - } - - log.Info("Blob upload completed", - "hash", blobHash[:8]+"...", - "size", humanize.Bytes(uint64(size)), - "duration", formatDuration(duration), - "speed", speedStr) -} - -// UpdateChunkingActivity updates the last chunking time -func (pr *ProgressReporter) UpdateChunkingActivity() { - pr.stats.mu.Lock() - pr.stats.lastChunkingTime = time.Now().UTC() - pr.stats.mu.Unlock() -} - -// ReportUploadProgress reports current upload progress with instantaneous speed -func (pr *ProgressReporter) ReportUploadProgress(blobHash string, bytesUploaded, totalSize int64, instantSpeed float64) { - // Update the current upload info with progress - if uploadInfo, ok := pr.stats.CurrentUpload.Load().(*UploadInfo); ok && uploadInfo != nil { - now := time.Now() - - // Only log at the configured interval - if now.Sub(uploadInfo.LastLogTime) >= UploadProgressInterval { - // Format speed in bits/second using humanize - bitsPerSec := instantSpeed * 8 - speedStr := humanize.SI(bitsPerSec, "bit/sec") - - percent := float64(bytesUploaded) / float64(totalSize) * 100 - - // Calculate ETA based on current speed - etaStr := "unknown" - - if instantSpeed > 0 && bytesUploaded < totalSize { - remainingBytes := totalSize - bytesUploaded - remainingSeconds := float64(remainingBytes) / instantSpeed - eta := time.Duration(remainingSeconds * float64(time.Second)) - etaStr = formatDuration(eta) - } - - log.Info("Blob upload progress", - "hash", blobHash[:8]+"...", - "progress", fmt.Sprintf("%.1f%%", percent), - "uploaded", humanize.Bytes(uint64(bytesUploaded)), - "total", humanize.Bytes(uint64(totalSize)), - "speed", speedStr, - "eta", etaStr) - - uploadInfo.LastLogTime = now - } - } -} diff --git a/internal/snapshot/scanner.go b/internal/snapshot/scanner.go index f4efafb..9a8eb27 100644 --- a/internal/snapshot/scanner.go +++ b/internal/snapshot/scanner.go @@ -48,28 +48,32 @@ type compiledPattern struct { // Scanner scans directories and populates the database with file and chunk information type Scanner struct { - fs afero.Fs - chunker *chunker.Chunker - packer *blob.Packer - repos *database.Repositories - storage storage.Storer - maxBlobSize int64 - compressionLevel int - ageRecipient string - snapshotID string // Current snapshot being processed - currentSourcePath string // Current source directory being scanned (for restore path stripping) + fs afero.Fs + chunker *chunker.Chunker + packer *blob.Packer + repos *database.Repositories + storage storage.Storer + maxBlobSize int64 + compressionLevel int + ageRecipient string + snapshotID string // Current snapshot being processed + // currentSourcePath is the source directory being scanned (used for + // restore path stripping). + currentSourcePath string exclude []string // Glob patterns for files/directories to exclude compiledExclude []compiledPattern // Compiled glob patterns progress *ProgressReporter - skipErrors bool // Skip file read errors (log loudly but continue) - ui *ui.Writer // User-facing output; never nil (defaults to a discarding writer) + skipErrors bool // Skip file read errors (log loudly but continue) + // ui is the user-facing output; never nil (defaults to a discarding writer). + ui *ui.Writer // In-memory cache of known chunk hashes for fast existence checks knownChunks map[string]struct{} knownChunksMu sync.RWMutex - // Pending chunk hashes - chunks that have been added to packer but not yet committed to DB - // When a blob finalizes, the committed chunks are removed from this set + // Pending chunk hashes - chunks that have been added to packer but not + // yet committed to DB. When a blob finalizes, the committed chunks are + // removed from this set. pendingChunkHashes map[string]struct{} pendingChunkHashesMu sync.Mutex @@ -82,9 +86,29 @@ type Scanner struct { packerMu sync.Mutex // Blocks chunk production during blob creation // Context for cancellation - scanCtx context.Context + scanCtx context.Context //nolint:containedctx // set per-Scan for packer callbacks } +// Periodic status output intervals and thresholds for the scan and +// process phases. +const ( + // scanStatusInterval is how often periodic progress lines are printed + // during the scan and process phases. + scanStatusInterval = 15 * time.Second + + // assocStatusInterval is how often progress is printed while + // associating unchanged files with the snapshot. + assocStatusInterval = 5 * time.Second + + // uploadProgressMinSeconds is the minimum elapsed time between live + // upload-progress reporter updates. + uploadProgressMinSeconds = 0.5 + + // snapshotAssocBatchSize is how many file IDs are associated with the + // snapshot per transaction. + snapshotAssocBatchSize = 1000 +) + // ScannerConfig contains configuration for the scanner type ScannerConfig struct { FS afero.Fs @@ -171,9 +195,12 @@ func NewScanner(cfg ScannerConfig) *Scanner { } // Scan scans a directory and populates the database -func (s *Scanner) Scan(ctx context.Context, path string, snapshotID string) (*ScanResult, error) { +func (s *Scanner) Scan( + ctx context.Context, path string, snapshotID string, +) (*ScanResult, error) { s.snapshotID = snapshotID - s.currentSourcePath = path // Store source path for file records (used during restore) + // Store source path for file records (used during restore) + s.currentSourcePath = path s.scanCtx = ctx result := &ScanResult{ StartTime: time.Now().UTC(), @@ -220,7 +247,8 @@ func (s *Scanner) Scan(ctx context.Context, path string, snapshotID string) (*Sc // Phase 1c: Associate unchanged files with this snapshot (no new records needed) if len(scanResult.UnchangedFileIDs) > 0 { - s.ui.Begin("Associating %s unchanged files with the snapshot.", s.ui.Count(len(scanResult.UnchangedFileIDs))) + s.ui.Beginf("Associating %s unchanged files with the snapshot.", + s.ui.Count(len(scanResult.UnchangedFileIDs))) err := s.batchAddFilesToSnapshot(ctx, scanResult.UnchangedFileIDs) if err != nil { @@ -233,15 +261,19 @@ func (s *Scanner) Scan(ctx context.Context, path string, snapshotID string) (*Sc // Phase 2: Process files and create chunks if len(filesToProcess) > 0 { - s.ui.Begin("Backing up %s snapshot source files (chunking, compressing, encrypting, uploading).", s.ui.Count(len(filesToProcess))) - log.Info("Phase 2/3: Creating snapshot (chunking, compressing, encrypting, and uploading blobs)") + s.ui.Beginf("Backing up %s snapshot source files "+ + "(chunking, compressing, encrypting, uploading).", + s.ui.Count(len(filesToProcess))) + log.Info("Phase 2/3: Creating snapshot " + + "(chunking, compressing, encrypting, and uploading blobs)") err := s.processPhase(ctx, filesToProcess, result) if err != nil { return nil, fmt.Errorf("process phase failed: %w", err) } } else { - s.ui.Info("Snapshot file backup skipped: no changed files (creating metadata-only snapshot).") + s.ui.Infof("Snapshot file backup skipped: no changed files " + + "(creating metadata-only snapshot).") log.Info("Phase 2/3: Skipping (no files need processing, metadata-only snapshot)") } @@ -251,33 +283,45 @@ func (s *Scanner) Scan(ctx context.Context, path string, snapshotID string) (*Sc return result, nil } -// loadDatabaseState loads known files and chunks from the database into memory for fast lookup -// This avoids per-file and per-chunk database queries during the scan and process phases -func (s *Scanner) loadDatabaseState(ctx context.Context, path string) (map[string]*database.File, error) { - s.ui.Begin("Loading known files from local index database.") +// GetProgress returns the progress reporter for this scanner +func (s *Scanner) GetProgress() *ProgressReporter { + return s.progress +} + +// loadDatabaseState loads known files and chunks from the database into +// memory for fast lookup. This avoids per-file and per-chunk database +// queries during the scan and process phases. +func (s *Scanner) loadDatabaseState( + ctx context.Context, path string, +) (map[string]*database.File, error) { + s.ui.Beginf("Loading known files from local index database.") knownFiles, err := s.loadKnownFiles(ctx, path) if err != nil { return nil, fmt.Errorf("loading known files: %w", err) } - s.ui.Complete("Loaded %s known files from local index database.", s.ui.Count(len(knownFiles))) + s.ui.Completef("Loaded %s known files from local index database.", + s.ui.Count(len(knownFiles))) - s.ui.Begin("Loading known chunks from local index database.") + s.ui.Beginf("Loading known chunks from local index database.") err = s.loadKnownChunks(ctx) if err != nil { return nil, fmt.Errorf("loading known chunks: %w", err) } - s.ui.Complete("Loaded %s known chunks from local index database.", s.ui.Count(len(s.knownChunks))) + s.ui.Completef("Loaded %s known chunks from local index database.", + s.ui.Count(len(s.knownChunks))) return knownFiles, nil } // summarizeScanPhase calculates total size to process, updates progress tracking, // and prints the scan phase summary with file counts and sizes -func (s *Scanner) summarizeScanPhase(result *ScanResult, filesToProcess []*FileToProcess) { +func (s *Scanner) summarizeScanPhase( + result *ScanResult, filesToProcess []*FileToProcess, +) { var totalSizeToProcess int64 for _, file := range filesToProcess { totalSizeToProcess += file.FileInfo.Size() @@ -290,11 +334,12 @@ func (s *Scanner) summarizeScanPhase(result *ScanResult, filesToProcess []*FileT log.Info("Phase 1 complete", "total_files", len(filesToProcess), - "total_size", humanize.Bytes(uint64(totalSizeToProcess)), + "total_size", humanize.Bytes(safeUint64(totalSizeToProcess)), "files_skipped", result.FilesSkipped, - "bytes_skipped", humanize.Bytes(uint64(result.BytesSkipped))) + "bytes_skipped", humanize.Bytes(safeUint64(result.BytesSkipped))) - msg := fmt.Sprintf("Enumerated %s snapshot source files (%s total), %s to back up (%s)", + msg := fmt.Sprintf( + "Enumerated %s snapshot source files (%s total), %s to back up (%s)", s.ui.Count(result.FilesScanned), s.ui.Size(totalSizeToProcess+result.BytesSkipped), s.ui.Count(len(filesToProcess)), @@ -305,7 +350,7 @@ func (s *Scanner) summarizeScanPhase(result *ScanResult, filesToProcess []*FileT s.ui.Size(result.BytesDeleted)) } - s.ui.Complete("%s.", msg) + s.ui.Completef("%s.", msg) } // finalizeScanResult populates final blob statistics in the scan result @@ -331,7 +376,9 @@ func (s *Scanner) finalizeScanResult(ctx context.Context, result *ScanResult) { // loadKnownFiles loads all known files from the database into a map for fast lookup // This avoids per-file database queries during the scan phase -func (s *Scanner) loadKnownFiles(ctx context.Context, path string) (map[string]*database.File, error) { +func (s *Scanner) loadKnownFiles( + ctx context.Context, path string, +) (map[string]*database.File, error) { files, err := s.repos.Files.ListByPrefix(ctx, path) if err != nil { return nil, fmt.Errorf("listing files by prefix: %w", err) @@ -345,8 +392,9 @@ func (s *Scanner) loadKnownFiles(ctx context.Context, path string) (map[string]* return result, nil } -// loadKnownChunks loads all known chunk hashes from the database into a map for fast lookup -// This avoids per-chunk database queries during file processing +// loadKnownChunks loads all known chunk hashes from the database into a +// map for fast lookup. This avoids per-chunk database queries during file +// processing. func (s *Scanner) loadKnownChunks(ctx context.Context) error { chunks, err := s.repos.Chunks.List(ctx) if err != nil { @@ -398,7 +446,8 @@ func (s *Scanner) removePendingChunkHashes(hashes []string) { delete(s.pendingChunkHashes, hash) } s.pendingChunkHashesMu.Unlock() - log.Debug("removePendingChunkHashes: done", "count", len(hashes), "duration", time.Since(start)) + log.Debug("removePendingChunkHashes: done", + "count", len(hashes), "duration", time.Since(start)) } // isChunkPending returns true if the chunk is still pending (not yet committed to DB) @@ -510,19 +559,24 @@ func (s *Scanner) flushCompletedPendingFiles(ctx context.Context) error { txStart := time.Now() err := s.executeBatchFileFlush(ctx, allFiles, allFileIDs, allFileChunks, allChunkFiles) - log.Debug("flushCompletedPendingFiles: transaction done", "duration", time.Since(txStart)) - log.Debug("flushCompletedPendingFiles: total duration", "duration", time.Since(flushStart)) + log.Debug("flushCompletedPendingFiles: transaction done", + "duration", time.Since(txStart)) + log.Debug("flushCompletedPendingFiles: total duration", + "duration", time.Since(flushStart)) return err } // partitionPendingByChunkStatus separates pending files into those whose chunks -// are all committed to DB (ready to flush) and those still waiting on pending chunks. -// Updates s.pendingFiles to contain only the still-pending files. -func (s *Scanner) partitionPendingByChunkStatus() (canFlush []pendingFileData, stillPendingCount int) { +// are all committed to DB (ready to flush) and those still waiting on pending +// chunks. Updates s.pendingFiles to contain only the still-pending files. +func (s *Scanner) partitionPendingByChunkStatus() ([]pendingFileData, int) { + var canFlush []pendingFileData + log.Debug("flushCompletedPendingFiles: acquiring pendingFilesMu lock") s.pendingFilesMu.Lock() - log.Debug("flushCompletedPendingFiles: acquired lock", "pending_files", len(s.pendingFiles)) + log.Debug("flushCompletedPendingFiles: acquired lock", + "pending_files", len(s.pendingFiles)) var stillPending []pendingFileData @@ -548,7 +602,10 @@ func (s *Scanner) partitionPendingByChunkStatus() (canFlush []pendingFileData, s } } - log.Debug("flushCompletedPendingFiles: check done", "duration", time.Since(checkStart), "can_flush", len(canFlush), "still_pending", len(stillPending)) + log.Debug("flushCompletedPendingFiles: check done", + "duration", time.Since(checkStart), + "can_flush", len(canFlush), + "still_pending", len(stillPending)) s.pendingFiles = stillPending s.pendingFilesMu.Unlock() @@ -557,9 +614,12 @@ func (s *Scanner) partitionPendingByChunkStatus() (canFlush []pendingFileData, s return canFlush, len(stillPending) } -// collectBatchFlushData aggregates file records, IDs, file-chunk mappings, and chunk-file -// mappings from the given pending file data for efficient batch database operations -func (s *Scanner) collectBatchFlushData(canFlush []pendingFileData) ([]*database.File, []types.FileID, []database.FileChunk, []database.ChunkFile) { +// collectBatchFlushData aggregates file records, IDs, file-chunk mappings, +// and chunk-file mappings from the given pending file data for efficient +// batch database operations +func (s *Scanner) collectBatchFlushData( + canFlush []pendingFileData, +) ([]*database.File, []types.FileID, []database.FileChunk, []database.ChunkFile) { log.Debug("flushCompletedPendingFiles: collecting data for batch ops") collectStart := time.Now() @@ -567,10 +627,11 @@ func (s *Scanner) collectBatchFlushData(canFlush []pendingFileData) ([]*database var ( allFileChunks []database.FileChunk allChunkFiles []database.ChunkFile - allFileIDs []types.FileID - allFiles []*database.File ) + allFileIDs := make([]types.FileID, 0, len(canFlush)) + allFiles := make([]*database.File, 0, len(canFlush)) + for _, data := range canFlush { allFileChunks = append(allFileChunks, data.fileChunks...) allChunkFiles = append(allChunkFiles, data.chunkFiles...) @@ -587,82 +648,42 @@ func (s *Scanner) collectBatchFlushData(canFlush []pendingFileData) ([]*database return allFiles, allFileIDs, allFileChunks, allChunkFiles } -// executeBatchFileFlush writes all collected file data to the database in a single transaction, -// including deleting old mappings, creating file records, and adding snapshot associations -func (s *Scanner) executeBatchFileFlush(ctx context.Context, allFiles []*database.File, allFileIDs []types.FileID, allFileChunks []database.FileChunk, allChunkFiles []database.ChunkFile) error { +// executeBatchFileFlush writes all collected file data to the database in a +// single transaction, including deleting old mappings, creating file +// records, and adding snapshot associations +func (s *Scanner) executeBatchFileFlush( + ctx context.Context, + allFiles []*database.File, + allFileIDs []types.FileID, + allFileChunks []database.FileChunk, + allChunkFiles []database.ChunkFile, +) error { return s.repos.WithTx(ctx, func(txCtx context.Context, tx *sql.Tx) error { log.Debug("flushCompletedPendingFiles: inside transaction") - // Batch delete old file_chunks and chunk_files - log.Debug("flushCompletedPendingFiles: deleting old file_chunks") - - opStart := time.Now() - - err := s.repos.FileChunks.DeleteByFileIDs(txCtx, tx, allFileIDs) + err := s.batchDeleteOldMappings(txCtx, tx, allFileIDs) if err != nil { - return fmt.Errorf("batch deleting old file chunks: %w", err) + return err } - log.Debug("flushCompletedPendingFiles: deleted file_chunks", "duration", time.Since(opStart)) - - log.Debug("flushCompletedPendingFiles: deleting old chunk_files") - - opStart = time.Now() - - err = s.repos.ChunkFiles.DeleteByFileIDs(txCtx, tx, allFileIDs) + err = s.batchInsertFileData( + txCtx, tx, allFiles, allFileChunks, allChunkFiles) if err != nil { - return fmt.Errorf("batch deleting old chunk files: %w", err) + return err } - log.Debug("flushCompletedPendingFiles: deleted chunk_files", "duration", time.Since(opStart)) - - // Batch create/update file records - log.Debug("flushCompletedPendingFiles: creating files") - - opStart = time.Now() - - err = s.repos.Files.CreateBatch(txCtx, tx, allFiles) - if err != nil { - return fmt.Errorf("batch creating file records: %w", err) - } - - log.Debug("flushCompletedPendingFiles: created files", "duration", time.Since(opStart)) - - // Batch insert file_chunks - log.Debug("flushCompletedPendingFiles: inserting file_chunks") - - opStart = time.Now() - - err = s.repos.FileChunks.CreateBatch(txCtx, tx, allFileChunks) - if err != nil { - return fmt.Errorf("batch creating file chunks: %w", err) - } - - log.Debug("flushCompletedPendingFiles: inserted file_chunks", "duration", time.Since(opStart)) - - // Batch insert chunk_files - log.Debug("flushCompletedPendingFiles: inserting chunk_files") - - opStart = time.Now() - - err = s.repos.ChunkFiles.CreateBatch(txCtx, tx, allChunkFiles) - if err != nil { - return fmt.Errorf("batch creating chunk files: %w", err) - } - - log.Debug("flushCompletedPendingFiles: inserted chunk_files", "duration", time.Since(opStart)) - // Batch add files to snapshot log.Debug("flushCompletedPendingFiles: adding files to snapshot") - opStart = time.Now() + opStart := time.Now() err = s.repos.Snapshots.AddFilesByIDBatch(txCtx, tx, s.snapshotID, allFileIDs) if err != nil { return fmt.Errorf("batch adding files to snapshot: %w", err) } - log.Debug("flushCompletedPendingFiles: added files to snapshot", "duration", time.Since(opStart)) + log.Debug("flushCompletedPendingFiles: added files to snapshot", + "duration", time.Since(opStart)) log.Debug("flushCompletedPendingFiles: transaction complete") @@ -670,6 +691,89 @@ func (s *Scanner) executeBatchFileFlush(ctx context.Context, allFiles []*databas }) } +// batchDeleteOldMappings removes stale file_chunks and chunk_files rows +// for the given files inside the flush transaction. +func (s *Scanner) batchDeleteOldMappings( + txCtx context.Context, tx *sql.Tx, allFileIDs []types.FileID, +) error { + // Batch delete old file_chunks and chunk_files + log.Debug("flushCompletedPendingFiles: deleting old file_chunks") + + opStart := time.Now() + + err := s.repos.FileChunks.DeleteByFileIDs(txCtx, tx, allFileIDs) + if err != nil { + return fmt.Errorf("batch deleting old file chunks: %w", err) + } + + log.Debug("flushCompletedPendingFiles: deleted file_chunks", + "duration", time.Since(opStart)) + + log.Debug("flushCompletedPendingFiles: deleting old chunk_files") + + opStart = time.Now() + + err = s.repos.ChunkFiles.DeleteByFileIDs(txCtx, tx, allFileIDs) + if err != nil { + return fmt.Errorf("batch deleting old chunk files: %w", err) + } + + log.Debug("flushCompletedPendingFiles: deleted chunk_files", + "duration", time.Since(opStart)) + + return nil +} + +// batchInsertFileData creates file records and their chunk association +// rows inside the flush transaction. +func (s *Scanner) batchInsertFileData( + txCtx context.Context, tx *sql.Tx, + allFiles []*database.File, + allFileChunks []database.FileChunk, + allChunkFiles []database.ChunkFile, +) error { + // Batch create/update file records + log.Debug("flushCompletedPendingFiles: creating files") + + opStart := time.Now() + + err := s.repos.Files.CreateBatch(txCtx, tx, allFiles) + if err != nil { + return fmt.Errorf("batch creating file records: %w", err) + } + + log.Debug("flushCompletedPendingFiles: created files", + "duration", time.Since(opStart)) + + // Batch insert file_chunks + log.Debug("flushCompletedPendingFiles: inserting file_chunks") + + opStart = time.Now() + + err = s.repos.FileChunks.CreateBatch(txCtx, tx, allFileChunks) + if err != nil { + return fmt.Errorf("batch creating file chunks: %w", err) + } + + log.Debug("flushCompletedPendingFiles: inserted file_chunks", + "duration", time.Since(opStart)) + + // Batch insert chunk_files + log.Debug("flushCompletedPendingFiles: inserting chunk_files") + + opStart = time.Now() + + err = s.repos.ChunkFiles.CreateBatch(txCtx, tx, allChunkFiles) + if err != nil { + return fmt.Errorf("batch creating chunk files: %w", err) + } + + log.Debug("flushCompletedPendingFiles: inserted chunk_files", + "duration", time.Since(opStart)) + + return nil +} + // ScanPhaseResult contains the results of the scan phase type ScanPhaseResult struct { FilesToProcess []*FileToProcess @@ -677,40 +781,31 @@ type ScanPhaseResult struct { } // scanPhase performs the initial directory scan to identify files to process -// It uses the pre-loaded knownFiles map for fast change detection without DB queries -// It also populates existingFiles map for deletion detection -// Returns files needing processing and IDs of unchanged files for snapshot association -func (s *Scanner) scanPhase(ctx context.Context, path string, result *ScanResult, existingFiles map[string]struct{}, knownFiles map[string]*database.File) (*ScanPhaseResult, error) { +// It uses the pre-loaded knownFiles map for fast change detection without DB +// queries. It also populates existingFiles map for deletion detection. +// Returns files needing processing and IDs of unchanged files for snapshot +// association. +func (s *Scanner) scanPhase( + ctx context.Context, + path string, + result *ScanResult, + existingFiles map[string]struct{}, + knownFiles map[string]*database.File, +) (*ScanPhaseResult, error) { // Use known file count as estimate for progress (accurate for subsequent backups) estimatedTotal := int64(len(knownFiles)) - var filesToProcess []*FileToProcess - - var unchangedFileIDs []types.FileID // Just IDs - no new records needed - - var mu sync.Mutex + collector := &scanCollector{} // Set up periodic status output startTime := time.Now() lastStatusTime := time.Now() - statusInterval := 15 * time.Second - - var filesScanned int64 log.Debug("Starting directory walk", "path", path) - err := afero.Walk(s.fs, path, func(filePath string, info os.FileInfo, err error) error { + walkFn := func(filePath string, info os.FileInfo, err error) error { if err != nil { - if s.skipErrors { - log.Error("Failed to access file (skipping due to --skip-errors)", "path", filePath, "error", err) - s.ui.Error("Failed to access %s: %v. Skipping (--skip-errors).", s.ui.Path(filePath), err) - - return nil // Continue scanning - } - - log.Debug("Error accessing filesystem entry", "path", filePath, "error", err) - - return wrapPermissionError(filePath, err) + return s.handleWalkError(filePath, err) } // Check context cancellation @@ -729,42 +824,9 @@ func (s *Scanner) scanPhase(ctx context.Context, path string, result *ScanResult return nil } - // Handle symlinks - if info.Mode()&os.ModeSymlink != 0 { - file := s.buildSymlinkEntry(filePath, info) - if file != nil { - existingFiles[filePath] = struct{}{} - - mu.Lock() - - filesToProcess = append(filesToProcess, &FileToProcess{ - Path: filePath, - FileInfo: info, - File: file, - }) - filesScanned++ - mu.Unlock() - s.updateScanEntryStats(result, true, info) - } - - return nil - } - - // Handle directories (record for permission/ownership preservation and empty-dir support) - if info.IsDir() { - file := s.buildDirectoryEntry(filePath, info) - existingFiles[filePath] = struct{}{} - - mu.Lock() - - filesToProcess = append(filesToProcess, &FileToProcess{ - Path: filePath, - FileInfo: info, - File: file, - }) - filesScanned++ - mu.Unlock() - + // Handle symlinks and directories + if handled := s.recordSpecialEntry( + filePath, info, existingFiles, collector, result); handled { return nil } @@ -779,48 +841,136 @@ func (s *Scanner) scanPhase(ctx context.Context, path string, result *ScanResult // Check file against in-memory map (no DB query!) file, needsProcessing := s.checkFileInMemory(filePath, info, knownFiles) - mu.Lock() - if needsProcessing { - // New or changed file - will create record after processing - filesToProcess = append(filesToProcess, &FileToProcess{ - Path: filePath, - FileInfo: info, - File: file, - }) - } else if !file.ID.IsZero() { - // Unchanged file with existing ID - just need snapshot association - unchangedFileIDs = append(unchangedFileIDs, file.ID) - } - - filesScanned++ - changedCount := len(filesToProcess) - mu.Unlock() + filesScanned, changedCount := collector.recordRegularFile( + filePath, info, file, needsProcessing) // Update result stats s.updateScanEntryStats(result, needsProcessing, info) // Output periodic status - if time.Since(lastStatusTime) >= statusInterval { + if time.Since(lastStatusTime) >= scanStatusInterval { s.printScanProgressLine(filesScanned, changedCount, estimatedTotal, startTime) lastStatusTime = time.Now() } return nil - }) + } + + err := afero.Walk(s.fs, path, walkFn) if err != nil { return nil, err } return &ScanPhaseResult{ - FilesToProcess: filesToProcess, - UnchangedFileIDs: unchangedFileIDs, + FilesToProcess: collector.filesToProcess, + UnchangedFileIDs: collector.unchangedFileIDs, }, nil } +// recordSpecialEntry records symlinks and directories (which have no +// data to chunk) and reports whether it handled the entry. +func (s *Scanner) recordSpecialEntry( + filePath string, info os.FileInfo, + existingFiles map[string]struct{}, + collector *scanCollector, result *ScanResult, +) bool { + // Handle symlinks + if info.Mode()&os.ModeSymlink != 0 { + file := s.buildSymlinkEntry(filePath, info) + if file != nil { + existingFiles[filePath] = struct{}{} + collector.addToProcess(filePath, info, file) + s.updateScanEntryStats(result, true, info) + } + + return true + } + + // Handle directories (record for permission/ownership preservation + // and empty-dir support) + if info.IsDir() { + file := s.buildDirectoryEntry(filePath, info) + existingFiles[filePath] = struct{}{} + collector.addToProcess(filePath, info, file) + + return true + } + + return false +} + +// handleWalkError deals with a filesystem error surfaced by the walk: +// skipped (with a UI notice) under --skip-errors, fatal otherwise. +func (s *Scanner) handleWalkError(filePath string, err error) error { + if s.skipErrors { + log.Error("Failed to access file (skipping due to --skip-errors)", + "path", filePath, "error", err) + s.ui.Errorf("Failed to access %s: %v. Skipping (--skip-errors).", + s.ui.Path(filePath), err) + + return nil // Continue scanning + } + + log.Debug("Error accessing filesystem entry", "path", filePath, "error", err) + + return wrapPermissionError(filePath, err) +} + +// scanCollector accumulates scan-phase walk results under a mutex. +type scanCollector struct { + mu sync.Mutex + filesToProcess []*FileToProcess + unchangedFileIDs []types.FileID // Just IDs - no new records needed + filesScanned int64 +} + +// addToProcess queues a file entry for processing. +func (c *scanCollector) addToProcess( + path string, info os.FileInfo, file *database.File, +) { + c.mu.Lock() + defer c.mu.Unlock() + + c.filesToProcess = append(c.filesToProcess, &FileToProcess{ + Path: path, + FileInfo: info, + File: file, + }) + c.filesScanned++ +} + +// recordRegularFile records a regular file as changed (to process) or +// unchanged (snapshot association only) and returns the running scanned +// and changed counts. +func (c *scanCollector) recordRegularFile( + path string, info os.FileInfo, file *database.File, needsProcessing bool, +) (int64, int) { + c.mu.Lock() + defer c.mu.Unlock() + + if needsProcessing { + // New or changed file - will create record after processing + c.filesToProcess = append(c.filesToProcess, &FileToProcess{ + Path: path, + FileInfo: info, + File: file, + }) + } else if !file.ID.IsZero() { + // Unchanged file with existing ID - just need snapshot association + c.unchangedFileIDs = append(c.unchangedFileIDs, file.ID) + } + + c.filesScanned++ + + return c.filesScanned, len(c.filesToProcess) +} + // updateScanEntryStats updates the scan result and progress reporter statistics // for a single scanned file entry based on whether it needs processing -func (s *Scanner) updateScanEntryStats(result *ScanResult, needsProcessing bool, info os.FileInfo) { +func (s *Scanner) updateScanEntryStats( + result *ScanResult, needsProcessing bool, info os.FileInfo, +) { if needsProcessing { result.BytesScanned += info.Size() if s.progress != nil { @@ -844,47 +994,55 @@ func (s *Scanner) updateScanEntryStats(result *ScanResult, needsProcessing bool, // printScanProgressLine prints a periodic progress line during the scan phase, // showing files scanned, percentage complete (if estimate available), and ETA -func (s *Scanner) printScanProgressLine(filesScanned int64, changedCount int, estimatedTotal int64, startTime time.Time) { +func (s *Scanner) printScanProgressLine( + filesScanned int64, changedCount int, estimatedTotal int64, startTime time.Time, +) { elapsed := time.Since(startTime) rate := float64(filesScanned) / elapsed.Seconds() - if estimatedTotal > 0 { - // Show actual scanned vs estimate (may exceed estimate if files were added) - pct := float64(filesScanned) / float64(estimatedTotal) * 100 - if pct > 100 { - pct = 100 // Cap at 100% for display - } - - remaining := max(estimatedTotal-filesScanned, 0) - - var eta time.Duration - if rate > 0 && remaining > 0 { - eta = time.Duration(float64(remaining)/rate) * time.Second - } - - if eta > 0 { - s.ui.Progress("Snapshot source files enumeration: %s files (~%s), %s changed or new, %.0f files/sec, enumeration elapsed: %s, enumeration ETA: %s (est remain %s).", - s.ui.Count(int(filesScanned)), - s.ui.Percent(pct), - s.ui.Count(changedCount), - rate, - s.ui.Duration(elapsed), - s.ui.Time(time.Now().Add(eta)), - s.ui.Duration(eta)) - } else { - s.ui.Progress("Snapshot source files enumeration: %s files (~%s), %s changed or new, %.0f files/sec, enumeration elapsed: %s.", - s.ui.Count(int(filesScanned)), - s.ui.Percent(pct), - s.ui.Count(changedCount), - rate, - s.ui.Duration(elapsed)) - } - } else { - s.ui.Progress("Snapshot source files enumeration: %s files seen, %s changed or new, %.0f files/sec, enumeration elapsed: %s.", + if estimatedTotal <= 0 { + s.ui.Progressf("Snapshot source files enumeration: %s files seen, "+ + "%s changed or new, %.0f files/sec, enumeration elapsed: %s.", s.ui.Count(int(filesScanned)), s.ui.Count(changedCount), rate, s.ui.Duration(elapsed)) + + return + } + + // Show actual scanned vs estimate (may exceed estimate if files were added) + pct := float64(filesScanned) / float64(estimatedTotal) * percentScale + if pct > percentScale { + pct = percentScale // Cap at 100% for display + } + + remaining := max(estimatedTotal-filesScanned, 0) + + var eta time.Duration + if rate > 0 && remaining > 0 { + eta = time.Duration(float64(remaining)/rate) * time.Second + } + + if eta > 0 { + s.ui.Progressf("Snapshot source files enumeration: %s files (~%s), "+ + "%s changed or new, %.0f files/sec, enumeration elapsed: %s, "+ + "enumeration ETA: %s (est remain %s).", + s.ui.Count(int(filesScanned)), + s.ui.Percent(pct), + s.ui.Count(changedCount), + rate, + s.ui.Duration(elapsed), + s.ui.Time(time.Now().Add(eta)), + s.ui.Duration(eta)) + } else { + s.ui.Progressf("Snapshot source files enumeration: %s files (~%s), "+ + "%s changed or new, %.0f files/sec, enumeration elapsed: %s.", + s.ui.Count(int(filesScanned)), + s.ui.Percent(pct), + s.ui.Count(changedCount), + rate, + s.ui.Duration(elapsed)) } } @@ -958,7 +1116,9 @@ func (s *Scanner) recordNonRegularFile(ctx context.Context, ftp *FileToProcess) // checkFileInMemory checks if a file needs processing using the in-memory map // No database access is performed - this is purely CPU/memory work -func (s *Scanner) checkFileInMemory(path string, info os.FileInfo, knownFiles map[string]*database.File) (*database.File, bool) { +func (s *Scanner) checkFileInMemory( + path string, info os.FileInfo, knownFiles map[string]*database.File, +) (*database.File, bool) { // Get file stats stat, ok := info.Sys().(interface { Uid() uint32 @@ -985,9 +1145,10 @@ func (s *Scanner) checkFileInMemory(path string, info os.FileInfo, knownFiles ma } file := &database.File{ - ID: fileID, - Path: types.FilePath(path), - SourcePath: types.SourcePath(s.currentSourcePath), // Store source directory for restore path stripping + ID: fileID, + Path: types.FilePath(path), + // Store source directory for restore path stripping + SourcePath: types.SourcePath(s.currentSourcePath), MTime: info.ModTime(), Size: info.Size(), Mode: uint32(info.Mode()), @@ -1015,14 +1176,13 @@ func (s *Scanner) checkFileInMemory(path string, info os.FileInfo, knownFiles ma // batchAddFilesToSnapshot adds existing file IDs to the snapshot association table // This is used for unchanged files that already have records in the database -func (s *Scanner) batchAddFilesToSnapshot(ctx context.Context, fileIDs []types.FileID) error { - const batchSize = 1000 - +func (s *Scanner) batchAddFilesToSnapshot( + ctx context.Context, fileIDs []types.FileID, +) error { startTime := time.Now() lastStatusTime := time.Now() - statusInterval := 5 * time.Second - for i := 0; i < len(fileIDs); i += batchSize { + for i := 0; i < len(fileIDs); i += snapshotAssocBatchSize { // Check context cancellation select { case <-ctx.Done(): @@ -1030,7 +1190,7 @@ func (s *Scanner) batchAddFilesToSnapshot(ctx context.Context, fileIDs []types.F default: } - end := min(i+batchSize, len(fileIDs)) + end := min(i+snapshotAssocBatchSize, len(fileIDs)) batch := fileIDs[i:end] @@ -1049,11 +1209,11 @@ func (s *Scanner) batchAddFilesToSnapshot(ctx context.Context, fileIDs []types.F } // Periodic status - if time.Since(lastStatusTime) >= statusInterval { + if time.Since(lastStatusTime) >= assocStatusInterval { elapsed := time.Since(startTime) rate := float64(end) / elapsed.Seconds() - pct := float64(end) / float64(len(fileIDs)) * 100 - s.ui.Progress("Snapshot unchanged-file association: %s/%s (%s), %.0f files/sec.", + pct := float64(end) / float64(len(fileIDs)) * percentScale + s.ui.Progressf("Snapshot unchanged-file association: %s/%s (%s), %.0f files/sec.", s.ui.Count(end), s.ui.Count(len(fileIDs)), s.ui.Percent(pct), rate) lastStatusTime = time.Now() @@ -1062,14 +1222,17 @@ func (s *Scanner) batchAddFilesToSnapshot(ctx context.Context, fileIDs []types.F elapsed := time.Since(startTime) rate := float64(len(fileIDs)) / elapsed.Seconds() - s.ui.Complete("Associated %s unchanged files with the snapshot in %s (%.0f files/sec).", + s.ui.Completef("Associated %s unchanged files with the snapshot in %s "+ + "(%.0f files/sec).", s.ui.Count(len(fileIDs)), s.ui.Duration(elapsed), rate) return nil } // processPhase processes the files that need backing up -func (s *Scanner) processPhase(ctx context.Context, filesToProcess []*FileToProcess, result *ScanResult) error { +func (s *Scanner) processPhase( + ctx context.Context, filesToProcess []*FileToProcess, result *ScanResult, +) error { // Calculate total bytes to process var totalBytes int64 for _, f := range filesToProcess { @@ -1078,7 +1241,6 @@ func (s *Scanner) processPhase(ctx context.Context, filesToProcess []*FileToProc // Set up periodic status output lastStatusTime := time.Now() - statusInterval := 15 * time.Second startTime := time.Now() filesProcessed := 0 @@ -1112,8 +1274,9 @@ func (s *Scanner) processPhase(ctx context.Context, filesToProcess []*FileToProc bytesProcessed += fileToProcess.FileInfo.Size() // Output periodic status - if time.Since(lastStatusTime) >= statusInterval { - s.printProcessingProgress(filesProcessed, totalFiles, bytesProcessed, totalBytes, startTime) + if time.Since(lastStatusTime) >= scanStatusInterval { + s.printProcessingProgress( + filesProcessed, totalFiles, bytesProcessed, totalBytes, startTime) lastStatusTime = time.Now() } @@ -1125,12 +1288,15 @@ func (s *Scanner) processPhase(ctx context.Context, filesToProcess []*FileToProc // processFileWithErrorHandling wraps processFileStreaming with error recovery for // deleted files and skip-errors mode. Returns (skipped, error). -func (s *Scanner) processFileWithErrorHandling(ctx context.Context, fileToProcess *FileToProcess, result *ScanResult) (bool, error) { +func (s *Scanner) processFileWithErrorHandling( + ctx context.Context, fileToProcess *FileToProcess, result *ScanResult, +) (bool, error) { err := s.processFileStreaming(ctx, fileToProcess, result) if err != nil { // Handle files that were deleted between scan and process phases if errors.Is(err, os.ErrNotExist) { - log.Warn("File was deleted during backup, skipping", "path", fileToProcess.Path) + log.Warn("File was deleted during backup, skipping", + "path", fileToProcess.Path) result.FilesSkipped++ @@ -1138,8 +1304,10 @@ func (s *Scanner) processFileWithErrorHandling(ctx context.Context, fileToProces } // Skip file read errors if --skip-errors is enabled if s.skipErrors { - log.Error("Failed to process file (skipping due to --skip-errors)", "path", fileToProcess.Path, "error", err) - s.ui.Error("Failed to process %s: %v. Skipping (--skip-errors).", s.ui.Path(fileToProcess.Path), err) + log.Error("Failed to process file (skipping due to --skip-errors)", + "path", fileToProcess.Path, "error", err) + s.ui.Errorf("Failed to process %s: %v. Skipping (--skip-errors).", + s.ui.Path(fileToProcess.Path), err) result.FilesSkipped++ @@ -1154,9 +1322,12 @@ func (s *Scanner) processFileWithErrorHandling(ctx context.Context, fileToProces // printProcessingProgress prints a periodic progress line during the process phase, // showing files processed, bytes transferred, throughput, and ETA -func (s *Scanner) printProcessingProgress(filesProcessed, totalFiles int, bytesProcessed, totalBytes int64, startTime time.Time) { +func (s *Scanner) printProcessingProgress( + filesProcessed, totalFiles int, bytesProcessed, totalBytes int64, + startTime time.Time, +) { elapsed := time.Since(startTime) - pct := float64(bytesProcessed) / float64(totalBytes) * 100 + pct := float64(bytesProcessed) / float64(totalBytes) * percentScale byteRate := float64(bytesProcessed) / elapsed.Seconds() fileRate := float64(filesProcessed) / elapsed.Seconds() @@ -1169,7 +1340,8 @@ func (s *Scanner) printProcessingProgress(filesProcessed, totalFiles int, bytesP } if eta > 0 { - s.ui.Progress("Snapshot backup: %s/%s files (%s), %s/%s, %s, %.0f files/sec, backup elapsed: %s, backup ETA: %s (est remain %s).", + s.ui.Progressf("Snapshot backup: %s/%s files (%s), %s/%s, %s, "+ + "%.0f files/sec, backup elapsed: %s, backup ETA: %s (est remain %s).", s.ui.Count(filesProcessed), s.ui.Count(totalFiles), s.ui.Percent(pct), @@ -1181,7 +1353,8 @@ func (s *Scanner) printProcessingProgress(filesProcessed, totalFiles int, bytesP s.ui.Time(time.Now().Add(eta)), s.ui.Duration(eta)) } else { - s.ui.Progress("Snapshot backup: %s/%s files (%s), %s/%s, %s, %.0f files/sec, backup elapsed: %s.", + s.ui.Progressf("Snapshot backup: %s/%s files (%s), %s/%s, %s, "+ + "%.0f files/sec, backup elapsed: %s.", s.ui.Count(filesProcessed), s.ui.Count(totalFiles), s.ui.Percent(pct), @@ -1193,14 +1366,15 @@ func (s *Scanner) printProcessingProgress(filesProcessed, totalFiles int, bytesP } } -// finalizeProcessPhase flushes the packer, writes remaining pending files to the database, -// and handles local blob storage when no remote storage is configured +// finalizeProcessPhase flushes the packer, writes remaining pending files to +// the database, and handles local blob storage when no remote storage is +// configured func (s *Scanner) finalizeProcessPhase(ctx context.Context, result *ScanResult) error { // Final packer flush first - this commits remaining chunks to DB // and handleBlobReady will flush files whose chunks are now committed s.packerMu.Lock() - err := s.packer.Flush() + err := s.packer.Flush(ctx) if err != nil { s.packerMu.Unlock() @@ -1208,8 +1382,8 @@ func (s *Scanner) finalizeProcessPhase(ctx context.Context, result *ScanResult) } s.packerMu.Unlock() - // Flush any remaining pending files (e.g., files with only pre-existing chunks - // that didn't trigger a blob finalize) + // Flush any remaining pending files (e.g., files with only pre-existing + // chunks that didn't trigger a blob finalize) err = s.flushAllPending(ctx) if err != nil { return fmt.Errorf("flushing remaining pending files: %w", err) @@ -1227,7 +1401,8 @@ func (s *Scanner) finalizeProcessPhase(ctx context.Context, result *ScanResult) } err = s.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { - return s.repos.Snapshots.AddBlob(ctx, tx, s.snapshotID, blobID, types.BlobHash(b.Hash)) + return s.repos.Snapshots.AddBlob(ctx, tx, s.snapshotID, blobID, + types.BlobHash(b.Hash)) }) if err != nil { return fmt.Errorf("storing blob metadata: %w", err) @@ -1241,7 +1416,7 @@ func (s *Scanner) finalizeProcessPhase(ctx context.Context, result *ScanResult) } // handleBlobReady is called by the packer when a blob is finalized -func (s *Scanner) handleBlobReady(blobWithReader *blob.BlobWithReader) error { +func (s *Scanner) handleBlobReady(blobWithReader *blob.WithReader) error { startTime := time.Now().UTC() finishedBlob := blobWithReader.FinishedBlob @@ -1255,7 +1430,8 @@ func (s *Scanner) handleBlobReady(blobWithReader *blob.BlobWithReader) error { ctx = context.Background() } - blobPath := fmt.Sprintf("blobs/%s/%s/%s", finishedBlob.Hash[:2], finishedBlob.Hash[2:4], finishedBlob.Hash) + blobPath := fmt.Sprintf("blobs/%s/%s/%s", + finishedBlob.Hash[:2], finishedBlob.Hash[2:4], finishedBlob.Hash) blobExists, err := s.uploadBlobIfNeeded(ctx, blobPath, blobWithReader, startTime) if err != nil { @@ -1285,8 +1461,14 @@ func (s *Scanner) handleBlobReady(blobWithReader *blob.BlobWithReader) error { return nil } -// uploadBlobIfNeeded uploads the blob to storage if it doesn't already exist, returns whether it existed -func (s *Scanner) uploadBlobIfNeeded(ctx context.Context, blobPath string, blobWithReader *blob.BlobWithReader, startTime time.Time) (bool, error) { +// uploadBlobIfNeeded uploads the blob to storage if it doesn't already +// exist, returns whether it existed +func (s *Scanner) uploadBlobIfNeeded( + ctx context.Context, + blobPath string, + blobWithReader *blob.WithReader, + startTime time.Time, +) (bool, error) { finishedBlob := blobWithReader.FinishedBlob // Check if blob already exists (deduplication after restart) @@ -1295,19 +1477,23 @@ func (s *Scanner) uploadBlobIfNeeded(ctx context.Context, blobPath string, blobW _, err := s.storage.Stat(ctx, blobPath) if err == nil { log.Info("Blob already exists in storage, skipping upload", - "hash", finishedBlob.Hash, "size", humanize.Bytes(uint64(finishedBlob.Compressed))) - s.ui.Info("Blob %s (%s) already exists at %s. Skipping upload.", - s.ui.Hex(finishedBlob.Hash), s.ui.Size(finishedBlob.Compressed), s.ui.Path(destination)) + "hash", finishedBlob.Hash, + "size", humanize.Bytes(safeUint64(finishedBlob.Compressed))) + s.ui.Infof("Blob %s (%s) already exists at %s. Skipping upload.", + s.ui.Hex(finishedBlob.Hash), s.ui.Size(finishedBlob.Compressed), + s.ui.Path(destination)) return true, nil } - s.ui.Begin("Uploading blob %s (%s) to %s.", - s.ui.Hex(finishedBlob.Hash), s.ui.Size(finishedBlob.Compressed), s.ui.Path(destination)) + s.ui.Beginf("Uploading blob %s (%s) to %s.", + s.ui.Hex(finishedBlob.Hash), s.ui.Size(finishedBlob.Compressed), + s.ui.Path(destination)) progressCallback := s.makeUploadProgressCallback(ctx, finishedBlob, startTime) - err = s.storage.PutWithProgress(ctx, blobPath, blobWithReader.Reader, finishedBlob.Compressed, progressCallback) + err = s.storage.PutWithProgress(ctx, blobPath, blobWithReader.Reader, + finishedBlob.Compressed, progressCallback) if err != nil { log.Error("Failed to upload blob", "hash", finishedBlob.Hash, "error", err) @@ -1317,7 +1503,7 @@ func (s *Scanner) uploadBlobIfNeeded(ctx context.Context, blobPath string, blobW uploadDuration := time.Since(startTime) uploadSpeedBps := float64(finishedBlob.Compressed) / uploadDuration.Seconds() - s.ui.Complete("Uploaded blob %s (%s) in %s at %s.", + s.ui.Completef("Uploaded blob %s (%s) in %s at %s.", s.ui.Hex(finishedBlob.Hash), s.ui.Size(finishedBlob.Compressed), s.ui.Duration(uploadDuration), @@ -1325,12 +1511,14 @@ func (s *Scanner) uploadBlobIfNeeded(ctx context.Context, blobPath string, blobW log.Info("Successfully uploaded blob to storage", "path", blobPath, - "size", humanize.Bytes(uint64(finishedBlob.Compressed)), + "size", humanize.Bytes(safeUint64(finishedBlob.Compressed)), "duration", uploadDuration, - "speed", humanize.SI(uploadSpeedBps*8, "bps")) + "speed", humanize.SI(uploadSpeedBps*bitsPerByte, "bps")) if s.progress != nil { - s.progress.ReportUploadComplete(finishedBlob.Hash, finishedBlob.Compressed, uploadDuration) + s.progress.ReportUploadComplete(finishedBlob.Hash, finishedBlob.Compressed, + uploadDuration) + stats := s.progress.GetStats() stats.BlobsUploaded.Add(1) stats.BytesUploaded.Add(finishedBlob.Compressed) @@ -1342,7 +1530,9 @@ func (s *Scanner) uploadBlobIfNeeded(ctx context.Context, blobPath string, blobW // makeUploadProgressCallback creates a progress callback for blob uploads. // It updates the live progress reporter ~twice/sec for ETAs and prints a // human-readable status line to s.output at most every 15 seconds. -func (s *Scanner) makeUploadProgressCallback(ctx context.Context, finishedBlob *blob.FinishedBlob, uploadStart time.Time) func(int64) error { +func (s *Scanner) makeUploadProgressCallback( + ctx context.Context, finishedBlob *blob.FinishedBlob, uploadStart time.Time, +) func(int64) error { lastProgressTime := time.Now() lastProgressBytes := int64(0) lastStdoutTime := time.Now() @@ -1353,12 +1543,13 @@ func (s *Scanner) makeUploadProgressCallback(ctx context.Context, finishedBlob * now := time.Now() elapsed := now.Sub(lastProgressTime).Seconds() - if elapsed > 0.5 { + if elapsed > uploadProgressMinSeconds { bytesSinceLastUpdate := uploaded - lastProgressBytes speed := float64(bytesSinceLastUpdate) / elapsed if s.progress != nil { - s.progress.ReportUploadProgress(finishedBlob.Hash, uploaded, finishedBlob.Compressed, speed) + s.progress.ReportUploadProgress(finishedBlob.Hash, uploaded, + finishedBlob.Compressed, speed) } lastProgressTime = now @@ -1368,15 +1559,17 @@ func (s *Scanner) makeUploadProgressCallback(ctx context.Context, finishedBlob * // Periodic stdout status line so the user knows the upload is alive. if now.Sub(lastStdoutTime) >= stdoutInterval { totalElapsed := now.Sub(uploadStart) - pct := float64(uploaded) / float64(finishedBlob.Compressed) * 100 + pct := float64(uploaded) / float64(finishedBlob.Compressed) * percentScale avgSpeed := float64(uploaded) / totalElapsed.Seconds() var eta time.Duration if avgSpeed > 0 { - eta = time.Duration(float64(finishedBlob.Compressed-uploaded)/avgSpeed) * time.Second + eta = time.Duration( + float64(finishedBlob.Compressed-uploaded)/avgSpeed) * time.Second } - s.ui.Progress("Blob upload %s: %s / %s (%s) at %s, blob upload elapsed: %s, blob upload ETA: %s (est remain %s).", + s.ui.Progressf("Blob upload %s: %s / %s (%s) at %s, "+ + "blob upload elapsed: %s, blob upload ETA: %s (est remain %s).", s.ui.Hex(finishedBlob.Hash), s.ui.Size(uploaded), s.ui.Size(finishedBlob.Compressed), @@ -1398,7 +1591,12 @@ func (s *Scanner) makeUploadProgressCallback(ctx context.Context, finishedBlob * } // recordBlobMetadata stores blob upload metadata in the database -func (s *Scanner) recordBlobMetadata(ctx context.Context, finishedBlob *blob.FinishedBlob, blobExists bool, startTime time.Time) error { +func (s *Scanner) recordBlobMetadata( + ctx context.Context, + finishedBlob *blob.FinishedBlob, + blobExists bool, + startTime time.Time, +) error { finishedBlobID, err := types.ParseBlobID(finishedBlob.ID) if err != nil { return fmt.Errorf("parsing finished blob ID: %w", err) @@ -1412,7 +1610,8 @@ func (s *Scanner) recordBlobMetadata(ctx context.Context, finishedBlob *blob.Fin return fmt.Errorf("updating blob upload timestamp: %w", err) } - err = s.repos.Snapshots.AddBlob(txCtx, tx, s.snapshotID, finishedBlobID, types.BlobHash(finishedBlob.Hash)) + err = s.repos.Snapshots.AddBlob(txCtx, tx, s.snapshotID, finishedBlobID, + types.BlobHash(finishedBlob.Hash)) if err != nil { return fmt.Errorf("adding blob to snapshot: %w", err) } @@ -1437,7 +1636,7 @@ func (s *Scanner) recordBlobMetadata(ctx context.Context, finishedBlob *blob.Fin } // cleanupBlobTempFile closes and removes the blob's temporary file -func (s *Scanner) cleanupBlobTempFile(blobWithReader *blob.BlobWithReader) { +func (s *Scanner) cleanupBlobTempFile(blobWithReader *blob.WithReader) { if blobWithReader.TempFile != nil { tempName := blobWithReader.TempFile.Name() @@ -1461,7 +1660,9 @@ type streamingChunkInfo struct { } // processFileStreaming processes a file by streaming chunks directly to the packer -func (s *Scanner) processFileStreaming(ctx context.Context, fileToProcess *FileToProcess, result *ScanResult) error { +func (s *Scanner) processFileStreaming( + ctx context.Context, fileToProcess *FileToProcess, result *ScanResult, +) error { // Symlinks and directories have no data to chunk — just record them in the DB. mode := os.FileMode(fileToProcess.File.Mode) if mode&os.ModeSymlink != 0 || mode.IsDir() { @@ -1505,7 +1706,7 @@ func (s *Scanner) processFileStreaming(ctx context.Context, fileToProcess *FileT s.updateChunkStats(chunkExists, chunk.Size, result) if !chunkExists { - err := s.addChunkToPacker(chunk) + err := s.addChunkToPacker(ctx, chunk) if err != nil { return err } @@ -1529,7 +1730,9 @@ func (s *Scanner) processFileStreaming(ctx context.Context, fileToProcess *FileT } // updateChunkStats updates scan result and progress stats for a processed chunk -func (s *Scanner) updateChunkStats(chunkExists bool, chunkSize int64, result *ScanResult) { +func (s *Scanner) updateChunkStats( + chunkExists bool, chunkSize int64, result *ScanResult, +) { if chunkExists { result.FilesSkipped++ @@ -1549,20 +1752,21 @@ func (s *Scanner) updateChunkStats(chunkExists bool, chunkSize int64, result *Sc } } -// addChunkToPacker adds a chunk to the blob packer, finalizing the current blob if needed -func (s *Scanner) addChunkToPacker(chunk chunker.Chunk) error { +// addChunkToPacker adds a chunk to the blob packer, finalizing the current +// blob if needed +func (s *Scanner) addChunkToPacker(ctx context.Context, chunk chunker.Chunk) error { s.packerMu.Lock() - err := s.packer.AddChunk(&blob.ChunkRef{Hash: chunk.Hash, Data: chunk.Data}) + err := s.packer.AddChunk(ctx, &blob.ChunkRef{Hash: chunk.Hash, Data: chunk.Data}) if errors.Is(err, blob.ErrBlobSizeLimitExceeded) { - err := s.packer.FinalizeBlob() + err = s.packer.FinalizeBlob(ctx) if err != nil { s.packerMu.Unlock() return fmt.Errorf("finalizing blob: %w", err) } - err = s.packer.AddChunk(&blob.ChunkRef{Hash: chunk.Hash, Data: chunk.Data}) + err = s.packer.AddChunk(ctx, &blob.ChunkRef{Hash: chunk.Hash, Data: chunk.Data}) if err != nil { s.packerMu.Unlock() @@ -1578,8 +1782,11 @@ func (s *Scanner) addChunkToPacker(chunk chunker.Chunk) error { return nil } -// queueFileForBatchInsert builds file/chunk associations and queues the file for batch DB insert -func (s *Scanner) queueFileForBatchInsert(ctx context.Context, fileToProcess *FileToProcess, chunks []streamingChunkInfo) { +// queueFileForBatchInsert builds file/chunk associations and queues the +// file for batch DB insert +func (s *Scanner) queueFileForBatchInsert( + ctx context.Context, fileToProcess *FileToProcess, chunks []streamingChunkInfo, +) { fileChunks := make([]database.FileChunk, len(chunks)) chunkFiles := make([]database.ChunkFile, len(chunks)) @@ -1604,14 +1811,15 @@ func (s *Scanner) queueFileForBatchInsert(ctx context.Context, fileToProcess *Fi }) } -// GetProgress returns the progress reporter for this scanner -func (s *Scanner) GetProgress() *ProgressReporter { - return s.progress -} - -// detectDeletedFilesFromMap finds files that existed in previous snapshots but no longer exist -// Uses pre-loaded maps to avoid any filesystem or database access -func (s *Scanner) detectDeletedFilesFromMap(ctx context.Context, knownFiles map[string]*database.File, existingFiles map[string]struct{}, result *ScanResult) error { +// detectDeletedFilesFromMap finds files that existed in previous snapshots +// but no longer exist. Uses pre-loaded maps to avoid any filesystem or +// database access. +func (s *Scanner) detectDeletedFilesFromMap( + ctx context.Context, + knownFiles map[string]*database.File, + existingFiles map[string]struct{}, + result *ScanResult, +) error { if len(knownFiles) == 0 { return nil } @@ -1635,7 +1843,8 @@ func (s *Scanner) detectDeletedFilesFromMap(ctx context.Context, knownFiles map[ } if result.FilesDeleted > 0 { - s.ui.Info("Snapshot source files enumeration detected %s deleted files.", s.ui.Count(result.FilesDeleted)) + s.ui.Infof("Snapshot source files enumeration detected %s deleted files.", + s.ui.Count(result.FilesDeleted)) } return nil @@ -1658,7 +1867,8 @@ func wrapPermissionError(path string, err error) error { "then quit and reopen the terminal and re-run the backup", path, err) } - return fmt.Errorf("cannot read %s: %w (check file permissions, or run with --skip-errors to continue past unreadable files)", path, err) + return fmt.Errorf("cannot read %s: %w (check file permissions, or run "+ + "with --skip-errors to continue past unreadable files)", path, err) } // compileExcludePatterns compiles the exclude patterns into glob matchers @@ -1725,39 +1935,66 @@ func (s *Scanner) shouldExclude(filePath, rootPath string) bool { // Check each pattern for _, cp := range s.compiledExclude { - if cp.anchored { - // Anchored pattern: must match from the root - // Match the relative path directly - if cp.pattern.Match(relPath) { - return true - } - // Also check if any prefix of the path matches (for directory patterns) - parts := strings.Split(relPath, "/") - for i := 1; i <= len(parts); i++ { - prefix := strings.Join(parts[:i], "/") - if cp.pattern.Match(prefix) { - return true - } - } - } else { - // Unanchored pattern: can match anywhere in path - // Check the full relative path - if cp.pattern.Match(relPath) { - return true - } - // Check each path component and subpath - parts := strings.Split(relPath, "/") - for i := range parts { - // Match individual component (e.g., ".git" matches ".git" directory) - if cp.pattern.Match(parts[i]) { - return true - } - // Match subpath from this component onwards - subpath := strings.Join(parts[i:], "/") - if cp.pattern.Match(subpath) { - return true - } - } + if cp.matches(relPath) { + return true + } + } + + return false +} + +// matches reports whether the compiled pattern matches the given +// slash-normalized relative path, honoring anchoring. +func (cp compiledPattern) matches(relPath string) bool { + if cp.anchored { + return cp.matchesAnchored(relPath) + } + + return cp.matchesUnanchored(relPath) +} + +// matchesAnchored matches the pattern from the root of the source +// directory: the relative path itself or any of its leading directory +// prefixes must match. +func (cp compiledPattern) matchesAnchored(relPath string) bool { + // Match the relative path directly + if cp.pattern.Match(relPath) { + return true + } + + // Also check if any prefix of the path matches (for directory patterns) + parts := strings.Split(relPath, "/") + for i := 1; i <= len(parts); i++ { + prefix := strings.Join(parts[:i], "/") + if cp.pattern.Match(prefix) { + return true + } + } + + return false +} + +// matchesUnanchored matches the pattern anywhere in the path: against the +// full relative path, each individual component, and every trailing +// subpath. +func (cp compiledPattern) matchesUnanchored(relPath string) bool { + // Check the full relative path + if cp.pattern.Match(relPath) { + return true + } + + // Check each path component and subpath + parts := strings.Split(relPath, "/") + for i := range parts { + // Match individual component (e.g., ".git" matches ".git" directory) + if cp.pattern.Match(parts[i]) { + return true + } + + // Match subpath from this component onwards + subpath := strings.Join(parts[i:], "/") + if cp.pattern.Match(subpath) { + return true } } diff --git a/internal/snapshot/scanner_test.go b/internal/snapshot/scanner_test.go index 617067b..96c3e83 100644 --- a/internal/snapshot/scanner_test.go +++ b/internal/snapshot/scanner_test.go @@ -3,6 +3,7 @@ package snapshot_test import ( "context" "database/sql" + "os" "path/filepath" "testing" "time" @@ -14,9 +15,114 @@ import ( "sneak.berlin/go/vaultik/internal/types" ) +// Shared test fixture values for the snapshot_test package. +const ( + // testHost is the hostname recorded on test snapshot rows. + testHost = "test-host" + + // testVersion is the vaultik version recorded on test snapshot rows. + testVersion = "test" + + // testAgePublicKey is the fixed age public key used for test encryption. + testAgePublicKey = "age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05gl0sjq9q9wjg" +) + +// TestMain initializes the shared logger once, before any tests run, so +// parallel tests never race on the logger's global state. +func TestMain(m *testing.M) { + log.Initialize(log.Config{}) + os.Exit(m.Run()) +} + +// createTestSnapshotRecord inserts an empty snapshot row used as the +// association target for scan tests. +func createTestSnapshotRecord( + ctx context.Context, t *testing.T, repos *database.Repositories, snapshotID string, +) { + t.Helper() + + err := repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { + snapshot := &database.Snapshot{ + ID: types.SnapshotID(snapshotID), + Hostname: testHost, + VaultikVersion: testVersion, + StartedAt: time.Now(), + CompletedAt: nil, + FileCount: 0, + ChunkCount: 0, + BlobCount: 0, + TotalSize: 0, + BlobSize: 0, + CompressionRatio: 1.0, + } + + return repos.Snapshots.Create(ctx, tx, snapshot) + }) + if err != nil { + t.Fatalf("failed to create snapshot: %v", err) + } +} + +// verifySimpleScanDatabase checks the database contents produced by +// TestScannerSimpleDirectory's scan. +func verifySimpleScanDatabase( + ctx context.Context, t *testing.T, repos *database.Repositories, +) { + t.Helper() + + // Verify files in database - includes regular files and directories + files, err := repos.Files.ListByPrefix(ctx, "/source") + if err != nil { + t.Fatalf("failed to list files: %v", err) + } + + // 6 regular files + 3 directories (/source, /source/subdir, /source/subdir2) + if len(files) != 9 { + t.Errorf("expected 9 entries in database (6 files + 3 dirs), got %d", len(files)) + } + + // Verify specific file + file1, err := repos.Files.GetByPath(ctx, "/source/file1.txt") + if err != nil { + t.Fatalf("failed to get file1.txt: %v", err) + } + + if file1.Size != 13 { + t.Errorf("expected file1.txt size 13, got %d", file1.Size) + } + + if file1.Mode != 0644 { + t.Errorf("expected file1.txt mode 0644, got %o", file1.Mode) + } + + // Verify chunks were created + chunks, err := repos.FileChunks.GetByFile(ctx, "/source/file1.txt") + if err != nil { + t.Fatalf("failed to get chunks for file1.txt: %v", err) + } + + if len(chunks) != 1 { // Small file should be one chunk + t.Errorf("expected 1 chunk for file1.txt, got %d", len(chunks)) + } + + // Verify deduplication - file3.txt and file4.txt have different content + // but we should still have the correct number of unique chunks + allChunks, err := repos.Chunks.List(ctx) + if err != nil { + t.Fatalf("failed to list all chunks: %v", err) + } + + // We should have at most 6 chunks (one per unique file content) + // Empty file might not create a chunk + if len(allChunks) > 6 { + t.Errorf("expected at most 6 chunks, got %d", len(allChunks)) + } +} + func TestScannerSimpleDirectory(t *testing.T) { // Initialize logger for tests log.Initialize(log.Config{}) + t.Parallel() // Create in-memory filesystem fs := afero.NewMemMapFs() @@ -74,38 +180,17 @@ func TestScannerSimpleDirectory(t *testing.T) { Repositories: repos, MaxBlobSize: int64(1024 * 1024), // 1MB blobs CompressionLevel: 3, - AgeRecipients: []string{"age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05gl0sjq9q9wjg"}, // Test public key + AgeRecipients: []string{testAgePublicKey}, }) // Create a snapshot record for testing ctx := context.Background() snapshotID := "test-snapshot-001" - err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { - snapshot := &database.Snapshot{ - ID: types.SnapshotID(snapshotID), - Hostname: "test-host", - VaultikVersion: "test", - StartedAt: time.Now(), - CompletedAt: nil, - FileCount: 0, - ChunkCount: 0, - BlobCount: 0, - TotalSize: 0, - BlobSize: 0, - CompressionRatio: 1.0, - } - - return repos.Snapshots.Create(ctx, tx, snapshot) - }) - if err != nil { - t.Fatalf("failed to create snapshot: %v", err) - } + createTestSnapshotRecord(ctx, t, repos, snapshotID) // Scan the directory - var result *snapshot.ScanResult - - result, err = scanner.Scan(ctx, "/source", snapshotID) + result, err := scanner.Scan(ctx, "/source", snapshotID) if err != nil { t.Fatalf("scan failed: %v", err) } @@ -120,58 +205,13 @@ func TestScannerSimpleDirectory(t *testing.T) { t.Errorf("expected at least 97 bytes scanned, got %d", result.BytesScanned) } - // Verify files in database - includes regular files and directories - files, err := repos.Files.ListByPrefix(ctx, "/source") - if err != nil { - t.Fatalf("failed to list files: %v", err) - } - - // 6 regular files + 3 directories (/source, /source/subdir, /source/subdir2) - if len(files) != 9 { - t.Errorf("expected 9 entries in database (6 files + 3 dirs), got %d", len(files)) - } - - // Verify specific file - file1, err := repos.Files.GetByPath(ctx, "/source/file1.txt") - if err != nil { - t.Fatalf("failed to get file1.txt: %v", err) - } - - if file1.Size != 13 { - t.Errorf("expected file1.txt size 13, got %d", file1.Size) - } - - if file1.Mode != 0644 { - t.Errorf("expected file1.txt mode 0644, got %o", file1.Mode) - } - - // Verify chunks were created - chunks, err := repos.FileChunks.GetByFile(ctx, "/source/file1.txt") - if err != nil { - t.Fatalf("failed to get chunks for file1.txt: %v", err) - } - - if len(chunks) != 1 { // Small file should be one chunk - t.Errorf("expected 1 chunk for file1.txt, got %d", len(chunks)) - } - - // Verify deduplication - file3.txt and file4.txt have different content - // but we should still have the correct number of unique chunks - allChunks, err := repos.Chunks.List(ctx) - if err != nil { - t.Fatalf("failed to list all chunks: %v", err) - } - - // We should have at most 6 chunks (one per unique file content) - // Empty file might not create a chunk - if len(allChunks) > 6 { - t.Errorf("expected at most 6 chunks, got %d", len(allChunks)) - } + verifySimpleScanDatabase(ctx, t, repos) } func TestScannerLargeFile(t *testing.T) { // Initialize logger for tests log.Initialize(log.Config{}) + t.Parallel() // Create in-memory filesystem fs := afero.NewMemMapFs() @@ -182,6 +222,7 @@ func TestScannerLargeFile(t *testing.T) { // Fill with pseudo-random data to ensure chunk boundaries for i := range largeContent { // Simple pseudo-random generator for deterministic tests + //nolint:gosec // G115: intentional byte truncation of test data largeContent[i] = byte((i * 7919) ^ (i >> 3)) } @@ -216,38 +257,17 @@ func TestScannerLargeFile(t *testing.T) { Repositories: repos, MaxBlobSize: int64(1024 * 1024), CompressionLevel: 3, - AgeRecipients: []string{"age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05gl0sjq9q9wjg"}, // Test public key + AgeRecipients: []string{testAgePublicKey}, }) // Create a snapshot record for testing ctx := context.Background() snapshotID := "test-snapshot-001" - err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { - snapshot := &database.Snapshot{ - ID: types.SnapshotID(snapshotID), - Hostname: "test-host", - VaultikVersion: "test", - StartedAt: time.Now(), - CompletedAt: nil, - FileCount: 0, - ChunkCount: 0, - BlobCount: 0, - TotalSize: 0, - BlobSize: 0, - CompressionRatio: 1.0, - } - - return repos.Snapshots.Create(ctx, tx, snapshot) - }) - if err != nil { - t.Fatalf("failed to create snapshot: %v", err) - } + createTestSnapshotRecord(ctx, t, repos, snapshotID) // Scan the directory - var result *snapshot.ScanResult - - result, err = scanner.Scan(ctx, "/source", snapshotID) + result, err := scanner.Scan(ctx, "/source", snapshotID) if err != nil { t.Fatalf("scan failed: %v", err) } @@ -259,7 +279,8 @@ func TestScannerLargeFile(t *testing.T) { // The file size should be at least 1MB if result.BytesScanned < 1024*1024 { - t.Errorf("expected at least %d bytes scanned, got %d", 1024*1024, result.BytesScanned) + t.Errorf("expected at least %d bytes scanned, got %d", + 1024*1024, result.BytesScanned) } // Verify chunks diff --git a/internal/snapshot/snapshot.go b/internal/snapshot/snapshot.go index c6c0e4c..2efe8d8 100644 --- a/internal/snapshot/snapshot.go +++ b/internal/snapshot/snapshot.go @@ -1,3 +1,6 @@ +// Package snapshot implements snapshot creation: scanning source +// directories, chunking and deduplicating file data, packing chunks into +// encrypted blobs, and exporting per-snapshot metadata to remote storage. package snapshot // Snapshot Metadata Export Process @@ -58,6 +61,8 @@ import ( ) // SnapshotManager handles snapshot creation and metadata export +// +//nolint:revive // renaming snapshot.SnapshotManager is a cross-package API change type SnapshotManager struct { repos *database.Repositories storage storage.Storer @@ -66,6 +71,8 @@ type SnapshotManager struct { } // SnapshotManagerParams holds dependencies for NewSnapshotManager +// +//nolint:revive // renaming this alongside SnapshotManager is a cross-package API change type SnapshotManagerParams struct { fx.In @@ -88,15 +95,22 @@ func (sm *SnapshotManager) SetFilesystem(fs afero.Fs) { sm.fs = fs } -// CreateSnapshot creates a new snapshot record in the database at the start of a backup. +// CreateSnapshot creates a new snapshot record in the database at the +// start of a backup. +// // Deprecated: Use CreateSnapshotWithName instead for multi-snapshot support. -func (sm *SnapshotManager) CreateSnapshot(ctx context.Context, hostname, version, gitRevision string) (string, error) { +func (sm *SnapshotManager) CreateSnapshot( + ctx context.Context, hostname, version, gitRevision string, +) (string, error) { return sm.CreateSnapshotWithName(ctx, hostname, "", version, gitRevision) } -// CreateSnapshotWithName creates a new snapshot record with an optional snapshot name. -// The snapshot ID format is: hostname_name_timestamp or hostname_timestamp if name is empty. -func (sm *SnapshotManager) CreateSnapshotWithName(ctx context.Context, hostname, name, version, gitRevision string) (string, error) { +// CreateSnapshotWithName creates a new snapshot record with an optional +// snapshot name. The snapshot ID format is: hostname_name_timestamp or +// hostname_timestamp if name is empty. +func (sm *SnapshotManager) CreateSnapshotWithName( + ctx context.Context, hostname, name, version, gitRevision string, +) (string, error) { // Use short hostname (strip domain if present) shortHostname := hostname if before, _, ok := strings.Cut(hostname, "."); ok { @@ -141,7 +155,9 @@ func (sm *SnapshotManager) CreateSnapshotWithName(ctx context.Context, hostname, } // UpdateSnapshotStats updates the statistics for a snapshot during backup -func (sm *SnapshotManager) UpdateSnapshotStats(ctx context.Context, snapshotID string, stats BackupStats) error { +func (sm *SnapshotManager) UpdateSnapshotStats( + ctx context.Context, snapshotID string, stats BackupStats, +) error { err := sm.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { return sm.repos.Snapshots.UpdateCounts(ctx, tx, snapshotID, int64(stats.FilesScanned), @@ -160,7 +176,9 @@ func (sm *SnapshotManager) UpdateSnapshotStats(ctx context.Context, snapshotID s // UpdateSnapshotStatsExtended updates snapshot statistics with extended metrics. // This includes compression level, uncompressed blob size, and upload duration. -func (sm *SnapshotManager) UpdateSnapshotStatsExtended(ctx context.Context, snapshotID string, stats ExtendedBackupStats) error { +func (sm *SnapshotManager) UpdateSnapshotStatsExtended( + ctx context.Context, snapshotID string, stats ExtendedBackupStats, +) error { return sm.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { // First update basic stats err := sm.repos.Snapshots.UpdateCounts(ctx, tx, snapshotID, @@ -187,7 +205,9 @@ func (sm *SnapshotManager) UpdateSnapshotStatsExtended(ctx context.Context, snap // is populated with every blob holding any chunk referenced by the // snapshot's files (including deduplicated blobs uploaded by prior // snapshots). Without this, fully-deduplicated snapshots are unrestorable. -func (sm *SnapshotManager) CompleteSnapshot(ctx context.Context, snapshotID string) error { +func (sm *SnapshotManager) CompleteSnapshot( + ctx context.Context, snapshotID string, +) error { err := sm.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { added, err := sm.repos.Snapshots.PopulateReferencedBlobs(ctx, tx, snapshotID) if err != nil { @@ -226,8 +246,11 @@ func (sm *SnapshotManager) CompleteSnapshot(ctx context.Context, snapshotID stri // - Reopening the main database after this method returns // // This ensures database consistency during the copy operation. -func (sm *SnapshotManager) ExportSnapshotMetadata(ctx context.Context, dbPath string, snapshotID string) error { - log.Info("Phase 3/3: Exporting snapshot metadata", "snapshot_id", snapshotID, "source_db", dbPath) +func (sm *SnapshotManager) ExportSnapshotMetadata( + ctx context.Context, dbPath string, snapshotID string, +) error { + log.Info("Phase 3/3: Exporting snapshot metadata", + "snapshot_id", snapshotID, "source_db", dbPath) // Create temp directory for all temporary files tempDir, err := afero.TempDir(sm.fs, "", "vaultik-snapshot-*") @@ -271,13 +294,127 @@ func (sm *SnapshotManager) ExportSnapshotMetadata(ctx context.Context, dbPath st return nil } -// prepareExportDB copies, cleans, vacuums, and compresses the snapshot database for export. -// Returns the compressed data and the path to the temporary database (needed for manifest generation). -func (sm *SnapshotManager) prepareExportDB(ctx context.Context, dbPath, snapshotID, tempDir string) ([]byte, string, error) { +// CleanupIncompleteSnapshots removes incomplete snapshots that don't have +// metadata in S3. This is critical for data safety: incomplete snapshots +// can cause deduplication to skip files that were never successfully +// backed up, resulting in data loss. +func (sm *SnapshotManager) CleanupIncompleteSnapshots( + ctx context.Context, hostname string, +) error { + log.Info("Checking for incomplete snapshots", "hostname", hostname) + + // Get all incomplete snapshots for this hostname + incompleteSnapshots, err := sm.repos.Snapshots.GetIncompleteByHostname(ctx, hostname) + if err != nil { + return fmt.Errorf("getting incomplete snapshots: %w", err) + } + + if len(incompleteSnapshots) == 0 { + log.Debug("No incomplete snapshots found") + + return nil + } + + log.Info("Found incomplete snapshots", "count", len(incompleteSnapshots)) + + // Check each incomplete snapshot for metadata in storage + for _, snapshot := range incompleteSnapshots { + // Check if metadata exists in storage (paths use the hashed + // remote key so we don't leak host info to the listing). + metadataKey := fmt.Sprintf("metadata/%s/db.zst", + RemoteSnapshotKey(snapshot.ID.String())) + + _, err := sm.storage.Stat(ctx, metadataKey) + if err != nil { + // Metadata doesn't exist in S3 - this is an incomplete snapshot + log.Info("Cleaning up incomplete snapshot record", + "snapshot_id", snapshot.ID, "started_at", snapshot.StartedAt) + + // Delete the snapshot and all its associations + err := sm.deleteSnapshot(ctx, snapshot.ID.String()) + if err != nil { + return fmt.Errorf("deleting incomplete snapshot %s: %w", + snapshot.ID, err) + } + + log.Info("Deleted incomplete snapshot record and associated data", + "snapshot_id", snapshot.ID) + } else { + // Metadata exists - this snapshot was completed but database wasn't updated + // This shouldn't happen in normal operation, but mark it complete + log.Warn("Found snapshot with remote metadata but incomplete in database", + "snapshot_id", snapshot.ID) + + err := sm.repos.Snapshots.MarkComplete(ctx, nil, snapshot.ID.String()) + if err != nil { + log.Error("Failed to mark snapshot as complete in database", + "snapshot_id", snapshot.ID, "error", err) + } + } + } + + return nil +} + +// CleanupOrphanedData removes files, chunks, and blobs that are no longer +// referenced by any snapshot. This should be called periodically to clean +// up data from deleted or incomplete snapshots. +func (sm *SnapshotManager) CleanupOrphanedData(ctx context.Context) error { + // Order is important to respect foreign key constraints: + // 1. Delete orphaned files (will cascade delete file_chunks) + // 2. Delete orphaned blobs (will cascade delete blob_chunks for deleted blobs) + // 3. Delete orphaned blob_chunks (where blob exists but chunk doesn't) + // 4. Delete orphaned chunks (now safe after all blob_chunks are gone) + + // Delete orphaned files (files not in any snapshot) + log.Debug("Deleting orphaned file records from database") + + err := sm.repos.Files.DeleteOrphaned(ctx) + if err != nil { + return fmt.Errorf("deleting orphaned files: %w", err) + } + + // Delete orphaned blobs (blobs not in any snapshot) + // This will cascade delete blob_chunks for deleted blobs + log.Debug("Deleting orphaned blob records from database") + + err = sm.repos.Blobs.DeleteOrphaned(ctx) + if err != nil { + return fmt.Errorf("deleting orphaned blobs: %w", err) + } + + // Delete orphaned blob_chunks entries + // This handles cases where the blob still exists but chunks were deleted + log.Debug("Deleting orphaned blob_chunks associations from database") + + err = sm.repos.BlobChunks.DeleteOrphaned(ctx) + if err != nil { + return fmt.Errorf("deleting orphaned blob_chunks: %w", err) + } + + // Delete orphaned chunks (chunks not referenced by any file) + // This must come after cleaning up blob_chunks to avoid foreign key violations + log.Debug("Deleting orphaned chunk records from database") + + err = sm.repos.Chunks.DeleteOrphaned(ctx) + if err != nil { + return fmt.Errorf("deleting orphaned chunks: %w", err) + } + + return nil +} + +// prepareExportDB copies, cleans, vacuums, and compresses the snapshot +// database for export. Returns the compressed data and the path to the +// temporary database (needed for manifest generation). +func (sm *SnapshotManager) prepareExportDB( + ctx context.Context, dbPath, snapshotID, tempDir string, +) ([]byte, string, error) { // Step 1: Copy database to temp file // The main database should be closed at this point tempDBPath := filepath.Join(tempDir, "snapshot.db") - log.Debug("Copying database to temporary location", "source", dbPath, "destination", tempDBPath) + log.Debug("Copying database to temporary location", + "source", dbPath, "destination", tempDBPath) err := sm.copyFile(dbPath, tempDBPath) if err != nil { @@ -296,22 +433,24 @@ func (sm *SnapshotManager) prepareExportDB(ctx context.Context, dbPath, snapshot log.Info("Temporary database cleanup complete", "db_path", tempDBPath, - "size_after_clean", humanize.Bytes(uint64(sm.getFileSize(tempDBPath))), + "size_after_clean", humanize.Bytes(safeUint64(sm.getFileSize(tempDBPath))), "files", stats.FileCount, "chunks", stats.ChunkCount, "blobs", stats.BlobCount, - "total_compressed_size", humanize.Bytes(uint64(stats.CompressedSize)), - "total_uncompressed_size", humanize.Bytes(uint64(stats.UncompressedSize)), - "compression_ratio", fmt.Sprintf("%.2fx", float64(stats.UncompressedSize)/float64(stats.CompressedSize))) + "total_compressed_size", humanize.Bytes(safeUint64(stats.CompressedSize)), + "total_uncompressed_size", humanize.Bytes(safeUint64(stats.UncompressedSize)), + "compression_ratio", fmt.Sprintf("%.2fx", + float64(stats.UncompressedSize)/float64(stats.CompressedSize))) // Step 3: VACUUM the database to remove deleted data and compact // This is critical for security - ensures no stale/deleted data is uploaded - err = sm.vacuumDatabase(tempDBPath) + err = sm.vacuumDatabase(ctx, tempDBPath) if err != nil { return nil, "", fmt.Errorf("vacuuming database: %w", err) } - log.Debug("Database vacuumed", "size", humanize.Bytes(uint64(sm.getFileSize(tempDBPath)))) + log.Debug("Database vacuumed", + "size", humanize.Bytes(safeUint64(sm.getFileSize(tempDBPath)))) // Step 4: Compress and encrypt the binary database file compressedPath := filepath.Join(tempDir, "db.zst.age") @@ -322,8 +461,8 @@ func (sm *SnapshotManager) prepareExportDB(ctx context.Context, dbPath, snapshot } log.Debug("Compression complete", - "original_size", humanize.Bytes(uint64(sm.getFileSize(tempDBPath))), - "compressed_size", humanize.Bytes(uint64(sm.getFileSize(compressedPath)))) + "original_size", humanize.Bytes(safeUint64(sm.getFileSize(tempDBPath))), + "compressed_size", humanize.Bytes(safeUint64(sm.getFileSize(compressedPath)))) // Step 5: Read compressed and encrypted data for upload finalData, err := afero.ReadFile(sm.fs, compressedPath) @@ -340,7 +479,9 @@ func (sm *SnapshotManager) prepareExportDB(ctx context.Context, dbPath, snapshot // We never write the human-readable snapshot ID into any unencrypted // part of remote storage so a listing of the destination bucket leaks // no host, configuration, or scheduling information. -func (sm *SnapshotManager) uploadSnapshotArtifacts(ctx context.Context, snapshotID string, dbData, manifestData []byte) error { +func (sm *SnapshotManager) uploadSnapshotArtifacts( + ctx context.Context, snapshotID string, dbData, manifestData []byte, +) error { remoteKey := RemoteSnapshotKey(snapshotID) // Upload database backup (compressed and encrypted) @@ -354,7 +495,8 @@ func (sm *SnapshotManager) uploadSnapshotArtifacts(ctx context.Context, snapshot } dbUploadDuration := time.Since(dbUploadStart) - dbUploadSpeed := float64(len(dbData)) * 8 / dbUploadDuration.Seconds() // bits per second + // bits per second + dbUploadSpeed := float64(len(dbData)) * bitsPerByte / dbUploadDuration.Seconds() log.Info("Uploaded snapshot database", "path", dbKey, "size", humanize.Bytes(uint64(len(dbData))), @@ -371,7 +513,9 @@ func (sm *SnapshotManager) uploadSnapshotArtifacts(ctx context.Context, snapshot } manifestUploadDuration := time.Since(manifestUploadStart) - manifestUploadSpeed := float64(len(manifestData)) * 8 / manifestUploadDuration.Seconds() // bits per second + // bits per second + manifestUploadSpeed := float64(len(manifestData)) * bitsPerByte / + manifestUploadDuration.Seconds() log.Info("Uploaded blob manifest", "path", manifestKey, "size", humanize.Bytes(uint64(len(manifestData))), @@ -393,16 +537,19 @@ type CleanupStats struct { // cleanSnapshotDB removes all data except for the specified snapshot // // The cleanup is performed in a specific order to maintain referential integrity: -// 1. Delete other snapshots -// 2. Delete orphaned snapshot associations (snapshot_files, snapshot_blobs) for deleted snapshots -// 3. Delete orphaned files (not in the current snapshot) -// 4. Delete orphaned chunk-to-file mappings (references to deleted files) -// 5. Delete orphaned blobs (not in the current snapshot) -// 6. Delete orphaned blob-to-chunk mappings (references to deleted chunks) -// 7. Delete orphaned chunks (not referenced by any file) +// 1. Delete other snapshots +// 2. Delete orphaned snapshot associations (snapshot_files, snapshot_blobs) +// for deleted snapshots +// 3. Delete orphaned files (not in the current snapshot) +// 4. Delete orphaned chunk-to-file mappings (references to deleted files) +// 5. Delete orphaned blobs (not in the current snapshot) +// 6. Delete orphaned blob-to-chunk mappings (references to deleted chunks) +// 7. Delete orphaned chunks (not referenced by any file) // // Each step is implemented as a separate method for clarity and maintainability. -func (sm *SnapshotManager) cleanSnapshotDB(ctx context.Context, dbPath string, snapshotID string) (*CleanupStats, error) { +func (sm *SnapshotManager) cleanSnapshotDB( + ctx context.Context, dbPath string, snapshotID string, +) (*CleanupStats, error) { // Open the temp database db, err := database.New(ctx, dbPath) if err != nil { @@ -428,39 +575,31 @@ func (sm *SnapshotManager) cleanSnapshotDB(ctx context.Context, dbPath string, s }() // Execute cleanup steps in order - err = sm.deleteOtherSnapshots(ctx, tx, snapshotID) - if err != nil { - return nil, fmt.Errorf("step 1 - delete other snapshots: %w", err) + steps := []struct { + name string + fn func() error + }{ + {"delete other snapshots", + func() error { return sm.deleteOtherSnapshots(ctx, tx, snapshotID) }}, + {"delete orphaned snapshot associations", + func() error { return sm.deleteOrphanedSnapshotAssociations(ctx, tx, snapshotID) }}, + {"delete orphaned files", + func() error { return sm.deleteOrphanedFiles(ctx, tx, snapshotID) }}, + {"delete orphaned chunk-to-file mappings", + func() error { return sm.deleteOrphanedChunkToFileMappings(ctx, tx) }}, + {"delete orphaned blobs", + func() error { return sm.deleteOrphanedBlobs(ctx, tx, snapshotID) }}, + {"delete orphaned blob-to-chunk mappings", + func() error { return sm.deleteOrphanedBlobToChunkMappings(ctx, tx) }}, + {"delete orphaned chunks", + func() error { return sm.deleteOrphanedChunks(ctx, tx) }}, } - err = sm.deleteOrphanedSnapshotAssociations(ctx, tx, snapshotID) - if err != nil { - return nil, fmt.Errorf("step 2 - delete orphaned snapshot associations: %w", err) - } - - err = sm.deleteOrphanedFiles(ctx, tx, snapshotID) - if err != nil { - return nil, fmt.Errorf("step 3 - delete orphaned files: %w", err) - } - - err = sm.deleteOrphanedChunkToFileMappings(ctx, tx) - if err != nil { - return nil, fmt.Errorf("step 4 - delete orphaned chunk-to-file mappings: %w", err) - } - - err = sm.deleteOrphanedBlobs(ctx, tx, snapshotID) - if err != nil { - return nil, fmt.Errorf("step 5 - delete orphaned blobs: %w", err) - } - - err = sm.deleteOrphanedBlobToChunkMappings(ctx, tx) - if err != nil { - return nil, fmt.Errorf("step 6 - delete orphaned blob-to-chunk mappings: %w", err) - } - - err = sm.deleteOrphanedChunks(ctx, tx) - if err != nil { - return nil, fmt.Errorf("step 7 - delete orphaned chunks: %w", err) + for i, step := range steps { + err = step.fn() + if err != nil { + return nil, fmt.Errorf("step %d - %s: %w", i+1, step.name, err) + } } // Commit transaction @@ -471,13 +610,19 @@ func (sm *SnapshotManager) cleanSnapshotDB(ctx context.Context, dbPath string, s return nil, fmt.Errorf("committing transaction: %w", err) } - // Collect statistics about the cleaned database + return sm.collectCleanupStats(ctx, db, snapshotID) +} + +// collectCleanupStats gathers statistics about the cleaned database. +func (sm *SnapshotManager) collectCleanupStats( + ctx context.Context, db *database.DB, snapshotID string, +) (*CleanupStats, error) { stats := &CleanupStats{} // Count files var fileCount int - err = db.QueryRowWithLog(ctx, "SELECT COUNT(*) FROM files").Scan(&fileCount) + err := db.QueryRowWithLog(ctx, "SELECT COUNT(*) FROM files").Scan(&fileCount) if err != nil { return nil, fmt.Errorf("counting files: %w", err) } @@ -501,9 +646,12 @@ func (sm *SnapshotManager) cleanSnapshotDB(ctx context.Context, dbPath string, s ) err = db.QueryRowWithLog(ctx, ` - SELECT COUNT(*), COALESCE(SUM(compressed_size), 0), COALESCE(SUM(uncompressed_size), 0) - FROM blobs - WHERE blob_hash IN (SELECT blob_hash FROM snapshot_blobs WHERE snapshot_id = ?) + SELECT COUNT(*), + COALESCE(SUM(compressed_size), 0), + COALESCE(SUM(uncompressed_size), 0) + FROM blobs + WHERE blob_hash IN + (SELECT blob_hash FROM snapshot_blobs WHERE snapshot_id = ?) `, snapshotID).Scan(&blobCount, &compressedSize, &uncompressedSize) if err != nil { return nil, fmt.Errorf("counting blobs and sizes: %w", err) @@ -518,9 +666,10 @@ func (sm *SnapshotManager) cleanSnapshotDB(ctx context.Context, dbPath string, s // vacuumDatabase runs VACUUM on the database to remove deleted data and compact // This is critical for security - ensures no stale/deleted data pages are uploaded -func (sm *SnapshotManager) vacuumDatabase(dbPath string) error { +func (sm *SnapshotManager) vacuumDatabase(ctx context.Context, dbPath string) error { log.Debug("Running VACUUM on database", "path", dbPath) - cmd := exec.Command("sqlite3", dbPath, "VACUUM;") + //nolint:gosec // G204: fixed argv; dbPath is our own temp file path + cmd := exec.CommandContext(ctx, "sqlite3", dbPath, "VACUUM;") output, err := cmd.CombinedOutput() if err != nil { @@ -557,7 +706,8 @@ func (sm *SnapshotManager) compressFile(inputPath, outputPath string) error { // Use blobgen for compression and encryption log.Debug("Compressing and encrypting data") - writer, err := blobgen.NewWriter(output, sm.config.CompressionLevel, sm.config.AgeRecipients) + writer, err := blobgen.NewWriter(output, sm.config.CompressionLevel, + sm.config.AgeRecipients) if err != nil { return fmt.Errorf("creating blobgen writer: %w", err) } @@ -636,7 +786,9 @@ func (sm *SnapshotManager) copyFile(src, dst string) error { } // generateBlobManifest creates a compressed JSON list of all blobs in the snapshot -func (sm *SnapshotManager) generateBlobManifest(ctx context.Context, dbPath string, snapshotID string) ([]byte, error) { +func (sm *SnapshotManager) generateBlobManifest( + ctx context.Context, dbPath string, snapshotID string, +) ([]byte, error) { // Open the cleaned database using the database package db, err := database.New(ctx, dbPath) if err != nil { @@ -734,61 +886,10 @@ type ExtendedBackupStats struct { UploadDurationMs int64 // Total milliseconds spent uploading to S3 } -// CleanupIncompleteSnapshots removes incomplete snapshots that don't have metadata in S3. -// This is critical for data safety: incomplete snapshots can cause deduplication to skip -// files that were never successfully backed up, resulting in data loss. -func (sm *SnapshotManager) CleanupIncompleteSnapshots(ctx context.Context, hostname string) error { - log.Info("Checking for incomplete snapshots", "hostname", hostname) - - // Get all incomplete snapshots for this hostname - incompleteSnapshots, err := sm.repos.Snapshots.GetIncompleteByHostname(ctx, hostname) - if err != nil { - return fmt.Errorf("getting incomplete snapshots: %w", err) - } - - if len(incompleteSnapshots) == 0 { - log.Debug("No incomplete snapshots found") - - return nil - } - - log.Info("Found incomplete snapshots", "count", len(incompleteSnapshots)) - - // Check each incomplete snapshot for metadata in storage - for _, snapshot := range incompleteSnapshots { - // Check if metadata exists in storage (paths use the hashed - // remote key so we don't leak host info to the listing). - metadataKey := fmt.Sprintf("metadata/%s/db.zst", RemoteSnapshotKey(snapshot.ID.String())) - - _, err := sm.storage.Stat(ctx, metadataKey) - if err != nil { - // Metadata doesn't exist in S3 - this is an incomplete snapshot - log.Info("Cleaning up incomplete snapshot record", "snapshot_id", snapshot.ID, "started_at", snapshot.StartedAt) - - // Delete the snapshot and all its associations - err := sm.deleteSnapshot(ctx, snapshot.ID.String()) - if err != nil { - return fmt.Errorf("deleting incomplete snapshot %s: %w", snapshot.ID, err) - } - - log.Info("Deleted incomplete snapshot record and associated data", "snapshot_id", snapshot.ID) - } else { - // Metadata exists - this snapshot was completed but database wasn't updated - // This shouldn't happen in normal operation, but mark it complete - log.Warn("Found snapshot with remote metadata but incomplete in database", "snapshot_id", snapshot.ID) - - err := sm.repos.Snapshots.MarkComplete(ctx, nil, snapshot.ID.String()) - if err != nil { - log.Error("Failed to mark snapshot as complete in database", "snapshot_id", snapshot.ID, "error", err) - } - } - } - - return nil -} - // deleteSnapshot removes a snapshot and all its associations from the database -func (sm *SnapshotManager) deleteSnapshot(ctx context.Context, snapshotID string) error { +func (sm *SnapshotManager) deleteSnapshot( + ctx context.Context, snapshotID string, +) error { // Delete snapshot_files entries err := sm.repos.Snapshots.DeleteSnapshotFiles(ctx, snapshotID) if err != nil { @@ -824,61 +925,20 @@ func (sm *SnapshotManager) deleteSnapshot(ctx context.Context, snapshotID string return nil } -// CleanupOrphanedData removes files, chunks, and blobs that are no longer referenced by any snapshot. -// This should be called periodically to clean up data from deleted or incomplete snapshots. -func (sm *SnapshotManager) CleanupOrphanedData(ctx context.Context) error { - // Order is important to respect foreign key constraints: - // 1. Delete orphaned files (will cascade delete file_chunks) - // 2. Delete orphaned blobs (will cascade delete blob_chunks for deleted blobs) - // 3. Delete orphaned blob_chunks (where blob exists but chunk doesn't) - // 4. Delete orphaned chunks (now safe after all blob_chunks are gone) - - // Delete orphaned files (files not in any snapshot) - log.Debug("Deleting orphaned file records from database") - - err := sm.repos.Files.DeleteOrphaned(ctx) - if err != nil { - return fmt.Errorf("deleting orphaned files: %w", err) - } - - // Delete orphaned blobs (blobs not in any snapshot) - // This will cascade delete blob_chunks for deleted blobs - log.Debug("Deleting orphaned blob records from database") - - err = sm.repos.Blobs.DeleteOrphaned(ctx) - if err != nil { - return fmt.Errorf("deleting orphaned blobs: %w", err) - } - - // Delete orphaned blob_chunks entries - // This handles cases where the blob still exists but chunks were deleted - log.Debug("Deleting orphaned blob_chunks associations from database") - - err = sm.repos.BlobChunks.DeleteOrphaned(ctx) - if err != nil { - return fmt.Errorf("deleting orphaned blob_chunks: %w", err) - } - - // Delete orphaned chunks (chunks not referenced by any file) - // This must come after cleaning up blob_chunks to avoid foreign key violations - log.Debug("Deleting orphaned chunk records from database") - - err = sm.repos.Chunks.DeleteOrphaned(ctx) - if err != nil { - return fmt.Errorf("deleting orphaned chunks: %w", err) - } - - return nil -} - // deleteOtherSnapshots deletes all snapshots except the current one -func (sm *SnapshotManager) deleteOtherSnapshots(ctx context.Context, tx *sql.Tx, currentSnapshotID string) error { - log.Debug("[Temp DB Cleanup] Deleting all snapshot records except current", "keeping", currentSnapshotID) +func (sm *SnapshotManager) deleteOtherSnapshots( + ctx context.Context, tx *sql.Tx, currentSnapshotID string, +) error { + log.Debug("[Temp DB Cleanup] Deleting all snapshot records except current", + "keeping", currentSnapshotID) - // First delete uploads that reference other snapshots (no CASCADE DELETE on this FK) - database.LogSQL("Execute", "DELETE FROM uploads WHERE snapshot_id != ?", currentSnapshotID) + // First delete uploads that reference other snapshots (no CASCADE DELETE + // on this FK) + database.LogSQL("Execute", "DELETE FROM uploads WHERE snapshot_id != ?", + currentSnapshotID) - uploadResult, err := tx.ExecContext(ctx, "DELETE FROM uploads WHERE snapshot_id != ?", currentSnapshotID) + uploadResult, err := tx.ExecContext(ctx, + "DELETE FROM uploads WHERE snapshot_id != ?", currentSnapshotID) if err != nil { return fmt.Errorf("deleting uploads for other snapshots: %w", err) } @@ -887,66 +947,84 @@ func (sm *SnapshotManager) deleteOtherSnapshots(ctx context.Context, tx *sql.Tx, log.Debug("[Temp DB Cleanup] Deleted upload records", "count", uploadsDeleted) // Now we can safely delete the snapshots - database.LogSQL("Execute", "DELETE FROM snapshots WHERE id != ?", currentSnapshotID) + database.LogSQL("Execute", "DELETE FROM snapshots WHERE id != ?", + currentSnapshotID) - result, err := tx.ExecContext(ctx, "DELETE FROM snapshots WHERE id != ?", currentSnapshotID) + result, err := tx.ExecContext(ctx, "DELETE FROM snapshots WHERE id != ?", + currentSnapshotID) if err != nil { return fmt.Errorf("deleting other snapshots: %w", err) } rowsAffected, _ := result.RowsAffected() - log.Debug("[Temp DB Cleanup] Deleted snapshot records from database", "count", rowsAffected) + log.Debug("[Temp DB Cleanup] Deleted snapshot records from database", + "count", rowsAffected) return nil } -// deleteOrphanedSnapshotAssociations deletes snapshot_files and snapshot_blobs for deleted snapshots -func (sm *SnapshotManager) deleteOrphanedSnapshotAssociations(ctx context.Context, tx *sql.Tx, currentSnapshotID string) error { +// deleteOrphanedSnapshotAssociations deletes snapshot_files and +// snapshot_blobs for deleted snapshots +func (sm *SnapshotManager) deleteOrphanedSnapshotAssociations( + ctx context.Context, tx *sql.Tx, currentSnapshotID string, +) error { // Delete orphaned snapshot_files log.Debug("[Temp DB Cleanup] Deleting orphaned snapshot_files associations") - database.LogSQL("Execute", "DELETE FROM snapshot_files WHERE snapshot_id != ?", currentSnapshotID) + database.LogSQL("Execute", "DELETE FROM snapshot_files WHERE snapshot_id != ?", + currentSnapshotID) - result, err := tx.ExecContext(ctx, "DELETE FROM snapshot_files WHERE snapshot_id != ?", currentSnapshotID) + result, err := tx.ExecContext(ctx, + "DELETE FROM snapshot_files WHERE snapshot_id != ?", currentSnapshotID) if err != nil { return fmt.Errorf("deleting orphaned snapshot_files: %w", err) } rowsAffected, _ := result.RowsAffected() - log.Debug("[Temp DB Cleanup] Deleted snapshot_files associations", "count", rowsAffected) + log.Debug("[Temp DB Cleanup] Deleted snapshot_files associations", + "count", rowsAffected) // Delete orphaned snapshot_blobs log.Debug("[Temp DB Cleanup] Deleting orphaned snapshot_blobs associations") - database.LogSQL("Execute", "DELETE FROM snapshot_blobs WHERE snapshot_id != ?", currentSnapshotID) + database.LogSQL("Execute", "DELETE FROM snapshot_blobs WHERE snapshot_id != ?", + currentSnapshotID) - result, err = tx.ExecContext(ctx, "DELETE FROM snapshot_blobs WHERE snapshot_id != ?", currentSnapshotID) + result, err = tx.ExecContext(ctx, + "DELETE FROM snapshot_blobs WHERE snapshot_id != ?", currentSnapshotID) if err != nil { return fmt.Errorf("deleting orphaned snapshot_blobs: %w", err) } rowsAffected, _ = result.RowsAffected() - log.Debug("[Temp DB Cleanup] Deleted snapshot_blobs associations", "count", rowsAffected) + log.Debug("[Temp DB Cleanup] Deleted snapshot_blobs associations", + "count", rowsAffected) return nil } // deleteOrphanedFiles deletes files not in the current snapshot -func (sm *SnapshotManager) deleteOrphanedFiles(ctx context.Context, tx *sql.Tx, currentSnapshotID string) error { - log.Debug("[Temp DB Cleanup] Deleting file records not referenced by current snapshot") - database.LogSQL("Execute", `DELETE FROM files WHERE NOT EXISTS (SELECT 1 FROM snapshot_files WHERE snapshot_files.file_id = files.id AND snapshot_files.snapshot_id = ?)`, currentSnapshotID) +func (sm *SnapshotManager) deleteOrphanedFiles( + ctx context.Context, tx *sql.Tx, currentSnapshotID string, +) error { + log.Debug( + "[Temp DB Cleanup] Deleting file records not referenced by current snapshot") - result, err := tx.ExecContext(ctx, ` - DELETE FROM files + query := ` + DELETE FROM files WHERE NOT EXISTS ( - SELECT 1 FROM snapshot_files - WHERE snapshot_files.file_id = files.id + SELECT 1 FROM snapshot_files + WHERE snapshot_files.file_id = files.id AND snapshot_files.snapshot_id = ? - )`, currentSnapshotID) + )` + database.LogSQL("Execute", query, currentSnapshotID) + + result, err := tx.ExecContext(ctx, query, currentSnapshotID) if err != nil { return fmt.Errorf("deleting orphaned files: %w", err) } rowsAffected, _ := result.RowsAffected() - log.Debug("[Temp DB Cleanup] Deleted file records from database", "count", rowsAffected) + log.Debug("[Temp DB Cleanup] Deleted file records from database", + "count", rowsAffected) // Note: file_chunks will be deleted via CASCADE log.Debug("[Temp DB Cleanup] file_chunks associations deleted via CASCADE") @@ -955,65 +1033,81 @@ func (sm *SnapshotManager) deleteOrphanedFiles(ctx context.Context, tx *sql.Tx, } // deleteOrphanedChunkToFileMappings deletes chunk_files entries for deleted files -func (sm *SnapshotManager) deleteOrphanedChunkToFileMappings(ctx context.Context, tx *sql.Tx) error { +func (sm *SnapshotManager) deleteOrphanedChunkToFileMappings( + ctx context.Context, tx *sql.Tx, +) error { log.Debug("[Temp DB Cleanup] Deleting orphaned chunk_files associations") - database.LogSQL("Execute", `DELETE FROM chunk_files WHERE NOT EXISTS (SELECT 1 FROM files WHERE files.id = chunk_files.file_id)`) - result, err := tx.ExecContext(ctx, ` - DELETE FROM chunk_files + query := ` + DELETE FROM chunk_files WHERE NOT EXISTS ( - SELECT 1 FROM files + SELECT 1 FROM files WHERE files.id = chunk_files.file_id - )`) + )` + database.LogSQL("Execute", query) + + result, err := tx.ExecContext(ctx, query) if err != nil { return fmt.Errorf("deleting orphaned chunk_files: %w", err) } rowsAffected, _ := result.RowsAffected() - log.Debug("[Temp DB Cleanup] Deleted chunk_files associations", "count", rowsAffected) + log.Debug("[Temp DB Cleanup] Deleted chunk_files associations", + "count", rowsAffected) return nil } // deleteOrphanedBlobs deletes blobs not in the current snapshot -func (sm *SnapshotManager) deleteOrphanedBlobs(ctx context.Context, tx *sql.Tx, currentSnapshotID string) error { - log.Debug("[Temp DB Cleanup] Deleting blob records not referenced by current snapshot") - database.LogSQL("Execute", `DELETE FROM blobs WHERE NOT EXISTS (SELECT 1 FROM snapshot_blobs WHERE snapshot_blobs.blob_hash = blobs.blob_hash AND snapshot_blobs.snapshot_id = ?)`, currentSnapshotID) +func (sm *SnapshotManager) deleteOrphanedBlobs( + ctx context.Context, tx *sql.Tx, currentSnapshotID string, +) error { + log.Debug( + "[Temp DB Cleanup] Deleting blob records not referenced by current snapshot") - result, err := tx.ExecContext(ctx, ` - DELETE FROM blobs + query := ` + DELETE FROM blobs WHERE NOT EXISTS ( - SELECT 1 FROM snapshot_blobs - WHERE snapshot_blobs.blob_hash = blobs.blob_hash + SELECT 1 FROM snapshot_blobs + WHERE snapshot_blobs.blob_hash = blobs.blob_hash AND snapshot_blobs.snapshot_id = ? - )`, currentSnapshotID) + )` + database.LogSQL("Execute", query, currentSnapshotID) + + result, err := tx.ExecContext(ctx, query, currentSnapshotID) if err != nil { return fmt.Errorf("deleting orphaned blobs: %w", err) } rowsAffected, _ := result.RowsAffected() - log.Debug("[Temp DB Cleanup] Deleted blob records from database", "count", rowsAffected) + log.Debug("[Temp DB Cleanup] Deleted blob records from database", + "count", rowsAffected) return nil } // deleteOrphanedBlobToChunkMappings deletes blob_chunks entries for deleted blobs -func (sm *SnapshotManager) deleteOrphanedBlobToChunkMappings(ctx context.Context, tx *sql.Tx) error { +func (sm *SnapshotManager) deleteOrphanedBlobToChunkMappings( + ctx context.Context, tx *sql.Tx, +) error { log.Debug("[Temp DB Cleanup] Deleting orphaned blob_chunks associations") - database.LogSQL("Execute", `DELETE FROM blob_chunks WHERE NOT EXISTS (SELECT 1 FROM blobs WHERE blobs.id = blob_chunks.blob_id)`) - result, err := tx.ExecContext(ctx, ` - DELETE FROM blob_chunks + query := ` + DELETE FROM blob_chunks WHERE NOT EXISTS ( - SELECT 1 FROM blobs + SELECT 1 FROM blobs WHERE blobs.id = blob_chunks.blob_id - )`) + )` + database.LogSQL("Execute", query) + + result, err := tx.ExecContext(ctx, query) if err != nil { return fmt.Errorf("deleting orphaned blob_chunks: %w", err) } rowsAffected, _ := result.RowsAffected() - log.Debug("[Temp DB Cleanup] Deleted blob_chunks associations", "count", rowsAffected) + log.Debug("[Temp DB Cleanup] Deleted blob_chunks associations", + "count", rowsAffected) return nil } @@ -1040,7 +1134,8 @@ func (sm *SnapshotManager) deleteOrphanedChunks(ctx context.Context, tx *sql.Tx) } rowsAffected, _ := result.RowsAffected() - log.Debug("[Temp DB Cleanup] Deleted chunk records from database", "count", rowsAffected) + log.Debug("[Temp DB Cleanup] Deleted chunk records from database", + "count", rowsAffected) return nil } diff --git a/internal/snapshot/snapshot_test.go b/internal/snapshot/snapshot_test.go index 2c1dfe5..fd83d72 100644 --- a/internal/snapshot/snapshot_test.go +++ b/internal/snapshot/snapshot_test.go @@ -1,3 +1,4 @@ +//nolint:testpackage // exercises unexported SnapshotManager internals package snapshot import ( @@ -37,9 +38,65 @@ func copyFile(fs afero.Fs, src, dst string) error { return err } +// verifyCleanedDB opens the cleaned database and checks that the kept +// snapshot survived while the orphan file and chunk were removed. +func verifyCleanedDB( + ctx context.Context, + t *testing.T, + tempDBPath, snapshotID string, + file *database.File, + chunk *database.Chunk, +) { + t.Helper() + + cleanedDB, err := database.New(ctx, tempDBPath) + if err != nil { + t.Fatalf("failed to open cleaned database: %v", err) + } + defer func() { + err := cleanedDB.Close() + if err != nil { + t.Errorf("failed to close database: %v", err) + } + }() + + cleanedRepos := database.NewRepositories(cleanedDB) + + // Verify snapshot exists + verifySnapshot, err := cleanedRepos.Snapshots.GetByID(ctx, snapshotID) + if err != nil { + t.Fatalf("failed to get snapshot: %v", err) + } + + if verifySnapshot == nil { + t.Error("snapshot should exist") + } + + // Verify orphan file is gone + f, err := cleanedRepos.Files.GetByPath(ctx, file.Path.String()) + if err != nil { + t.Fatalf("failed to check file: %v", err) + } + + if f != nil { + t.Error("orphan file should not exist") + } + + // Verify orphan chunk is gone + c, err := cleanedRepos.Chunks.GetByHash(ctx, chunk.ChunkHash.String()) + if err != nil { + t.Fatalf("failed to check chunk: %v", err) + } + + if c != nil { + t.Error("orphan chunk should not exist") + } +} + func TestCleanSnapshotDBEmptySnapshot(t *testing.T) { // Initialize logger log.Initialize(log.Config{}) + t.Parallel() ctx := context.Background() fs := afero.NewOsFs() @@ -115,53 +172,13 @@ func TestCleanSnapshotDBEmptySnapshot(t *testing.T) { } // Verify the cleaned database - cleanedDB, err := database.New(ctx, tempDBPath) - if err != nil { - t.Fatalf("failed to open cleaned database: %v", err) - } - defer func() { - err := cleanedDB.Close() - if err != nil { - t.Errorf("failed to close database: %v", err) - } - }() - - cleanedRepos := database.NewRepositories(cleanedDB) - - // Verify snapshot exists - verifySnapshot, err := cleanedRepos.Snapshots.GetByID(ctx, snapshot.ID.String()) - if err != nil { - t.Fatalf("failed to get snapshot: %v", err) - } - - if verifySnapshot == nil { - t.Error("snapshot should exist") - } - - // Verify orphan file is gone - f, err := cleanedRepos.Files.GetByPath(ctx, file.Path.String()) - if err != nil { - t.Fatalf("failed to check file: %v", err) - } - - if f != nil { - t.Error("orphan file should not exist") - } - - // Verify orphan chunk is gone - c, err := cleanedRepos.Chunks.GetByHash(ctx, chunk.ChunkHash.String()) - if err != nil { - t.Fatalf("failed to check chunk: %v", err) - } - - if c != nil { - t.Error("orphan chunk should not exist") - } + verifyCleanedDB(ctx, t, tempDBPath, snapshot.ID.String(), file, chunk) } func TestCleanSnapshotDBNonExistentSnapshot(t *testing.T) { // Initialize logger log.Initialize(log.Config{}) + t.Parallel() ctx := context.Background() fs := afero.NewOsFs() diff --git a/internal/storage/file.go b/internal/storage/file.go index c3f00a3..1dab239 100644 --- a/internal/storage/file.go +++ b/internal/storage/file.go @@ -42,18 +42,19 @@ func (f *FileStorer) SetFilesystem(fs afero.Fs) { f.fs = fs } -// fullPath returns the full filesystem path for a key. -func (f *FileStorer) fullPath(key string) string { - return filepath.Join(f.basePath, key) -} +// storageDirPerm is the mode used for directories created under the +// storage base path. +const storageDirPerm = 0o755 // Put stores data at the specified key. -func (f *FileStorer) Put(ctx context.Context, key string, data io.Reader) error { +func (f *FileStorer) Put(_ context.Context, key string, data io.Reader) error { path := f.fullPath(key) // Create parent directories dir := filepath.Dir(path) - if err := f.fs.MkdirAll(dir, 0755); err != nil { + + err := f.fs.MkdirAll(dir, storageDirPerm) + if err != nil { return fmt.Errorf("creating directories: %w", err) } @@ -72,12 +73,17 @@ func (f *FileStorer) Put(ctx context.Context, key string, data io.Reader) error } // PutWithProgress stores data with progress reporting. -func (f *FileStorer) PutWithProgress(ctx context.Context, key string, data io.Reader, size int64, progress ProgressCallback) error { +func (f *FileStorer) PutWithProgress( + _ context.Context, key string, data io.Reader, + _ int64, progress ProgressCallback, +) error { path := f.fullPath(key) // Create parent directories dir := filepath.Dir(path) - if err := f.fs.MkdirAll(dir, 0755); err != nil { + + err := f.fs.MkdirAll(dir, storageDirPerm) + if err != nil { return fmt.Errorf("creating directories: %w", err) } @@ -102,7 +108,7 @@ func (f *FileStorer) PutWithProgress(ctx context.Context, key string, data io.Re } // Get retrieves data from the specified key. -func (f *FileStorer) Get(ctx context.Context, key string) (io.ReadCloser, error) { +func (f *FileStorer) Get(_ context.Context, key string) (io.ReadCloser, error) { path := f.fullPath(key) file, err := f.fs.Open(path) @@ -118,7 +124,7 @@ func (f *FileStorer) Get(ctx context.Context, key string) (io.ReadCloser, error) } // Stat returns metadata about an object without retrieving its contents. -func (f *FileStorer) Stat(ctx context.Context, key string) (*ObjectInfo, error) { +func (f *FileStorer) Stat(_ context.Context, key string) (*ObjectInfo, error) { path := f.fullPath(key) info, err := f.fs.Stat(path) @@ -137,7 +143,7 @@ func (f *FileStorer) Stat(ctx context.Context, key string) (*ObjectInfo, error) } // Delete removes an object. -func (f *FileStorer) Delete(ctx context.Context, key string) error { +func (f *FileStorer) Delete(_ context.Context, key string) error { path := f.fullPath(key) err := f.fs.Remove(path) @@ -233,7 +239,7 @@ func (f *FileStorer) ListStream(ctx context.Context, prefix string) <-chan Objec if err != nil { ch <- ObjectInfo{Err: err} - return nil // Continue walking despite errors + return nil //nolint:nilerr // continue walking despite errors } if !info.IsDir() { @@ -259,13 +265,18 @@ func (f *FileStorer) ListStream(ctx context.Context, prefix string) <-chan Objec } // Info returns human-readable storage location information. -func (f *FileStorer) Info() StorageInfo { - return StorageInfo{ - Type: "file", +func (f *FileStorer) Info() Info { + return Info{ + Type: schemeFile, Location: f.basePath, } } +// fullPath returns the full filesystem path for a key. +func (f *FileStorer) fullPath(key string) string { + return filepath.Join(f.basePath, key) +} + // progressWriter wraps an io.Writer to track write progress. type progressWriter struct { writer io.Writer diff --git a/internal/storage/module.go b/internal/storage/module.go index 42cc28a..8699a7c 100644 --- a/internal/storage/module.go +++ b/internal/storage/module.go @@ -10,9 +10,17 @@ import ( "sneak.berlin/go/vaultik/internal/s3" ) +// defaultS3Region is used when neither the URL nor the config specify one. +const defaultS3Region = "us-east-1" + +// defaultS3Endpoint is the AWS endpoint used when none is configured. +const defaultS3Endpoint = "s3.amazonaws.com" + // Module exports storage functionality as an fx module. // It provides a Storer implementation based on the configured storage URL // or falls back to legacy S3 configuration. +// +//nolint:gochecknoglobals // fx module definitions are package globals var Module = fx.Module("storage", fx.Provide(NewStorer), ) @@ -20,6 +28,8 @@ var Module = fx.Module("storage", // NewStorer creates a Storer based on configuration. // If StorageURL is set, it uses URL-based configuration. // Otherwise, it falls back to legacy S3 configuration. +// +//nolint:ireturn // fx provider intentionally returns the Storer interface func NewStorer(cfg *config.Config) (Storer, error) { if cfg.StorageURL != "" { return storerFromURL(cfg.StorageURL, cfg) @@ -28,6 +38,7 @@ func NewStorer(cfg *config.Config) (Storer, error) { return storerFromLegacyS3Config(cfg) } +//nolint:ireturn // factory intentionally returns the Storer interface func storerFromURL(rawURL string, cfg *config.Config) (Storer, error) { parsed, err := ParseStorageURL(rawURL) if err != nil { @@ -35,59 +46,74 @@ func storerFromURL(rawURL string, cfg *config.Config) (Storer, error) { } switch parsed.Scheme { - case "file": + case schemeFile: return NewFileStorer(parsed.Prefix) - case "s3": - // Build endpoint URL - endpoint := parsed.Endpoint - if endpoint == "" { - endpoint = "s3.amazonaws.com" - } + case schemeS3: + return storerFromParsedS3URL(parsed, cfg) - // Add protocol if not present - if parsed.UseSSL && !strings.HasPrefix(endpoint, "https://") && !strings.HasPrefix(endpoint, "http://") { - endpoint = "https://" + endpoint - } else if !parsed.UseSSL && !strings.HasPrefix(endpoint, "http://") && !strings.HasPrefix(endpoint, "https://") { - endpoint = "http://" + endpoint - } - - region := parsed.Region - if region == "" { - region = cfg.S3.Region - if region == "" { - region = "us-east-1" - } - } - - // Credentials come from config (not URL for security) - client, err := s3.NewClient(context.Background(), s3.Config{ - Endpoint: endpoint, - Bucket: parsed.Bucket, - Prefix: parsed.Prefix, - AccessKeyID: cfg.S3.AccessKeyID, - SecretAccessKey: cfg.S3.SecretAccessKey, - Region: region, - }) - if err != nil { - return nil, fmt.Errorf("creating S3 client: %w", err) - } - - return NewS3Storer(client), nil - - case "rclone": - return NewRcloneStorer(context.Background(), parsed.RcloneRemote, parsed.Prefix) + case schemeRclone: + return NewRcloneStorer( + context.Background(), parsed.RcloneRemote, parsed.Prefix) default: - return nil, fmt.Errorf("unsupported storage scheme: %s", parsed.Scheme) + return nil, fmt.Errorf("%w: %s", ErrUnsupportedStorage, parsed.Scheme) } } +// storerFromParsedS3URL builds an S3 storer from a parsed s3:// URL, +// filling endpoint protocol and region defaults from the config. +// +//nolint:ireturn // factory intentionally returns the Storer interface +func storerFromParsedS3URL(parsed *URL, cfg *config.Config) (Storer, error) { + // Build endpoint URL + endpoint := parsed.Endpoint + if endpoint == "" { + endpoint = defaultS3Endpoint + } + + // Add protocol if not present + hasProtocol := strings.HasPrefix(endpoint, "https://") || + strings.HasPrefix(endpoint, "http://") + if !hasProtocol { + if parsed.UseSSL { + endpoint = "https://" + endpoint + } else { + endpoint = "http://" + endpoint + } + } + + region := parsed.Region + if region == "" { + region = cfg.S3.Region + if region == "" { + region = defaultS3Region + } + } + + // Credentials come from config (not URL for security) + client, err := s3.NewClient(context.Background(), s3.Config{ + Endpoint: endpoint, + Bucket: parsed.Bucket, + Prefix: parsed.Prefix, + AccessKeyID: cfg.S3.AccessKeyID, + SecretAccessKey: cfg.S3.SecretAccessKey, + Region: region, + }) + if err != nil { + return nil, fmt.Errorf("creating S3 client: %w", err) + } + + return NewS3Storer(client), nil +} + +//nolint:ireturn // factory intentionally returns the Storer interface func storerFromLegacyS3Config(cfg *config.Config) (Storer, error) { endpoint := cfg.S3.Endpoint // Ensure protocol is present - if !strings.HasPrefix(endpoint, "http://") && !strings.HasPrefix(endpoint, "https://") { + if !strings.HasPrefix(endpoint, "http://") && + !strings.HasPrefix(endpoint, "https://") { if cfg.S3.UseSSL { endpoint = "https://" + endpoint } else { @@ -97,7 +123,7 @@ func storerFromLegacyS3Config(cfg *config.Config) (Storer, error) { region := cfg.S3.Region if region == "" { - region = "us-east-1" + region = defaultS3Region } client, err := s3.NewClient(context.Background(), s3.Config{ diff --git a/internal/storage/rclone.go b/internal/storage/rclone.go index 071f4cb..c3a3433 100644 --- a/internal/storage/rclone.go +++ b/internal/storage/rclone.go @@ -69,7 +69,8 @@ func (r *RcloneStorer) Put(ctx context.Context, key string, data io.Reader) erro } // Upload the object - _, err = operations.Rcat(ctx, r.fsys, key, io.NopCloser(bytes.NewReader(buf)), time.Now(), nil) + _, err = operations.Rcat(ctx, r.fsys, key, + io.NopCloser(bytes.NewReader(buf)), time.Now(), nil) if err != nil { return fmt.Errorf("uploading object: %w", err) } @@ -78,7 +79,10 @@ func (r *RcloneStorer) Put(ctx context.Context, key string, data io.Reader) erro } // PutWithProgress stores data with progress reporting. -func (r *RcloneStorer) PutWithProgress(ctx context.Context, key string, data io.Reader, size int64, progress ProgressCallback) error { +func (r *RcloneStorer) PutWithProgress( + ctx context.Context, key string, data io.Reader, + _ int64, progress ProgressCallback, +) error { // Wrap reader with progress tracking pr := &progressReader{ reader: data, @@ -181,7 +185,9 @@ func (r *RcloneStorer) List(ctx context.Context, prefix string) ([]string, error } // ListStream returns a channel of ObjectInfo for large result sets. -func (r *RcloneStorer) ListStream(ctx context.Context, prefix string) <-chan ObjectInfo { +func (r *RcloneStorer) ListStream( + ctx context.Context, prefix string, +) <-chan ObjectInfo { ch := make(chan ObjectInfo) go func() { @@ -212,14 +218,14 @@ func (r *RcloneStorer) ListStream(ctx context.Context, prefix string) <-chan Obj } // Info returns human-readable storage location information. -func (r *RcloneStorer) Info() StorageInfo { +func (r *RcloneStorer) Info() Info { location := r.remote if r.path != "" { location += ":" + r.path } - return StorageInfo{ - Type: "rclone", + return Info{ + Type: schemeRclone, Location: location, } } diff --git a/internal/storage/s3.go b/internal/storage/s3.go index 64b3ede..580ac80 100644 --- a/internal/storage/s3.go +++ b/internal/storage/s3.go @@ -24,7 +24,10 @@ func (s *S3Storer) Put(ctx context.Context, key string, data io.Reader) error { } // PutWithProgress stores data with progress reporting. -func (s *S3Storer) PutWithProgress(ctx context.Context, key string, data io.Reader, size int64, progress ProgressCallback) error { +func (s *S3Storer) PutWithProgress( + ctx context.Context, key string, data io.Reader, + size int64, progress ProgressCallback, +) error { // Convert storage.ProgressCallback to s3.ProgressCallback var s3Progress s3.ProgressCallback if progress != nil { @@ -81,8 +84,8 @@ func (s *S3Storer) ListStream(ctx context.Context, prefix string) <-chan ObjectI } // Info returns human-readable storage location information. -func (s *S3Storer) Info() StorageInfo { - return StorageInfo{ +func (s *S3Storer) Info() Info { + return Info{ Type: "s3", Location: fmt.Sprintf("%s/%s", s.client.Endpoint(), s.client.BucketName()), } diff --git a/internal/storage/storer.go b/internal/storage/storer.go index 9cd4e25..2334a7d 100644 --- a/internal/storage/storer.go +++ b/internal/storage/storer.go @@ -30,14 +30,15 @@ type ObjectInfo struct { Err error // Error for streaming results (nil on success) } -// StorageInfo provides human-readable storage configuration. -type StorageInfo struct { +// Info provides human-readable storage configuration. +type Info struct { Type string // "s3" or "file" Location string // endpoint/bucket for S3, base path for filesystem } // Storer defines the interface for storage backends. -// All paths are relative to the storage root (bucket/prefix for S3, base directory for filesystem). +// All paths are relative to the storage root (bucket/prefix for S3, base +// directory for filesystem). type Storer interface { // Put stores data at the specified key. // Parent directories are created automatically for filesystem backends. @@ -46,7 +47,8 @@ type Storer interface { // PutWithProgress stores data with progress reporting. // Size must be the exact size of the data to store. // The progress callback is called periodically with bytes transferred. - PutWithProgress(ctx context.Context, key string, data io.Reader, size int64, progress ProgressCallback) error + PutWithProgress(ctx context.Context, key string, data io.Reader, + size int64, progress ProgressCallback) error // Get retrieves data from the specified key. // The caller must close the returned ReadCloser. @@ -70,5 +72,5 @@ type Storer interface { ListStream(ctx context.Context, prefix string) <-chan ObjectInfo // Info returns human-readable storage location information. - Info() StorageInfo + Info() Info } diff --git a/internal/storage/url.go b/internal/storage/url.go index 4ab169d..1e3ddf2 100644 --- a/internal/storage/url.go +++ b/internal/storage/url.go @@ -7,8 +7,26 @@ import ( "strings" ) -// StorageURL represents a parsed storage URL. -type StorageURL struct { +// Storage URL scheme names. +const ( + schemeFile = "file" + schemeS3 = "s3" + schemeRclone = "rclone" +) + +// Sentinel errors for storage URL parsing. +var ( + ErrEmptyStorageURL = errors.New("storage URL is empty") + ErrEmptyFilePath = errors.New("file URL path is empty") + ErrMissingBucket = errors.New("s3 URL missing bucket name") + ErrMissingRemote = errors.New("rclone URL missing remote name") + ErrUnsupportedScheme = errors.New( + "unsupported URL scheme: must start with s3://, file://, or rclone://") + ErrUnsupportedStorage = errors.New("unsupported storage scheme") +) + +// URL represents a parsed storage URL. +type URL struct { Scheme string // "s3", "file", or "rclone" Bucket string // S3 bucket name (empty for file/rclone) Prefix string // Path within bucket or filesystem base path @@ -23,20 +41,20 @@ type StorageURL struct { // - s3://bucket/prefix?endpoint=host®ion=us-east-1&ssl=true // - file:///absolute/path/to/backup // - rclone://remote/path/to/backups -func ParseStorageURL(rawURL string) (*StorageURL, error) { +func ParseStorageURL(rawURL string) (*URL, error) { if rawURL == "" { - return nil, errors.New("storage URL is empty") + return nil, ErrEmptyStorageURL } // Handle file:// URLs if after, ok := strings.CutPrefix(rawURL, "file://"); ok { path := after if path == "" { - return nil, errors.New("file URL path is empty") + return nil, ErrEmptyFilePath } - return &StorageURL{ - Scheme: "file", + return &URL{ + Scheme: schemeFile, Prefix: path, }, nil } @@ -50,7 +68,7 @@ func ParseStorageURL(rawURL string) (*StorageURL, error) { bucket := u.Host if bucket == "" { - return nil, errors.New("s3 URL missing bucket name") + return nil, ErrMissingBucket } prefix := strings.TrimPrefix(u.Path, "/") @@ -62,8 +80,8 @@ func ParseStorageURL(rawURL string) (*StorageURL, error) { useSSL = false } - return &StorageURL{ - Scheme: "s3", + return &URL{ + Scheme: schemeS3, Bucket: bucket, Prefix: prefix, Endpoint: query.Get("endpoint"), @@ -81,27 +99,27 @@ func ParseStorageURL(rawURL string) (*StorageURL, error) { remote := u.Host if remote == "" { - return nil, errors.New("rclone URL missing remote name") + return nil, ErrMissingRemote } path := strings.TrimPrefix(u.Path, "/") - return &StorageURL{ - Scheme: "rclone", + return &URL{ + Scheme: schemeRclone, Prefix: path, RcloneRemote: remote, }, nil } - return nil, errors.New("unsupported URL scheme: must start with s3://, file://, or rclone://") + return nil, ErrUnsupportedScheme } // String returns a human-readable representation of the storage URL. -func (u *StorageURL) String() string { +func (u *URL) String() string { switch u.Scheme { - case "file": + case schemeFile: return "file://" + u.Prefix - case "s3": + case schemeS3: endpoint := u.Endpoint if endpoint == "" { endpoint = "s3.amazonaws.com" @@ -112,7 +130,7 @@ func (u *StorageURL) String() string { } return fmt.Sprintf("s3://%s (endpoint: %s)", u.Bucket, endpoint) - case "rclone": + case schemeRclone: if u.Prefix != "" { return fmt.Sprintf("rclone://%s/%s", u.RcloneRemote, u.Prefix) } diff --git a/internal/types/types.go b/internal/types/types.go index 9abc02c..c076f41 100644 --- a/internal/types/types.go +++ b/internal/types/types.go @@ -1,16 +1,26 @@ -// Package types provides custom types for better type safety across the vaultik codebase. -// Using distinct types for IDs, hashes, paths, and credentials prevents accidental -// mixing of semantically different values that happen to share the same underlying type. +// Package types provides custom types for better type safety across the +// vaultik codebase. Using distinct types for IDs, hashes, paths, and +// credentials prevents accidental mixing of semantically different values +// that happen to share the same underlying type. package types import ( "database/sql/driver" + "errors" "fmt" "github.com/google/uuid" ) +// errCannotScan is returned when a database value cannot be scanned into +// an ID type. +var errCannotScan = errors.New("cannot scan value") + // FileID is a UUID identifying a file record in the database. +// +// used on values. +// +//nolint:recvcheck // Scan requires a pointer receiver; String/Value are type FileID uuid.UUID // NewFileID generates a new random FileID. @@ -54,7 +64,7 @@ func (id *FileID) Scan(src any) error { case []byte: s = string(v) default: - return fmt.Errorf("cannot scan %T into FileID", src) + return fmt.Errorf("%w: %T into FileID", errCannotScan, src) } parsed, err := uuid.Parse(s) @@ -69,6 +79,10 @@ func (id *FileID) Scan(src any) error { // BlobID is a UUID identifying a blob record in the database. // This is distinct from BlobHash which is the content-addressed hash of the blob. +// +// used on values. +// +//nolint:recvcheck // Scan requires a pointer receiver; String/Value are type BlobID uuid.UUID // NewBlobID generates a new random BlobID. @@ -112,7 +126,7 @@ func (id *BlobID) Scan(src any) error { case []byte: s = string(v) default: - return fmt.Errorf("cannot scan %T into BlobID", src) + return fmt.Errorf("%w: %T into BlobID", errCannotScan, src) } parsed, err := uuid.Parse(s) @@ -208,6 +222,8 @@ func (p GlobPattern) String() string { return string(p) } func (k AgeSecretKey) String() string { return "[REDACTED]" } func (k AWSSecretAccessKey) String() string { return "[REDACTED]" } -// Raw returns the actual value for sensitive types when explicitly needed -func (k AgeSecretKey) Raw() string { return string(k) } +// Raw returns the actual value for sensitive types when explicitly needed. +func (k AgeSecretKey) Raw() string { return string(k) } + +// Raw returns the actual value for sensitive types when explicitly needed. func (k AWSSecretAccessKey) Raw() string { return string(k) } diff --git a/internal/ui/ui.go b/internal/ui/ui.go index b54e6cd..8216c83 100644 --- a/internal/ui/ui.go +++ b/internal/ui/ui.go @@ -4,14 +4,14 @@ // // Message classes (see Writer methods): // -// - Begin — operation start, left-aligned, marker "》" (white) -// - Complete— operation completion, left-aligned, marker "》" (green) -// - Info — left-aligned neutral status, marker "》" (white) -// - Notice — left-aligned important note, marker "》" (cyan) -// - Warning — left-aligned warning, full word "Warning: " (orange/yellow) -// - Error — left-aligned error, full word "ERROR: " (red) -// - Progress— indented heartbeat / per-item update, marker " 》" (white) -// - Banner — application banner line, left-aligned, no marker +// - Beginf — operation start, left-aligned, marker "》" (white) +// - Completef— operation completion, left-aligned, marker "》" (green) +// - Infof — left-aligned neutral status, marker "》" (white) +// - Noticef — left-aligned important note, marker "》" (cyan) +// - Warningf — left-aligned warning, full word "Warning: " (orange/yellow) +// - Errorf — left-aligned error, full word "ERROR: " (red) +// - Progressf— indented heartbeat / per-item update, marker " 》" (white) +// - Bannerf — application banner line, left-aligned, no marker // // Value formatters (Hex, Size, Duration, Time, Path, Snapshot, Speed, // Count, Percent) return ANSI-colored strings the caller composes into @@ -45,6 +45,19 @@ const ( // Marker is the chevron prefix used for all non-error/warning lines. const Marker = "》" +const ( + // hexAbbrevLen is the number of hash characters Hex keeps before "...". + hexAbbrevLen = 12 + + // bitsPerByte converts bytes/sec into bits/sec for Speed. + bitsPerByte = 8 + + // SI thresholds for Speed's unit selection, in bits/sec. + gigabit = 1e9 + megabit = 1e6 + kilobit = 1e3 +) + // Writer formats and emits user-facing messages with optional ANSI color. // It also counts warnings and errors emitted so the caller can summarize at // the end of an operation ("Finished successfully." vs "Finished with @@ -103,19 +116,10 @@ func shouldColor(w io.Writer) bool { return term.IsTerminal(int(f.Fd())) } -// paint wraps s in the given ANSI color when color is enabled. -func (w *Writer) paint(color, s string) string { - if !w.color { - return s - } - - return color + s + ansiReset -} - // ───────────────────────── message methods ───────────────────────── -// Begin prints an operation-start line, left-aligned with a white marker. -func (w *Writer) Begin(format string, args ...any) { +// Beginf prints an operation-start line, left-aligned with a white marker. +func (w *Writer) Beginf(format string, args ...any) { if w.quiet { return } @@ -123,8 +127,8 @@ func (w *Writer) Begin(format string, args ...any) { w.emit(ansiWhite, Marker, "", format, args) } -// Complete prints an operation-completion line in green, left-aligned. -func (w *Writer) Complete(format string, args ...any) { +// Completef prints an operation-completion line in green, left-aligned. +func (w *Writer) Completef(format string, args ...any) { if w.quiet { return } @@ -132,8 +136,8 @@ func (w *Writer) Complete(format string, args ...any) { w.emit(ansiGreen, Marker, ansiGreen, format, args) } -// Info prints a neutral status line, left-aligned with a white marker. -func (w *Writer) Info(format string, args ...any) { +// Infof prints a neutral status line, left-aligned with a white marker. +func (w *Writer) Infof(format string, args ...any) { if w.quiet { return } @@ -141,8 +145,8 @@ func (w *Writer) Info(format string, args ...any) { w.emit(ansiWhite, Marker, "", format, args) } -// Notice prints an attention-worthy informational line, marker in cyan. -func (w *Writer) Notice(format string, args ...any) { +// Noticef prints an attention-worthy informational line, marker in cyan. +func (w *Writer) Noticef(format string, args ...any) { if w.quiet { return } @@ -150,27 +154,27 @@ func (w *Writer) Notice(format string, args ...any) { w.emit(ansiCyan, Marker, "", format, args) } -// Warning prints "⚠️ Warning: " in orange/yellow followed by the message. -func (w *Writer) Warning(format string, args ...any) { +// Warningf prints "⚠️ Warning: " in orange/yellow followed by the message. +func (w *Writer) Warningf(format string, args ...any) { w.warnings++ prefix := "⚠️ " + w.paint(ansiYellow+ansiBold, "Warning: ") _, _ = fmt.Fprintln(w.out, prefix+fmt.Sprintf(format, args...)) } -// Error prints "🛑 ERROR: " in red followed by the message. Goes to the +// Errorf prints "🛑 ERROR: " in red followed by the message. Goes to the // same writer as everything else; callers that want stderr should // construct a separate Writer for it. -func (w *Writer) Error(format string, args ...any) { +func (w *Writer) Errorf(format string, args ...any) { w.errors++ prefix := "🛑 " + w.paint(ansiRed+ansiBold, "ERROR: ") _, _ = fmt.Fprintln(w.out, prefix+fmt.Sprintf(format, args...)) } -// Detail prints an indented continuation line under a preceding Complete +// Detailf prints an indented continuation line under a preceding Completef // (or other top-level message). Marker " 》" (white) at column 2. -// Distinct from Progress (semantically a "heartbeat") in usage but +// Distinct from Progressf (semantically a "heartbeat") in usage but // visually identical. -func (w *Writer) Detail(format string, args ...any) { +func (w *Writer) Detailf(format string, args ...any) { if w.quiet { return } @@ -184,8 +188,8 @@ func (w *Writer) WarningCount() int { return w.warnings } // ErrorCount returns the number of Error() calls this writer has emitted. func (w *Writer) ErrorCount() int { return w.errors } -// Progress prints an indented heartbeat / per-item update, marker in white. -func (w *Writer) Progress(format string, args ...any) { +// Progressf prints an indented heartbeat / per-item update, marker in white. +func (w *Writer) Progressf(format string, args ...any) { if w.quiet { return } @@ -193,9 +197,9 @@ func (w *Writer) Progress(format string, args ...any) { w.emit(ansiWhite, " "+Marker, "", format, args) } -// Banner prints a line with no marker, left-aligned. Bold when color +// Bannerf prints a line with no marker, left-aligned. Bold when color // is enabled. Used for the application startup banner only. -func (w *Writer) Banner(format string, args ...any) { +func (w *Writer) Bannerf(format string, args ...any) { if w.quiet { return } @@ -208,17 +212,6 @@ func (w *Writer) Banner(format string, args ...any) { _, _ = fmt.Fprintln(w.out, body) } -// emit writes " \n" with the prefix painted in prefixColor -// and the body optionally painted in bodyColor (empty = no body color). -func (w *Writer) emit(prefixColor, prefix, bodyColor, format string, args []any) { - body := fmt.Sprintf(format, args...) - if bodyColor != "" { - body = w.paint(bodyColor, body) - } - - _, _ = fmt.Fprintln(w.out, w.paint(prefixColor, prefix)+" "+body) -} - // ───────────────────────── value formatters ───────────────────────── // // These return ANSI-colored strings the caller composes into a message @@ -228,8 +221,8 @@ func (w *Writer) emit(prefixColor, prefix, bodyColor, format string, args []any) // Long hashes are abbreviated to first 12 chars with "...". func (w *Writer) Hex(s string) string { short := s - if len(s) > 12 { - short = s[:12] + "..." + if len(s) > hexAbbrevLen { + short = s[:hexAbbrevLen] + "..." } return w.paint(ansiCyan, short) @@ -247,7 +240,7 @@ func (w *Writer) Path(p string) string { // Size colorizes a byte count using humanize.Bytes. func (w *Writer) Size(bytes int64) string { - return w.paint(ansiMagenta, humanize.Bytes(uint64(bytes))) + return w.paint(ansiMagenta, humanize.Bytes(uint64(bytes))) //nolint:gosec // G115: >=0 } // Speed colorizes a network transfer rate. Input is bytes/sec; output is @@ -258,17 +251,17 @@ func (w *Writer) Speed(bytesPerSec float64) string { return w.paint(ansiMagenta, "N/A") } - bitsPerSec := bytesPerSec * 8 + bitsPerSec := bytesPerSec * bitsPerByte var s string switch { - case bitsPerSec >= 1e9: - s = fmt.Sprintf("%.1f Gbit/sec", bitsPerSec/1e9) - case bitsPerSec >= 1e6: - s = fmt.Sprintf("%.0f Mbit/sec", bitsPerSec/1e6) - case bitsPerSec >= 1e3: - s = fmt.Sprintf("%.0f Kbit/sec", bitsPerSec/1e3) + case bitsPerSec >= gigabit: + s = fmt.Sprintf("%.1f Gbit/sec", bitsPerSec/gigabit) + case bitsPerSec >= megabit: + s = fmt.Sprintf("%.0f Mbit/sec", bitsPerSec/megabit) + case bitsPerSec >= kilobit: + s = fmt.Sprintf("%.0f Kbit/sec", bitsPerSec/kilobit) default: s = fmt.Sprintf("%.0f bit/sec", bitsPerSec) } @@ -286,7 +279,7 @@ func (w *Writer) Duration(d time.Duration) string { // "YYYY-MM-DD HH:MM:SS". No timezone is included — values are // displayed in the process's local zone. func (w *Writer) Time(t time.Time) string { - t = t.Local() + t = t.Local() //nolint:gosmopolitan // local-time display is intentional now := time.Now() if t.Year() == now.Year() && t.YearDay() == now.YearDay() { @@ -305,3 +298,25 @@ func (w *Writer) Count(n int) string { func (w *Writer) Percent(p float64) string { return w.paint(ansiMagenta, fmt.Sprintf("%.1f%%", p)) } + +// ───────────────────────── internal helpers ───────────────────────── + +// paint wraps s in the given ANSI color when color is enabled. +func (w *Writer) paint(color, s string) string { + if !w.color { + return s + } + + return color + s + ansiReset +} + +// emit writes " \n" with the prefix painted in prefixColor +// and the body optionally painted in bodyColor (empty = no body color). +func (w *Writer) emit(prefixColor, prefix, bodyColor, format string, args []any) { + body := fmt.Sprintf(format, args...) + if bodyColor != "" { + body = w.paint(bodyColor, body) + } + + _, _ = fmt.Fprintln(w.out, w.paint(prefixColor, prefix)+" "+body) +} diff --git a/internal/ui/ui_test.go b/internal/ui/ui_test.go index b432c52..7da92c3 100644 --- a/internal/ui/ui_test.go +++ b/internal/ui/ui_test.go @@ -1,37 +1,45 @@ -package ui +package ui_test import ( "bytes" "strings" "testing" "time" + + "sneak.berlin/go/vaultik/internal/ui" ) -func newTestWriter(color bool) (*Writer, *bytes.Buffer) { +func newTestWriter(color bool) (*ui.Writer, *bytes.Buffer) { buf := &bytes.Buffer{} - return NewWithColor(buf, color), buf + return ui.NewWithColor(buf, color), buf } func TestMessageMethodsPlain(t *testing.T) { + t.Parallel() + tests := []struct { method string - fn func(*Writer) + fn func(*ui.Writer) want string }{ - {"Begin", func(w *Writer) { w.Begin("starting %s", "thing") }, "》 starting thing\n"}, - {"Complete", func(w *Writer) { w.Complete("done %s", "thing") }, "》 done thing\n"}, - {"Info", func(w *Writer) { w.Info("status") }, "》 status\n"}, - {"Notice", func(w *Writer) { w.Notice("note") }, "》 note\n"}, - {"Warning", func(w *Writer) { w.Warning("oops") }, "⚠️ Warning: oops\n"}, - {"Error", func(w *Writer) { w.Error("boom") }, "🛑 ERROR: boom\n"}, - {"Progress", func(w *Writer) { w.Progress("p") }, " 》 p\n"}, - {"Detail", func(w *Writer) { w.Detail("d") }, " 》 d\n"}, - {"Banner", func(w *Writer) { w.Banner("hello") }, "hello\n"}, // plain mode, no bold + {"Begin", func(w *ui.Writer) { w.Beginf("starting %s", "thing") }, + "》 starting thing\n"}, + {"Complete", func(w *ui.Writer) { w.Completef("done %s", "thing") }, + "》 done thing\n"}, + {"Info", func(w *ui.Writer) { w.Infof("status") }, "》 status\n"}, + {"Notice", func(w *ui.Writer) { w.Noticef("note") }, "》 note\n"}, + {"Warning", func(w *ui.Writer) { w.Warningf("oops") }, "⚠️ Warning: oops\n"}, + {"Error", func(w *ui.Writer) { w.Errorf("boom") }, "🛑 ERROR: boom\n"}, + {"Progress", func(w *ui.Writer) { w.Progressf("p") }, " 》 p\n"}, + {"Detail", func(w *ui.Writer) { w.Detailf("d") }, " 》 d\n"}, + {"Banner", func(w *ui.Writer) { w.Bannerf("hello") }, "hello\n"}, // plain, no bold } for _, tt := range tests { t.Run(tt.method, func(t *testing.T) { + t.Parallel() + w, buf := newTestWriter(false) tt.fn(w) @@ -43,15 +51,17 @@ func TestMessageMethodsPlain(t *testing.T) { } func TestWarningErrorCounters(t *testing.T) { + t.Parallel() + w, _ := newTestWriter(false) if w.WarningCount() != 0 || w.ErrorCount() != 0 { t.Fatalf("expected fresh writer to have zero counts") } - w.Info("normal") - w.Warning("first warn") - w.Warning("second warn") - w.Error("only error") + w.Infof("normal") + w.Warningf("first warn") + w.Warningf("second warn") + w.Errorf("only error") if got, want := w.WarningCount(), 2; got != want { t.Errorf("WarningCount: got %d, want %d", got, want) @@ -63,8 +73,10 @@ func TestWarningErrorCounters(t *testing.T) { } func TestColorOutputContainsANSI(t *testing.T) { + t.Parallel() + w, buf := newTestWriter(true) - w.Error("boom") + w.Errorf("boom") out := buf.String() if !strings.Contains(out, "\033[") { @@ -77,8 +89,10 @@ func TestColorOutputContainsANSI(t *testing.T) { } func TestBannerBoldWhenColor(t *testing.T) { + t.Parallel() + w, buf := newTestWriter(true) - w.Banner("hello") + w.Bannerf("hello") out := buf.String() if !strings.Contains(out, "\033[1m") { @@ -87,6 +101,8 @@ func TestBannerBoldWhenColor(t *testing.T) { } func TestValueFormattersPlain(t *testing.T) { + t.Parallel() + w, _ := newTestWriter(false) if got := w.Hex("0123456789abcdef0123"); got != "0123456789ab..." { @@ -127,18 +143,26 @@ func TestValueFormattersPlain(t *testing.T) { } // Time format: today → HH:MM:SS, other day → YYYY-MM-DD HH:MM:SS. - today := time.Date(time.Now().Year(), time.Now().Month(), time.Now().Day(), 14, 30, 45, 0, time.Local) + // These construct local-zone times on purpose: Writer.Time displays + // in the process's local zone. + now := time.Now() + + today := time.Date(now.Year(), now.Month(), now.Day(), + 14, 30, 45, 0, time.Local) //nolint:gosmopolitan // local display if got := w.Time(today); got != "14:30:45" { t.Errorf("Time today: got %q, want 14:30:45", got) } - other := time.Date(2030, 1, 2, 3, 4, 5, 0, time.Local) + other := time.Date(2030, 1, 2, 3, 4, 5, 0, + time.Local) //nolint:gosmopolitan // local display if got := w.Time(other); got != "2030-01-02 03:04:05" { t.Errorf("Time other day: got %q", got) } } func TestValueFormattersColored(t *testing.T) { + t.Parallel() + w, _ := newTestWriter(true) hex := w.Hex("0123456789abcdef0123") diff --git a/internal/vaultik/blob_fetch.go b/internal/vaultik/blob_fetch.go index 1bd5931..fde9eb2 100644 --- a/internal/vaultik/blob_fetch.go +++ b/internal/vaultik/blob_fetch.go @@ -14,6 +14,10 @@ import ( "sneak.berlin/go/vaultik/internal/log" ) +// errBlobHashMismatch is returned when a fetched blob's content hash does +// not match the expected double-SHA-256 hash. +var errBlobHashMismatch = errors.New("blob hash mismatch") + // hashVerifyReader wraps a blobgen.Reader and verifies the double-SHA-256 hash // of decrypted plaintext when Close is called. It reuses the hash that // blobgen.Reader already computes internally via its TeeReader, avoiding @@ -46,7 +50,8 @@ func (h *hashVerifyReader) Close() error { actualHashHex := hex.EncodeToString(secondHasher.Sum(nil)) if actualHashHex != h.blobHash { - return fmt.Errorf("blob hash mismatch: expected %s, got %s", h.blobHash[:16], actualHashHex[:16]) + return fmt.Errorf("%w: expected %s, got %s", + errBlobHashMismatch, h.blobHash[:16], actualHashHex[:16]) } } @@ -61,7 +66,9 @@ func (h *hashVerifyReader) Close() error { // returns a streaming reader that computes the double-SHA-256 hash on the fly. // The hash is verified when the returned reader is closed (after fully reading). // This avoids buffering the entire blob in memory. -func (v *Vaultik) FetchAndDecryptBlob(ctx context.Context, blobHash string, expectedSize int64, identity age.Identity) (io.ReadCloser, error) { +func (v *Vaultik) FetchAndDecryptBlob( + ctx context.Context, blobHash string, expectedSize int64, identity age.Identity, +) (io.ReadCloser, error) { rc, _, err := v.FetchBlob(ctx, blobHash, expectedSize) if err != nil { return nil, err @@ -85,7 +92,9 @@ func (v *Vaultik) FetchAndDecryptBlob(ctx context.Context, blobHash string, expe // Times the Storage.Get and Storage.Stat round-trips separately at // debug level so we can see whether the size-only Stat (which is an // extra request on every fetch) is hurting throughput. -func (v *Vaultik) FetchBlob(ctx context.Context, blobHash string, expectedSize int64) (io.ReadCloser, int64, error) { +func (v *Vaultik) FetchBlob( + ctx context.Context, blobHash string, expectedSize int64, +) (io.ReadCloser, int64, error) { blobPath := fmt.Sprintf("blobs/%s/%s/%s", blobHash[:2], blobHash[2:4], blobHash) t0 := time.Now() diff --git a/internal/vaultik/blob_fetch_hash_test.go b/internal/vaultik/blob_fetch_hash_test.go index 041b989..8334eea 100644 --- a/internal/vaultik/blob_fetch_hash_test.go +++ b/internal/vaultik/blob_fetch_hash_test.go @@ -14,20 +14,17 @@ import ( "sneak.berlin/go/vaultik/internal/vaultik" ) -// TestFetchAndDecryptBlobVerifiesHash verifies that FetchAndDecryptBlob checks -// the double-SHA-256 hash of the decrypted plaintext against the expected blob hash. -func TestFetchAndDecryptBlobVerifiesHash(t *testing.T) { - identity, err := age.GenerateX25519Identity() - if err != nil { - t.Fatalf("generating identity: %v", err) - } - - // Create test data and encrypt it using blobgen.Writer - plaintext := []byte("hello world test data for blob hash verification") +// buildHashTestBlob encrypts plaintext with blobgen.Writer and returns +// the encrypted bytes plus the expected double-SHA-256 hash. +func buildHashTestBlob( + t *testing.T, identity *age.X25519Identity, plaintext []byte, +) ([]byte, string) { + t.Helper() var encBuf bytes.Buffer - writer, err := blobgen.NewWriter(&encBuf, 1, []string{identity.Recipient().String()}) + writer, err := blobgen.NewWriter(&encBuf, 1, + []string{identity.Recipient().String()}) if err != nil { t.Fatalf("creating blobgen writer: %v", err) } @@ -42,9 +39,8 @@ func TestFetchAndDecryptBlobVerifiesHash(t *testing.T) { t.Fatalf("closing writer: %v", err) } - encryptedData := encBuf.Bytes() - - // Compute correct double-SHA-256 hash of the plaintext (matches blobgen.Writer.Sum256) + // Compute the double-SHA-256 hash of the plaintext (matches + // blobgen.Writer.Sum256). firstHash := sha256.Sum256(plaintext) secondHash := sha256.Sum256(firstHash[:]) correctHash := hex.EncodeToString(secondHash[:]) @@ -52,12 +48,30 @@ func TestFetchAndDecryptBlobVerifiesHash(t *testing.T) { // Verify our hash matches what blobgen.Writer produces writerHash := hex.EncodeToString(writer.Sum256()) if correctHash != writerHash { - t.Fatalf("hash computation mismatch: manual=%s, writer=%s", correctHash, writerHash) + t.Fatalf("hash computation mismatch: manual=%s, writer=%s", + correctHash, writerHash) } + return encBuf.Bytes(), correctHash +} + +// TestFetchAndDecryptBlobVerifiesHash verifies that FetchAndDecryptBlob checks +// the double-SHA-256 hash of the decrypted plaintext against the expected blob hash. +func TestFetchAndDecryptBlobVerifiesHash(t *testing.T) { + t.Parallel() + + identity, err := age.GenerateX25519Identity() + if err != nil { + t.Fatalf("generating identity: %v", err) + } + + plaintext := []byte("hello world test data for blob hash verification") + encryptedData, correctHash := buildHashTestBlob(t, identity, plaintext) + // Set up mock storage with the blob at the correct path mockStorage := NewMockStorer() - blobPath := "blobs/" + correctHash[:2] + "/" + correctHash[2:4] + "/" + correctHash + blobPath := "blobs/" + correctHash[:2] + "/" + + correctHash[2:4] + "/" + correctHash mockStorage.mu.Lock() mockStorage.data[blobPath] = encryptedData @@ -67,7 +81,10 @@ func TestFetchAndDecryptBlobVerifiesHash(t *testing.T) { ctx := context.Background() t.Run("correct hash succeeds", func(t *testing.T) { - rc, err := tv.FetchAndDecryptBlob(ctx, correctHash, int64(len(encryptedData)), identity) + t.Parallel() + + rc, err := tv.FetchAndDecryptBlob( + ctx, correctHash, int64(len(encryptedData)), identity) if err != nil { t.Fatalf("expected success, got error: %v", err) } @@ -88,6 +105,8 @@ func TestFetchAndDecryptBlobVerifiesHash(t *testing.T) { }) t.Run("wrong hash fails", func(t *testing.T) { + t.Parallel() + // Use a fake hash that doesn't match the actual plaintext fakeHash := strings.Repeat("ab", 32) // 64 hex chars fakePath := "blobs/" + fakeHash[:2] + "/" + fakeHash[2:4] + "/" + fakeHash @@ -96,7 +115,8 @@ func TestFetchAndDecryptBlobVerifiesHash(t *testing.T) { mockStorage.data[fakePath] = encryptedData mockStorage.mu.Unlock() - rc, err := tv.FetchAndDecryptBlob(ctx, fakeHash, int64(len(encryptedData)), identity) + rc, err := tv.FetchAndDecryptBlob( + ctx, fakeHash, int64(len(encryptedData)), identity) if err != nil { t.Fatalf("unexpected error opening stream: %v", err) } diff --git a/internal/vaultik/blobcache.go b/internal/vaultik/blobcache.go index 1b39da7..f6a3bd6 100644 --- a/internal/vaultik/blobcache.go +++ b/internal/vaultik/blobcache.go @@ -1,6 +1,7 @@ package vaultik import ( + "errors" "fmt" "io" "os" @@ -8,6 +9,15 @@ import ( "sync" ) +// Sentinel errors for blob cache lookups. +var ( + errCacheKeyMissing = errors.New("key not in cache") + errCacheReadBeyondBlob = errors.New("read beyond blob size") +) + +// blobCacheFileMode is the permission mode for cached blob files. +const blobCacheFileMode = 0o600 + // blobDiskCacheEntry tracks a cached blob on disk. type blobDiskCacheEntry struct { key string @@ -61,54 +71,8 @@ func newBlobDiskCache(maxBytes int64) (*blobDiskCache, error) { }, nil } -func (c *blobDiskCache) path(key string) string { - return filepath.Join(c.dir, key) -} - -func (c *blobDiskCache) unlink(e *blobDiskCacheEntry) { - if e.prev != nil { - e.prev.next = e.next - } else { - c.head = e.next - } - - if e.next != nil { - e.next.prev = e.prev - } else { - c.tail = e.prev - } - - e.prev = nil - e.next = nil -} - -func (c *blobDiskCache) pushFront(e *blobDiskCacheEntry) { - e.prev = nil - - e.next = c.head - if c.head != nil { - c.head.prev = e - } - - c.head = e - if c.tail == nil { - c.tail = e - } -} - -func (c *blobDiskCache) evictLRU() { - if c.tail == nil { - return - } - - victim := c.tail - c.unlink(victim) - delete(c.items, victim.key) - c.curBytes -= victim.size - _ = os.Remove(c.path(victim.key)) -} - -// Put writes blob data to disk cache. Entries larger than maxBytes are silently skipped. +// Put writes blob data to disk cache. Entries larger than maxBytes are +// silently skipped. func (c *blobDiskCache) Put(key string, data []byte) error { entrySize := int64(len(data)) @@ -127,7 +91,7 @@ func (c *blobDiskCache) Put(key string, data []byte) error { delete(c.items, key) } - err := os.WriteFile(c.path(key), data, 0600) + err := os.WriteFile(c.path(key), data, blobCacheFileMode) if err != nil { return fmt.Errorf("writing blob to cache: %w", err) } @@ -166,7 +130,8 @@ func (c *blobDiskCache) PutFromReader(key string, r io.Reader) (int64, error) { } c.mu.Unlock() - f, err := os.OpenFile(c.path(key), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600) + f, err := os.OpenFile( + c.path(key), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, blobCacheFileMode) if err != nil { return 0, fmt.Errorf("creating cache file: %w", err) } @@ -255,13 +220,14 @@ func (c *blobDiskCache) ReadAt(key string, offset, length int64) ([]byte, error) if !ok { c.mu.Unlock() - return nil, fmt.Errorf("key %q not in cache", key) + return nil, fmt.Errorf("%w: %q", errCacheKeyMissing, key) } if offset+length > e.size { c.mu.Unlock() - return nil, fmt.Errorf("read beyond blob size: offset=%d length=%d size=%d", offset, length, e.size) + return nil, fmt.Errorf("%w: offset=%d length=%d size=%d", + errCacheReadBeyondBlob, offset, length, e.size) } c.unlink(e) @@ -379,3 +345,50 @@ func (c *blobDiskCache) Close() error { return os.RemoveAll(c.dir) } + +func (c *blobDiskCache) path(key string) string { + return filepath.Join(c.dir, key) +} + +func (c *blobDiskCache) unlink(e *blobDiskCacheEntry) { + if e.prev != nil { + e.prev.next = e.next + } else { + c.head = e.next + } + + if e.next != nil { + e.next.prev = e.prev + } else { + c.tail = e.prev + } + + e.prev = nil + e.next = nil +} + +func (c *blobDiskCache) pushFront(e *blobDiskCacheEntry) { + e.prev = nil + + e.next = c.head + if c.head != nil { + c.head.prev = e + } + + c.head = e + if c.tail == nil { + c.tail = e + } +} + +func (c *blobDiskCache) evictLRU() { + if c.tail == nil { + return + } + + victim := c.tail + c.unlink(victim) + delete(c.items, victim.key) + c.curBytes -= victim.size + _ = os.Remove(c.path(victim.key)) +} diff --git a/internal/vaultik/blobcache_test.go b/internal/vaultik/blobcache_test.go index 45259a0..5d61872 100644 --- a/internal/vaultik/blobcache_test.go +++ b/internal/vaultik/blobcache_test.go @@ -1,4 +1,4 @@ -package vaultik +package vaultik //nolint:testpackage // exercises unexported blobDiskCache import ( "bytes" @@ -8,6 +8,8 @@ import ( ) func TestBlobDiskCache_BasicGetPut(t *testing.T) { + t.Parallel() + cache, err := newBlobDiskCache(1 << 20) if err != nil { t.Fatal(err) @@ -37,6 +39,8 @@ func TestBlobDiskCache_BasicGetPut(t *testing.T) { } func TestBlobDiskCache_EvictionUnderPressure(t *testing.T) { + t.Parallel() + maxBytes := int64(1000) cache, err := newBlobDiskCache(maxBytes) @@ -49,7 +53,7 @@ func TestBlobDiskCache_EvictionUnderPressure(t *testing.T) { for i := range 5 { data := make([]byte, 300) - err := cache.Put(fmt.Sprintf("key%d", i), data) + err = cache.Put(fmt.Sprintf("key%d", i), data) if err != nil { t.Fatal(err) } @@ -69,6 +73,8 @@ func TestBlobDiskCache_EvictionUnderPressure(t *testing.T) { } func TestBlobDiskCache_OversizedEntryRejected(t *testing.T) { + t.Parallel() + cache, err := newBlobDiskCache(100) if err != nil { t.Fatal(err) @@ -88,6 +94,8 @@ func TestBlobDiskCache_OversizedEntryRejected(t *testing.T) { } func TestBlobDiskCache_UpdateInPlace(t *testing.T) { + t.Parallel() + cache, err := newBlobDiskCache(1 << 20) if err != nil { t.Fatal(err) @@ -123,6 +131,8 @@ func TestBlobDiskCache_UpdateInPlace(t *testing.T) { } func TestBlobDiskCache_ReadAt(t *testing.T) { + t.Parallel() + cache, err := newBlobDiskCache(1 << 20) if err != nil { t.Fatal(err) @@ -162,6 +172,8 @@ func TestBlobDiskCache_ReadAt(t *testing.T) { } func TestBlobDiskCache_Close(t *testing.T) { + t.Parallel() + cache, err := newBlobDiskCache(1 << 20) if err != nil { t.Fatal(err) @@ -179,6 +191,8 @@ func TestBlobDiskCache_Close(t *testing.T) { } func TestBlobDiskCache_LRUOrder(t *testing.T) { + t.Parallel() + cache, err := newBlobDiskCache(200) if err != nil { t.Fatal(err) diff --git a/internal/vaultik/helpers.go b/internal/vaultik/helpers.go index 726805e..cc57832 100644 --- a/internal/vaultik/helpers.go +++ b/internal/vaultik/helpers.go @@ -8,13 +8,56 @@ import ( "strings" "time" + "github.com/dustin/go-humanize" "sneak.berlin/go/vaultik/internal/types" ) +// percentScale converts a 0..1 ratio into a percentage. +const percentScale = 100 + +// progressLogEvery is how many processed items pass between progress +// log lines in long-running loops. +const progressLogEvery = 100 + +// ubytes renders a byte count with humanize.Bytes, clamping negative +// values to zero so the int64→uint64 conversion cannot overflow. +func ubytes(n int64) string { + if n < 0 { + n = 0 + } + + return humanize.Bytes(uint64(n)) +} + +// Sentinel errors for snapshot ID and duration parsing. +var ( + errMalformedSnapshotID = errors.New( + "invalid snapshot ID format: expected hostname_snapshotname_timestamp") + errInvalidDuration = errors.New("invalid duration") + errUnknownTimeUnit = errors.New("unknown time unit") +) + +// Time-unit lengths used by parseDuration. +const ( + day = 24 * time.Hour + week = 7 * day + month = 30 * day + year = 365 * day +) + +// Snapshot IDs split on "_" into hostname, optional name parts, and a +// trailing timestamp. +const ( + minSnapshotIDParts = 2 + minSnapshotIDNameParts = 3 +) + // SnapshotInfo contains information about a snapshot. // UncompressedSize and NewChunkSize are populated only when the snapshot // is present in the local database; LocallyTracked indicates whether // those values are meaningful. +// +//nolint:tagliatelle // snake_case is the established output format type SnapshotInfo struct { ID types.SnapshotID `json:"id"` Timestamp time.Time `json:"timestamp"` @@ -44,8 +87,8 @@ func formatBytes(bytes int64) string { // Format: hostname_snapshotname_2026-01-12T14:41:15Z func parseSnapshotTimestamp(snapshotID string) (time.Time, error) { parts := strings.Split(snapshotID, "_") - if len(parts) < 2 { - return time.Time{}, errors.New("invalid snapshot ID format: expected hostname_snapshotname_timestamp") + if len(parts) < minSnapshotIDParts { + return time.Time{}, errMalformedSnapshotID } // Last part is the RFC3339 timestamp @@ -65,7 +108,7 @@ func parseSnapshotTimestamp(snapshotID string) (time.Time, error) { // Returns the snapshot name, or empty string if the ID is malformed. func parseSnapshotName(snapshotID string) string { parts := strings.Split(snapshotID, "_") - if len(parts) < 3 { + if len(parts) < minSnapshotIDNameParts { // Format: hostname_timestamp — no snapshot name return "" } @@ -88,7 +131,7 @@ func parseDuration(s string) (time.Duration, error) { matches := re.FindAllStringSubmatch(s, -1) if len(matches) == 0 { - return 0, fmt.Errorf("invalid duration: %q", s) + return 0, fmt.Errorf("%w: %q", errInvalidDuration, s) } var total time.Duration @@ -102,15 +145,15 @@ func parseDuration(s string) (time.Duration, error) { unit := strings.ToLower(match[2]) switch unit { case "d", "day", "days": - total += time.Duration(n) * 24 * time.Hour + total += time.Duration(n) * day case "w", "week", "weeks": - total += time.Duration(n) * 7 * 24 * time.Hour + total += time.Duration(n) * week case "mo", "month", "months": - total += time.Duration(n) * 30 * 24 * time.Hour + total += time.Duration(n) * month case "y", "year", "years": - total += time.Duration(n) * 365 * 24 * time.Hour + total += time.Duration(n) * year default: - return 0, fmt.Errorf("unknown time unit %q", unit) + return 0, fmt.Errorf("%w %q", errUnknownTimeUnit, unit) } } diff --git a/internal/vaultik/helpers_test.go b/internal/vaultik/helpers_test.go index e648bec..53e6746 100644 --- a/internal/vaultik/helpers_test.go +++ b/internal/vaultik/helpers_test.go @@ -1,4 +1,4 @@ -package vaultik +package vaultik //nolint:testpackage // exercises unexported parse helpers import ( "testing" @@ -6,6 +6,8 @@ import ( ) func TestParseSnapshotName(t *testing.T) { + t.Parallel() + tests := []struct { name string snapshotID string @@ -30,15 +32,20 @@ func TestParseSnapshotName(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + t.Parallel() + got := parseSnapshotName(tt.snapshotID) if got != tt.want { - t.Errorf("parseSnapshotName(%q) = %q, want %q", tt.snapshotID, got, tt.want) + t.Errorf("parseSnapshotName(%q) = %q, want %q", + tt.snapshotID, got, tt.want) } }) } } func TestParseDuration(t *testing.T) { + t.Parallel() + tests := []struct { input string want time.Duration @@ -56,6 +63,8 @@ func TestParseDuration(t *testing.T) { for _, tt := range tests { t.Run(tt.input, func(t *testing.T) { + t.Parallel() + got, err := parseDuration(tt.input) if tt.err { if err == nil { @@ -77,6 +86,8 @@ func TestParseDuration(t *testing.T) { } func TestParseSnapshotTimestamp(t *testing.T) { + t.Parallel() + tests := []struct { name string snapshotID string @@ -106,9 +117,12 @@ func TestParseSnapshotTimestamp(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + t.Parallel() + _, err := parseSnapshotTimestamp(tt.snapshotID) if (err != nil) != tt.wantErr { - t.Errorf("parseSnapshotTimestamp(%q) error = %v, wantErr %v", tt.snapshotID, err, tt.wantErr) + t.Errorf("parseSnapshotTimestamp(%q) error = %v, wantErr %v", + tt.snapshotID, err, tt.wantErr) } }) } diff --git a/internal/vaultik/info.go b/internal/vaultik/info.go index 2d103a4..59d41fc 100644 --- a/internal/vaultik/info.go +++ b/internal/vaultik/info.go @@ -15,129 +15,160 @@ import ( // ShowInfo displays system and configuration information func (v *Vaultik) ShowInfo() error { // System Information - v.printfStdout("=== System Information ===\n") - v.printfStdout("OS/Architecture: %s/%s\n", runtime.GOOS, runtime.GOARCH) - v.printfStdout("Version: %s\n", v.Globals.Version) - v.printfStdout("Commit: %s\n", v.Globals.Commit) - v.printfStdout("Go Version: %s\n", runtime.Version()) + v.stdoutf("=== System Information ===\n") + v.stdoutf("OS/Architecture: %s/%s\n", runtime.GOOS, runtime.GOARCH) + v.stdoutf("Version: %s\n", v.Globals.Version) + v.stdoutf("Commit: %s\n", v.Globals.Commit) + v.stdoutf("Go Version: %s\n", runtime.Version()) v.printlnStdout() - // Storage Configuration. The backend is selected by storage_url - // (s3://, file://, rclone://); the legacy s3.* fields are only - // printed when they're actually populated, since the URL scheme - // is the primary configuration. - v.printfStdout("=== Storage Configuration ===\n") + v.showStorageConfig() + v.showBackupSettings() + + // Encryption Configuration + v.stdoutf("=== Encryption Configuration ===\n") + v.stdoutf("Recipients:\n") + + for _, recipient := range v.Config.AgeRecipients { + v.stdoutf(" - %s\n", recipient) + } + + v.printlnStdout() + v.showLocalDatabase() + + return nil +} + +// showStorageConfig prints the storage configuration section. The +// backend is selected by storage_url (s3://, file://, rclone://); the +// legacy s3.* fields are only printed when they're actually populated, +// since the URL scheme is the primary configuration. +func (v *Vaultik) showStorageConfig() { + v.stdoutf("=== Storage Configuration ===\n") storageInfo := v.Storage.Info() - v.printfStdout("Type: %s\n", storageInfo.Type) - v.printfStdout("Location: %s\n", storageInfo.Location) + v.stdoutf("Type: %s\n", storageInfo.Type) + v.stdoutf("Location: %s\n", storageInfo.Location) if v.Config.StorageURL != "" { - v.printfStdout("Storage URL: %s\n", v.Config.StorageURL) + v.stdoutf("Storage URL: %s\n", v.Config.StorageURL) } if v.Config.S3.Bucket != "" { - v.printfStdout("S3 Bucket: %s\n", v.Config.S3.Bucket) + v.stdoutf("S3 Bucket: %s\n", v.Config.S3.Bucket) } if v.Config.S3.Prefix != "" { - v.printfStdout("S3 Prefix: %s\n", v.Config.S3.Prefix) + v.stdoutf("S3 Prefix: %s\n", v.Config.S3.Prefix) } if v.Config.S3.Endpoint != "" { - v.printfStdout("S3 Endpoint: %s\n", v.Config.S3.Endpoint) + v.stdoutf("S3 Endpoint: %s\n", v.Config.S3.Endpoint) } if v.Config.S3.Region != "" { - v.printfStdout("S3 Region: %s\n", v.Config.S3.Region) + v.stdoutf("S3 Region: %s\n", v.Config.S3.Region) } v.printlnStdout() +} - // Backup Settings - v.printfStdout("=== Backup Settings ===\n") +// showBackupSettings prints the configured snapshots, exclude patterns, +// and chunking/compression settings. +func (v *Vaultik) showBackupSettings() { + v.stdoutf("=== Backup Settings ===\n") // Show configured snapshots - v.printfStdout("Snapshots:\n") + v.stdoutf("Snapshots:\n") for _, name := range v.Config.SnapshotNames() { snap := v.Config.Snapshots[name] - v.printfStdout(" %s:\n", name) + v.stdoutf(" %s:\n", name) for _, path := range snap.Paths { - v.printfStdout(" - %s\n", path) + v.stdoutf(" - %s\n", path) } if len(snap.Exclude) > 0 { - v.printfStdout(" exclude: %s\n", strings.Join(snap.Exclude, ", ")) + v.stdoutf(" exclude: %s\n", strings.Join(snap.Exclude, ", ")) } } // Global exclude patterns if len(v.Config.Exclude) > 0 { - v.printfStdout("Global Exclude: %s\n", strings.Join(v.Config.Exclude, ", ")) + v.stdoutf("Global Exclude: %s\n", strings.Join(v.Config.Exclude, ", ")) } - v.printfStdout("Compression: zstd level %d\n", v.Config.CompressionLevel) - v.printfStdout("Chunk Size: %s\n", humanize.Bytes(uint64(v.Config.ChunkSize))) - v.printfStdout("Blob Size Limit: %s\n", humanize.Bytes(uint64(v.Config.BlobSizeLimit))) + v.stdoutf("Compression: zstd level %d\n", v.Config.CompressionLevel) + v.stdoutf("Chunk Size: %s\n", ubytes(int64(v.Config.ChunkSize))) + v.stdoutf("Blob Size Limit: %s\n", ubytes(int64(v.Config.BlobSizeLimit))) v.printlnStdout() +} - // Encryption Configuration - v.printfStdout("=== Encryption Configuration ===\n") - v.printfStdout("Recipients:\n") - - for _, recipient := range v.Config.AgeRecipients { - v.printfStdout(" - %s\n", recipient) - } - - v.printlnStdout() - - // Local Database - v.printfStdout("=== Local Database ===\n") - v.printfStdout("Index Path: %s\n", v.Config.IndexPath) +// showLocalDatabase prints the local index database section, including +// record counts when the index exists. +func (v *Vaultik) showLocalDatabase() { + v.stdoutf("=== Local Database ===\n") + v.stdoutf("Index Path: %s\n", v.Config.IndexPath) // Check if index file exists and get its size info, err := v.Fs.Stat(v.Config.IndexPath) - if err == nil { - v.printfStdout("Index Size: %s\n", humanize.Bytes(uint64(info.Size()))) + if err != nil { + v.stdoutf("Index Size: (not created)\n") - // Get snapshot count from database - query := `SELECT COUNT(*) FROM snapshots WHERE completed_at IS NOT NULL` - - var snapshotCount int - - err := v.DB.Conn().QueryRowContext(v.ctx, query).Scan(&snapshotCount) - if err == nil { - v.printfStdout("Snapshots: %d\n", snapshotCount) - } - - // Get blob count from database - query = `SELECT COUNT(*) FROM blobs` - - var blobCount int - - err = v.DB.Conn().QueryRowContext(v.ctx, query).Scan(&blobCount) - if err == nil { - v.printfStdout("Blobs: %d\n", blobCount) - } - - // Get file count from database - query = `SELECT COUNT(*) FROM files` - - var fileCount int - - err = v.DB.Conn().QueryRowContext(v.ctx, query).Scan(&fileCount) - if err == nil { - v.printfStdout("Files: %d\n", fileCount) - } - } else { - v.printfStdout("Index Size: (not created)\n") + return } - return nil + v.stdoutf("Index Size: %s\n", ubytes(info.Size())) + + // Get snapshot count from database + query := `SELECT COUNT(*) FROM snapshots WHERE completed_at IS NOT NULL` + + var snapshotCount int + + err = v.DB.Conn().QueryRowContext(v.ctx, query).Scan(&snapshotCount) + if err == nil { + v.stdoutf("Snapshots: %d\n", snapshotCount) + } + + // Get blob count from database + query = `SELECT COUNT(*) FROM blobs` + + var blobCount int + + err = v.DB.Conn().QueryRowContext(v.ctx, query).Scan(&blobCount) + if err == nil { + v.stdoutf("Blobs: %d\n", blobCount) + } + + // Get file count from database + query = `SELECT COUNT(*) FROM files` + + var fileCount int + + err = v.DB.Conn().QueryRowContext(v.ctx, query).Scan(&fileCount) + if err == nil { + v.stdoutf("Files: %d\n", fileCount) + } } +// Table layout constants for the human-readable remote info output. +const ( + // snapshotIDColWidth is the SNAPSHOT column width in the remote + // info table. + snapshotIDColWidth = 45 + + // metadataKeyParts is the minimum "/"-separated segment count of a + // metadata object key (metadata//). + metadataKeyParts = 3 + + // blobKeyParts is the minimum "/"-separated segment count of a blob + // object key (blobs///). + blobKeyParts = 4 +) + // SnapshotMetadataInfo contains information about a single snapshot's metadata +// +//nolint:tagliatelle // snake_case is the established JSON output format type SnapshotMetadataInfo struct { SnapshotID string `json:"snapshot_id"` ManifestSize int64 `json:"manifest_size"` @@ -148,6 +179,8 @@ type SnapshotMetadataInfo struct { } // RemoteInfoResult contains all remote storage information +// +//nolint:tagliatelle // snake_case is the established JSON output format type RemoteInfoResult struct { // Storage info StorageType string `json:"storage_type"` @@ -182,11 +215,11 @@ func (v *Vaultik) RemoteInfo(jsonOutput bool) error { result.StorageLocation = storageInfo.Location if !jsonOutput { - v.printfStdout("=== Remote Storage ===\n") - v.printfStdout("Type: %s\n", storageInfo.Type) - v.printfStdout("Location: %s\n", storageInfo.Location) + v.stdoutf("=== Remote Storage ===\n") + v.stdoutf("Type: %s\n", storageInfo.Type) + v.stdoutf("Location: %s\n", storageInfo.Location) v.printlnStdout() - v.printfStdout("Scanning snapshot metadata...\n") + v.stdoutf("Scanning snapshot metadata...\n") } snapshotMetadata, snapshotIDs, err := v.collectSnapshotMetadata() @@ -195,7 +228,7 @@ func (v *Vaultik) RemoteInfo(jsonOutput bool) error { } if !jsonOutput { - v.printfStdout("Downloading %d manifest(s)...\n", len(snapshotIDs)) + v.stdoutf("Downloading %d manifest(s)...\n", len(snapshotIDs)) } referencedBlobs := v.collectReferencedBlobsFromManifests(snapshotIDs, snapshotMetadata) @@ -225,8 +258,11 @@ func (v *Vaultik) RemoteInfo(jsonOutput bool) error { return nil } -// collectSnapshotMetadata scans remote metadata and returns per-snapshot info and sorted IDs -func (v *Vaultik) collectSnapshotMetadata() (map[string]*SnapshotMetadataInfo, []string, error) { +// collectSnapshotMetadata scans remote metadata and returns +// per-snapshot info and sorted IDs. +func (v *Vaultik) collectSnapshotMetadata() ( + map[string]*SnapshotMetadataInfo, []string, error, +) { snapshotMetadata := make(map[string]*SnapshotMetadataInfo) metadataCh := v.Storage.ListStream(v.ctx, "metadata/") @@ -236,7 +272,7 @@ func (v *Vaultik) collectSnapshotMetadata() (map[string]*SnapshotMetadataInfo, [ } parts := strings.Split(obj.Key, "/") - if len(parts) < 3 { + if len(parts) < metadataKeyParts { continue } @@ -268,8 +304,11 @@ func (v *Vaultik) collectSnapshotMetadata() (map[string]*SnapshotMetadataInfo, [ return snapshotMetadata, snapshotIDs, nil } -// collectReferencedBlobsFromManifests downloads manifests and returns referenced blob hashes with sizes -func (v *Vaultik) collectReferencedBlobsFromManifests(snapshotIDs []string, snapshotMetadata map[string]*SnapshotMetadataInfo) map[string]int64 { +// collectReferencedBlobsFromManifests downloads manifests and returns +// referenced blob hashes with sizes. +func (v *Vaultik) collectReferencedBlobsFromManifests( + snapshotIDs []string, snapshotMetadata map[string]*SnapshotMetadataInfo, +) map[string]int64 { referencedBlobs := make(map[string]int64) for _, snapshotID := range snapshotIDs { @@ -307,8 +346,14 @@ func (v *Vaultik) collectReferencedBlobsFromManifests(snapshotIDs []string, snap return referencedBlobs } -// populateRemoteInfoResult fills in the result's snapshot and referenced blob stats -func (v *Vaultik) populateRemoteInfoResult(result *RemoteInfoResult, snapshotMetadata map[string]*SnapshotMetadataInfo, snapshotIDs []string, referencedBlobs map[string]int64) { +// populateRemoteInfoResult fills in the result's snapshot and +// referenced blob stats. +func (v *Vaultik) populateRemoteInfoResult( + result *RemoteInfoResult, + snapshotMetadata map[string]*SnapshotMetadataInfo, + snapshotIDs []string, + referencedBlobs map[string]int64, +) { var totalMetadataSize int64 for _, id := range snapshotIDs { @@ -327,9 +372,11 @@ func (v *Vaultik) populateRemoteInfoResult(result *RemoteInfoResult, snapshotMet } // scanRemoteBlobStorage lists all blobs on remote and computes orphan stats -func (v *Vaultik) scanRemoteBlobStorage(result *RemoteInfoResult, referencedBlobs map[string]int64, jsonOutput bool) error { +func (v *Vaultik) scanRemoteBlobStorage( + result *RemoteInfoResult, referencedBlobs map[string]int64, jsonOutput bool, +) error { if !jsonOutput { - v.printfStdout("Scanning blobs...\n") + v.stdoutf("Scanning blobs...\n") } blobCh := v.Storage.ListStream(v.ctx, "blobs/") @@ -341,7 +388,7 @@ func (v *Vaultik) scanRemoteBlobStorage(result *RemoteInfoResult, referencedBlob } parts := strings.Split(obj.Key, "/") - if len(parts) < 4 { + if len(parts) < blobKeyParts { continue } @@ -363,51 +410,74 @@ func (v *Vaultik) scanRemoteBlobStorage(result *RemoteInfoResult, referencedBlob // printRemoteInfoTable renders the human-readable remote info output func (v *Vaultik) printRemoteInfoTable(result *RemoteInfoResult) { - v.printfStdout("\n=== Snapshot Metadata ===\n") + const ( + rowFormat = "%-45s %12s %12s %12s %10s %12s\n" + sizeColWidth = 12 + countColWidth = 10 + ) + + v.stdoutf("\n=== Snapshot Metadata ===\n") if len(result.Snapshots) == 0 { - v.printfStdout("No snapshots found\n") + v.stdoutf("No snapshots found\n") } else { - v.printfStdout("%-45s %12s %12s %12s %10s %12s\n", "SNAPSHOT", "MANIFEST", "DATABASE", "TOTAL", "BLOBS", "BLOB SIZE") - v.printfStdout("%-45s %12s %12s %12s %10s %12s\n", strings.Repeat("-", 45), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 10), strings.Repeat("-", 12)) + separator := fmt.Sprintf(rowFormat, + strings.Repeat("-", snapshotIDColWidth), + strings.Repeat("-", sizeColWidth), + strings.Repeat("-", sizeColWidth), + strings.Repeat("-", sizeColWidth), + strings.Repeat("-", countColWidth), + strings.Repeat("-", sizeColWidth)) + + v.stdoutf(rowFormat, + "SNAPSHOT", "MANIFEST", "DATABASE", "TOTAL", "BLOBS", "BLOB SIZE") + v.stdoutf("%s", separator) for _, info := range result.Snapshots { - v.printfStdout("%-45s %12s %12s %12s %10s %12s\n", - truncateString(info.SnapshotID, 45), - humanize.Bytes(uint64(info.ManifestSize)), - humanize.Bytes(uint64(info.DatabaseSize)), - humanize.Bytes(uint64(info.TotalSize)), + v.stdoutf(rowFormat, + truncateString(info.SnapshotID, snapshotIDColWidth), + ubytes(info.ManifestSize), + ubytes(info.DatabaseSize), + ubytes(info.TotalSize), humanize.Comma(int64(info.BlobCount)), - humanize.Bytes(uint64(info.BlobsSize)), + ubytes(info.BlobsSize), ) } - v.printfStdout("%-45s %12s %12s %12s %10s %12s\n", strings.Repeat("-", 45), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 10), strings.Repeat("-", 12)) - v.printfStdout("%-45s %12s %12s %12s\n", fmt.Sprintf("Total (%d snapshots)", result.TotalMetadataCount), "", "", humanize.Bytes(uint64(result.TotalMetadataSize))) + v.stdoutf("%s", separator) + v.stdoutf("%-45s %12s %12s %12s\n", + fmt.Sprintf("Total (%d snapshots)", result.TotalMetadataCount), + "", "", ubytes(result.TotalMetadataSize)) } - v.printfStdout("\n=== Blob Storage ===\n") - v.printfStdout("Total blobs on remote: %s (%s)\n", - humanize.Comma(int64(result.TotalBlobCount)), humanize.Bytes(uint64(result.TotalBlobSize))) - v.printfStdout("Referenced by snapshots: %s (%s)\n", - humanize.Comma(int64(result.ReferencedBlobCount)), humanize.Bytes(uint64(result.ReferencedBlobSize))) - v.printfStdout("Orphaned (unreferenced): %s (%s)\n", - humanize.Comma(int64(result.OrphanedBlobCount)), humanize.Bytes(uint64(result.OrphanedBlobSize))) + v.stdoutf("\n=== Blob Storage ===\n") + v.stdoutf("Total blobs on remote: %s (%s)\n", + humanize.Comma(int64(result.TotalBlobCount)), + ubytes(result.TotalBlobSize)) + v.stdoutf("Referenced by snapshots: %s (%s)\n", + humanize.Comma(int64(result.ReferencedBlobCount)), + ubytes(result.ReferencedBlobSize)) + v.stdoutf("Orphaned (unreferenced): %s (%s)\n", + humanize.Comma(int64(result.OrphanedBlobCount)), + ubytes(result.OrphanedBlobSize)) if result.OrphanedBlobCount > 0 { - v.printfStdout("\nRun 'vaultik prune' to remove orphaned blobs.\n") + v.stdoutf("\nRun 'vaultik prune' to remove orphaned blobs.\n") } } +// ellipsis is appended by truncateString when it shortens a string. +const ellipsis = "..." + // truncateString truncates a string to maxLen, adding "..." if truncated func truncateString(s string, maxLen int) string { if len(s) <= maxLen { return s } - if maxLen <= 3 { + if maxLen <= len(ellipsis) { return s[:maxLen] } - return s[:maxLen-3] + "..." + return s[:maxLen-len(ellipsis)] + ellipsis } diff --git a/internal/vaultik/integration_test.go b/internal/vaultik/integration_test.go index 78a1c18..0e51afd 100644 --- a/internal/vaultik/integration_test.go +++ b/internal/vaultik/integration_test.go @@ -24,6 +24,16 @@ import ( "sneak.berlin/go/vaultik/internal/vaultik" ) +// Shared fixture values used across the vaultik integration tests. +const ( + testLabel = "test" + testHostname = "test-host" + testAgePublicKey = "age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05g" + + "l0sjq9q9wjg" + testAgeSecretKey = "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKU" + + "T68TXSFPK7APHXA2QS2NJA5" +) + // MockStorer implements storage.Storer for testing type MockStorer struct { mu sync.Mutex @@ -38,7 +48,7 @@ func NewMockStorer() *MockStorer { } } -func (m *MockStorer) Put(ctx context.Context, key string, reader io.Reader) error { +func (m *MockStorer) Put(_ context.Context, key string, reader io.Reader) error { m.mu.Lock() defer m.mu.Unlock() @@ -54,11 +64,14 @@ func (m *MockStorer) Put(ctx context.Context, key string, reader io.Reader) erro return nil } -func (m *MockStorer) PutWithProgress(ctx context.Context, key string, reader io.Reader, size int64, progress storage.ProgressCallback) error { +func (m *MockStorer) PutWithProgress( + ctx context.Context, key string, reader io.Reader, + _ int64, _ storage.ProgressCallback, +) error { return m.Put(ctx, key, reader) } -func (m *MockStorer) Get(ctx context.Context, key string) (io.ReadCloser, error) { +func (m *MockStorer) Get(_ context.Context, key string) (io.ReadCloser, error) { m.mu.Lock() defer m.mu.Unlock() @@ -72,7 +85,7 @@ func (m *MockStorer) Get(ctx context.Context, key string) (io.ReadCloser, error) return io.NopCloser(bytes.NewReader(data)), nil } -func (m *MockStorer) Stat(ctx context.Context, key string) (*storage.ObjectInfo, error) { +func (m *MockStorer) Stat(_ context.Context, key string) (*storage.ObjectInfo, error) { m.mu.Lock() defer m.mu.Unlock() @@ -89,7 +102,7 @@ func (m *MockStorer) Stat(ctx context.Context, key string) (*storage.ObjectInfo, }, nil } -func (m *MockStorer) Delete(ctx context.Context, key string) error { +func (m *MockStorer) Delete(_ context.Context, key string) error { m.mu.Lock() defer m.mu.Unlock() @@ -99,7 +112,7 @@ func (m *MockStorer) Delete(ctx context.Context, key string) error { return nil } -func (m *MockStorer) List(ctx context.Context, prefix string) ([]string, error) { +func (m *MockStorer) List(_ context.Context, prefix string) ([]string, error) { m.mu.Lock() defer m.mu.Unlock() @@ -108,7 +121,8 @@ func (m *MockStorer) List(ctx context.Context, prefix string) ([]string, error) var keys []string for key := range m.data { - if len(prefix) == 0 || (len(key) >= len(prefix) && key[:len(prefix)] == prefix) { + if len(prefix) == 0 || + (len(key) >= len(prefix) && key[:len(prefix)] == prefix) { keys = append(keys, key) } } @@ -116,7 +130,9 @@ func (m *MockStorer) List(ctx context.Context, prefix string) ([]string, error) return keys, nil } -func (m *MockStorer) ListStream(ctx context.Context, prefix string) <-chan storage.ObjectInfo { +func (m *MockStorer) ListStream( + _ context.Context, prefix string, +) <-chan storage.ObjectInfo { ch := make(chan storage.ObjectInfo) go func() { defer close(ch) @@ -125,7 +141,8 @@ func (m *MockStorer) ListStream(ctx context.Context, prefix string) <-chan stora defer m.mu.Unlock() for key, data := range m.data { - if len(prefix) == 0 || (len(key) >= len(prefix) && key[:len(prefix)] == prefix) { + if len(prefix) == 0 || + (len(key) >= len(prefix) && key[:len(prefix)] == prefix) { ch <- storage.ObjectInfo{ Key: key, Size: int64(len(data)), @@ -137,8 +154,8 @@ func (m *MockStorer) ListStream(ctx context.Context, prefix string) <-chan stora return ch } -func (m *MockStorer) Info() storage.StorageInfo { - return storage.StorageInfo{ +func (m *MockStorer) Info() storage.Info { + return storage.Info{ Type: "mock", Location: "memory", } @@ -163,123 +180,55 @@ func (m *MockStorer) GetStorageSize() int { return len(m.data) } -// TestEndToEndBackup tests the full backup workflow with mocked dependencies -func TestEndToEndBackup(t *testing.T) { - // Initialize logger - log.Initialize(log.Config{}) +// writeTestFileTree creates each file (and its parent directory) in fs. +func writeTestFileTree(t *testing.T, fs afero.Fs, files map[string]string) { + t.Helper() - // Create in-memory filesystem - fs := afero.NewMemMapFs() + for path, content := range files { + dir := filepath.Dir(path) - // Create test directory structure and files - testFiles := map[string]string{ - "/home/user/documents/file1.txt": "This is file 1 content", - "/home/user/documents/file2.txt": "This is file 2 content with more data", - "/home/user/pictures/photo1.jpg": "Binary photo data here...", - "/home/user/code/main.go": "package main\n\nfunc main() {\n\tprintln(\"Hello, World!\")\n}", - } - - // Create all directories first - dirs := []string{ - "/home/user/documents", - "/home/user/pictures", - "/home/user/code", - } - for _, dir := range dirs { err := fs.MkdirAll(dir, 0755) if err != nil { t.Fatalf("failed to create directory %s: %v", dir, err) } - } - // Create test files - for path, content := range testFiles { - err := afero.WriteFile(fs, path, []byte(content), 0644) + err = afero.WriteFile(fs, path, []byte(content), 0644) if err != nil { t.Fatalf("failed to create test file %s: %v", path, err) } } +} - // Create mock storage - mockStorage := NewMockStorer() +// createTestSnapshotRecord inserts a snapshot row so scans have a +// snapshot to attach to. +func createTestSnapshotRecord( + t *testing.T, repos *database.Repositories, snapshotID string, +) { + t.Helper() - // Create test configuration - cfg := &config.Config{ - Snapshots: map[string]config.SnapshotConfig{ - "test": { - Paths: []string{"/home/user"}, - }, - }, - Exclude: []string{"*.tmp", "*.log"}, - ChunkSize: config.Size(16 * 1024), // 16KB chunks - BlobSizeLimit: config.Size(100 * 1024), // 100KB blobs - CompressionLevel: 3, - AgeRecipients: []string{"age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05gl0sjq9q9wjg"}, // Test public key - AgeSecretKey: "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5", // Test private key - S3: config.S3Config{ - Endpoint: "http://localhost:9000", // MinIO endpoint for testing - Region: "us-east-1", - Bucket: "test-bucket", - AccessKeyID: "test-access", - SecretAccessKey: "test-secret", - }, - IndexPath: ":memory:", // In-memory SQLite database - } - - // For a true end-to-end test, we'll create a simpler test that focuses on - // the core backup logic using the scanner directly with our mock storage ctx := context.Background() - // Create in-memory database - db, err := database.New(ctx, ":memory:") - - require.NoError(t, err) - defer func() { - err := db.Close() - if err != nil { - t.Errorf("failed to close database: %v", err) - } - }() - - repos := database.NewRepositories(db) - - // Create scanner with mock storage - scanner := snapshot.NewScanner(snapshot.ScannerConfig{ - FS: fs, - ChunkSize: cfg.ChunkSize.Int64(), - Repositories: repos, - Storage: mockStorage, - MaxBlobSize: cfg.BlobSizeLimit.Int64(), - CompressionLevel: cfg.CompressionLevel, - AgeRecipients: cfg.AgeRecipients, - EnableProgress: false, - }) - - // Create a snapshot record - snapshotID := "test-snapshot-001" - err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { - snapshot := &database.Snapshot{ + err := repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { + snap := &database.Snapshot{ ID: types.SnapshotID(snapshotID), - Hostname: "test-host", + Hostname: testHostname, VaultikVersion: "test-version", StartedAt: time.Now(), } - return repos.Snapshots.Create(ctx, tx, snapshot) + return repos.Snapshots.Create(ctx, tx, snap) }) require.NoError(t, err) +} - // Run the backup scan - result, err := scanner.Scan(ctx, "/home/user", snapshotID) - require.NoError(t, err) - - // Verify scan results - // The scanner counts both files and directories, so we have: - // 4 files + 4 directories (/home, /home/user, /home/user/documents, /home/user/pictures, /home/user/code) - assert.GreaterOrEqual(t, result.FilesScanned, 4, "Should scan at least 4 files") - assert.Positive(t, result.BytesScanned, "Should scan some bytes") - assert.Positive(t, result.ChunksCreated, "Should create chunks") - assert.Positive(t, result.BlobsCreated, "Should create blobs") +// TestEndToEndBackup tests the full backup workflow with mocked dependencies +// verifyEndToEndBackupState checks storage upload calls, database file +// rows, and chunk mappings after the end-to-end backup scan. +func verifyEndToEndBackupState( + ctx context.Context, t *testing.T, + repos *database.Repositories, mockStorage *MockStorer, +) { + t.Helper() // Verify storage operations calls := mockStorage.GetCalls() @@ -313,9 +262,108 @@ func TestEndToEndBackup(t *testing.T) { assert.Equal(t, 4, regularFiles, "Should have 4 regular files in database") // Verify chunks were created by checking a specific file - fileChunks, err := repos.FileChunks.GetByPath(ctx, "/home/user/documents/file1.txt") + fileChunks, err := repos.FileChunks.GetByPath(ctx, + "/home/user/documents/file1.txt") require.NoError(t, err) assert.NotEmpty(t, fileChunks, "Should have chunks for file1.txt") +} + +// newEndToEndTestConfig builds the standard config used by the +// end-to-end backup test. +func newEndToEndTestConfig() *config.Config { + return &config.Config{ + Snapshots: map[string]config.SnapshotConfig{ + testLabel: { + Paths: []string{"/home/user"}, + }, + }, + Exclude: []string{"*.tmp", "*.log"}, + ChunkSize: config.Size(16 * 1024), // 16KB chunks + BlobSizeLimit: config.Size(100 * 1024), // 100KB blobs + CompressionLevel: 3, + AgeRecipients: []string{testAgePublicKey}, + AgeSecretKey: testAgeSecretKey, + S3: config.S3Config{ + Endpoint: "http://localhost:9000", // MinIO endpoint for testing + Region: "us-east-1", + Bucket: "test-bucket", + AccessKeyID: "test-access", + SecretAccessKey: "test-secret", + }, + IndexPath: ":memory:", // In-memory SQLite database + } +} + +func TestEndToEndBackup(t *testing.T) { + // Initialize logger + log.Initialize(log.Config{}) + t.Parallel() + + // Create in-memory filesystem + fs := afero.NewMemMapFs() + + // Create test directory structure and files + testFiles := map[string]string{ + "/home/user/documents/file1.txt": "This is file 1 content", + "/home/user/documents/file2.txt": "This is file 2 content with more data", + "/home/user/pictures/photo1.jpg": "Binary photo data here...", + "/home/user/code/main.go": "package main\n\nfunc main() {\n" + + "\tprintln(\"Hello, World!\")\n}", + } + + writeTestFileTree(t, fs, testFiles) + + // Create mock storage + mockStorage := NewMockStorer() + + cfg := newEndToEndTestConfig() + + // For a true end-to-end test, we'll create a simpler test that focuses on + // the core backup logic using the scanner directly with our mock storage + ctx := context.Background() + + // Create in-memory database + db, err := database.New(ctx, ":memory:") + + require.NoError(t, err) + defer func() { + err := db.Close() + if err != nil { + t.Errorf("failed to close database: %v", err) + } + }() + + repos := database.NewRepositories(db) + + // Create scanner with mock storage + scanner := snapshot.NewScanner(snapshot.ScannerConfig{ + FS: fs, + ChunkSize: cfg.ChunkSize.Int64(), + Repositories: repos, + Storage: mockStorage, + MaxBlobSize: cfg.BlobSizeLimit.Int64(), + CompressionLevel: cfg.CompressionLevel, + AgeRecipients: cfg.AgeRecipients, + EnableProgress: false, + }) + + // Create a snapshot record + snapshotID := "test-snapshot-001" + createTestSnapshotRecord(t, repos, snapshotID) + + // Run the backup scan + result, err := scanner.Scan(ctx, "/home/user", snapshotID) + require.NoError(t, err) + + // Verify scan results. The scanner counts both files and + // directories: 4 files + directories (/home, /home/user, + // /home/user/documents, /home/user/pictures, /home/user/code). + assert.GreaterOrEqual(t, result.FilesScanned, 4, "Should scan at least 4 files") + assert.Positive(t, result.BytesScanned, "Should scan some bytes") + assert.Positive(t, result.ChunksCreated, "Should create chunks") + assert.Positive(t, result.BlobsCreated, "Should create blobs") + + verifyEndToEndBackupState(ctx, t, repos, mockStorage) // Verify blobs were uploaded to storage assert.Positive(t, mockStorage.GetStorageSize(), "Should have blobs in storage") @@ -343,6 +391,7 @@ func TestEndToEndBackup(t *testing.T) { func TestBackupAndVerify(t *testing.T) { // Initialize logger log.Initialize(log.Config{}) + t.Parallel() // Create in-memory filesystem fs := afero.NewMemMapFs() @@ -379,22 +428,12 @@ func TestBackupAndVerify(t *testing.T) { Storage: mockStorage, MaxBlobSize: int64(1024 * 1024), // 1MB blobs CompressionLevel: 3, - AgeRecipients: []string{"age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05gl0sjq9q9wjg"}, // Test public key + AgeRecipients: []string{testAgePublicKey}, }) // Create a snapshot snapshotID := "test-snapshot-001" - err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { - snapshot := &database.Snapshot{ - ID: types.SnapshotID(snapshotID), - Hostname: "test-host", - VaultikVersion: "test-version", - StartedAt: time.Now(), - } - - return repos.Snapshots.Create(ctx, tx, snapshot) - }) - require.NoError(t, err) + createTestSnapshotRecord(t, repos, snapshotID) // Run the backup result, err := scanner.Scan(ctx, "/data", snapshotID) @@ -402,12 +441,14 @@ func TestBackupAndVerify(t *testing.T) { // Verify backup created blobs assert.Positive(t, result.BlobsCreated, "Should create at least one blob") - assert.Equal(t, mockStorage.GetStorageSize(), result.BlobsCreated, "Storage should have the blobs") + assert.Equal(t, mockStorage.GetStorageSize(), result.BlobsCreated, + "Storage should have the blobs") // Verify we can retrieve the blob from storage objects, err := mockStorage.List(ctx, "blobs/") require.NoError(t, err) - assert.Len(t, objects, result.BlobsCreated, "Should have correct number of blobs in storage") + assert.Len(t, objects, result.BlobsCreated, + "Should have correct number of blobs in storage") // Get the first blob and verify it exists if len(objects) > 0 { @@ -438,66 +479,15 @@ func TestBackupAndVerify(t *testing.T) { // TestBackupAndRestore tests the full backup and restore workflow // This test verifies that the restore code correctly handles the binary SQLite // database format that is exported by the snapshot manager. -func TestBackupAndRestore(t *testing.T) { - // Initialize logger - log.Initialize(log.Config{}) - - // Create real temp directory for the database (SQLite needs real filesystem) - realTempDir, err := os.MkdirTemp("", "vaultik-test-") - require.NoError(t, err) - - defer func() { _ = os.RemoveAll(realTempDir) }() - - // Use real OS filesystem for this test - fs := afero.NewOsFs() - - // Create test directory structure and files - dataDir := filepath.Join(realTempDir, "data") - testFiles := map[string]string{ - filepath.Join(dataDir, "file1.txt"): "This is file 1 content", - filepath.Join(dataDir, "file2.txt"): "This is file 2 content with more data", - filepath.Join(dataDir, "subdir", "file3.txt"): "This is file 3 in a subdirectory", - } - - // Create directories and files - for path, content := range testFiles { - dir := filepath.Dir(path) - - err := fs.MkdirAll(dir, 0755) - if err != nil { - t.Fatalf("failed to create directory %s: %v", dir, err) - } - - err = afero.WriteFile(fs, path, []byte(content), 0644) - if err != nil { - t.Fatalf("failed to create test file %s: %v", path, err) - } - } - - ctx := context.Background() - - // Create mock storage - mockStorage := NewMockStorer() - - // Test keypair - agePublicKey := "age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05gl0sjq9q9wjg" - ageSecretKey := "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5" - - // Create database file - dbPath := filepath.Join(realTempDir, "test.db") - db, err := database.New(ctx, dbPath) - require.NoError(t, err) - - defer func() { _ = db.Close() }() - - repos := database.NewRepositories(db) - - // Create config for snapshot manager - cfg := &config.Config{ - AgeSecretKey: ageSecretKey, - AgeRecipients: []string{agePublicKey}, - CompressionLevel: 3, - } +// runBackupPhase performs the backup half of the round-trip test: +// create the snapshot, scan the data directory, complete the snapshot, +// and export its metadata, verifying the metadata upload. +func runBackupPhase( + ctx context.Context, t *testing.T, fs afero.Fs, + repos *database.Repositories, mockStorage *MockStorer, + cfg *config.Config, dataDir, dbPath, agePublicKey string, +) string { + t.Helper() // Create snapshot manager sm := snapshot.NewSnapshotManager(snapshot.SnapshotManagerParams{ @@ -519,14 +509,16 @@ func TestBackupAndRestore(t *testing.T) { }) // Create a snapshot - snapshotID, err := sm.CreateSnapshot(ctx, "test-host", "test-version", "test-git") + snapshotID, err := sm.CreateSnapshotWithName( + ctx, testHostname, "", "test-version", "test-git") require.NoError(t, err) t.Logf("Created snapshot: %s", snapshotID) // Run the backup (scan) result, err := scanner.Scan(ctx, dataDir, snapshotID) require.NoError(t, err) - t.Logf("Scan complete: %d files, %d blobs", result.FilesScanned, result.BlobsCreated) + t.Logf("Scan complete: %d files, %d blobs", + result.FilesScanned, result.BlobsCreated) // Complete the snapshot err = sm.CompleteSnapshot(ctx, snapshotID) @@ -541,7 +533,63 @@ func TestBackupAndRestore(t *testing.T) { keys, err := mockStorage.List(ctx, "metadata/") require.NoError(t, err) t.Logf("Metadata keys: %v", keys) - assert.GreaterOrEqual(t, len(keys), 2, "Should have at least db.zst.age and manifest.json.zst") + assert.GreaterOrEqual(t, len(keys), 2, + "Should have at least db.zst.age and manifest.json.zst") + + return snapshotID +} + +func TestBackupAndRestore(t *testing.T) { + // Initialize logger + log.Initialize(log.Config{}) + t.Parallel() + + // Create real temp directory for the database (SQLite needs real filesystem) + realTempDir := t.TempDir() + + // Use real OS filesystem for this test + fs := afero.NewOsFs() + + // Create test directory structure and files + dataDir := filepath.Join(realTempDir, "data") + testFiles := map[string]string{ + filepath.Join(dataDir, "file1.txt"): "This is file 1 content", + filepath.Join(dataDir, "file2.txt"): "This is file 2 content " + + "with more data", + filepath.Join(dataDir, "subdir", "file3.txt"): "This is file 3 " + + "in a subdirectory", + } + + // Create directories and files + writeTestFileTree(t, fs, testFiles) + + ctx := context.Background() + + // Create mock storage + mockStorage := NewMockStorer() + + // Test keypair + agePublicKey := testAgePublicKey + ageSecretKey := testAgeSecretKey + + // Create database file + dbPath := filepath.Join(realTempDir, "test.db") + db, err := database.New(ctx, dbPath) + require.NoError(t, err) + + defer func() { _ = db.Close() }() + + repos := database.NewRepositories(db) + + // Create config for snapshot manager + cfg := &config.Config{ + AgeSecretKey: ageSecretKey, + AgeRecipients: []string{agePublicKey}, + CompressionLevel: 3, + } + + snapshotID := runBackupPhase( + ctx, t, fs, repos, mockStorage, cfg, dataDir, dbPath, agePublicKey) // Close the source database err = db.Close() @@ -570,49 +618,32 @@ func TestBackupAndRestore(t *testing.T) { for origPath, expectedContent := range testFiles { restoredPath := filepath.Join(restoreDir, origPath) restoredContent, err := afero.ReadFile(fs, restoredPath) - require.NoError(t, err, "Should be able to read restored file: %s", restoredPath) - assert.Equal(t, expectedContent, string(restoredContent), "Restored content should match original for: %s", origPath) + require.NoError(t, err, + "Should be able to read restored file: %s", restoredPath) + assert.Equal(t, expectedContent, string(restoredContent), + "Restored content should match original for: %s", origPath) } t.Log("Backup and restore test completed successfully") } -// TestEndToEndFileStorage exercises the full backup → restore loop against the -// real `file://` storage backend (FileStorer) on a real OS filesystem. This is -// the closest local approximation of a production backup: encrypted blobs get -// written to disk, the metadata SQLite database is exported through the same -// blobgen pipeline as a real backup, and restoration reads them back through -// the public Vaultik.Restore entrypoint. It is the canonical end-to-end smoke -// test for 1.0. -func TestEndToEndFileStorage(t *testing.T) { - log.Initialize(log.Config{}) - - // Real OS filesystem (SQLite + FileStorer both need it). - fs := afero.NewOsFs() - tempDir, err := os.MkdirTemp("", "vaultik-e2e-") - require.NoError(t, err) - - defer func() { _ = os.RemoveAll(tempDir) }() - - dataDir := filepath.Join(tempDir, "source") - storeDir := filepath.Join(tempDir, "remote") - restoreDir := filepath.Join(tempDir, "restored") - dbPath := filepath.Join(tempDir, "index.sqlite") - - // Write a representative mix of file sizes: - // - empty file - // - tiny text file - // - file just under chunk boundary - // - file forcing multiple chunks - // - nested subdirectories - chunkSize := int64(64 * 1024) - maxBlobSize := int64(512 * 1024) +// setupE2ESourceTree writes a representative mix of file sizes (empty +// file, tiny text file, file under the chunk boundary, multi-chunk +// file, nested subdirectories), a permission-restricted file, an empty +// directory, and a symlink. It returns the content map keyed by path +// plus the restricted-file, empty-dir, and symlink paths. +func setupE2ESourceTree( + t *testing.T, fs afero.Fs, dataDir string, chunkSize int64, +) (map[string][]byte, string, string, string) { + t.Helper() testFiles := map[string][]byte{ - filepath.Join(dataDir, "empty.txt"): {}, - filepath.Join(dataDir, "small.txt"): []byte("hello vaultik"), - filepath.Join(dataDir, "subdir", "medium.bin"): bytesPattern("medium-", int(chunkSize/2)), - filepath.Join(dataDir, "subdir", "large.bin"): bytesPattern("large-", int(chunkSize*4)), + filepath.Join(dataDir, "empty.txt"): {}, + filepath.Join(dataDir, "small.txt"): []byte("hello vaultik"), + filepath.Join(dataDir, "subdir", "medium.bin"): bytesPattern( + "medium-", int(chunkSize/2)), + filepath.Join(dataDir, "subdir", "large.bin"): bytesPattern( + "large-", int(chunkSize*4)), filepath.Join(dataDir, "deep", "nest", "leaf.txt"): []byte("leaf"), } @@ -624,6 +655,7 @@ func TestEndToEndFileStorage(t *testing.T) { // Create a file with non-default permissions. restrictedPath := filepath.Join(dataDir, "restricted.txt") require.NoError(t, afero.WriteFile(fs, restrictedPath, []byte("secret"), 0o600)) + testFiles[restrictedPath] = []byte("secret") // Create an empty directory (should survive round-trip). @@ -634,22 +666,38 @@ func TestEndToEndFileStorage(t *testing.T) { symlinkPath := filepath.Join(dataDir, "link-to-small") require.NoError(t, os.Symlink("small.txt", symlinkPath)) + return testFiles, restrictedPath, emptyDir, symlinkPath +} + +// TestEndToEndFileStorage exercises the full backup → restore loop against the +// real `file://` storage backend (FileStorer) on a real OS filesystem. This is +// the closest local approximation of a production backup: encrypted blobs get +// written to disk, the metadata SQLite database is exported through the same +// blobgen pipeline as a real backup, and restoration reads them back through +// the public Vaultik.Restore entrypoint. It is the canonical end-to-end smoke +// test for 1.0. +// runFileStorageBackup performs the backup half of the file-storage +// end-to-end test against a real on-disk FileStorer, verifies the +// on-disk layout, and closes the index database so the restore half +// runs from remote bytes only. +func runFileStorageBackup( + ctx context.Context, t *testing.T, fs afero.Fs, + dataDir, storeDir, dbPath string, + chunkSize, maxBlobSize int64, +) (*config.Config, *storage.FileStorer, string) { + t.Helper() + // FileStorer is the real-world local-disk backend. storer, err := storage.NewFileStorer(storeDir) require.NoError(t, err) - agePublicKey := "age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05gl0sjq9q9wjg" - ageSecretKey := "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5" - cfg := &config.Config{ - AgeRecipients: []string{agePublicKey}, - AgeSecretKey: ageSecretKey, + AgeRecipients: []string{testAgePublicKey}, + AgeSecretKey: testAgeSecretKey, CompressionLevel: 3, - Hostname: "test-host", + Hostname: testHostname, } - ctx := context.Background() - db, err := database.New(ctx, dbPath) require.NoError(t, err) @@ -674,7 +722,8 @@ func TestEndToEndFileStorage(t *testing.T) { Repositories: repos, }) - snapshotID, err := sm.CreateSnapshotWithName(ctx, cfg.Hostname, "e2e", "test-version", "test-git") + snapshotID, err := sm.CreateSnapshotWithName( + ctx, cfg.Hostname, "e2e", "test-version", "test-git") require.NoError(t, err) scanResult, err := scanner.Scan(ctx, dataDir, snapshotID) @@ -692,7 +741,8 @@ func TestEndToEndFileStorage(t *testing.T) { require.NoError(t, err) require.True(t, blobInfo.IsDir()) - metaInfo, err := os.Stat(filepath.Join(storeDir, "metadata", snapshot.RemoteSnapshotKey(snapshotID))) + metaInfo, err := os.Stat(filepath.Join( + storeDir, "metadata", snapshot.RemoteSnapshotKey(snapshotID))) require.NoError(t, err) require.True(t, metaInfo.IsDir()) @@ -700,6 +750,39 @@ func TestEndToEndFileStorage(t *testing.T) { // the remote bytes plus the secret key, with no help from the local index. require.NoError(t, db.Close()) + return cfg, storer, snapshotID +} + +func TestEndToEndFileStorage(t *testing.T) { + log.Initialize(log.Config{}) + t.Parallel() + + // Real OS filesystem (SQLite + FileStorer both need it). + fs := afero.NewOsFs() + tempDir := t.TempDir() + + dataDir := filepath.Join(tempDir, "source") + storeDir := filepath.Join(tempDir, "remote") + restoreDir := filepath.Join(tempDir, "restored") + dbPath := filepath.Join(tempDir, "index.sqlite") + + // Write a representative mix of file sizes: + // - empty file + // - tiny text file + // - file just under chunk boundary + // - file forcing multiple chunks + // - nested subdirectories + chunkSize := int64(64 * 1024) + maxBlobSize := int64(512 * 1024) + + testFiles, restrictedPath, emptyDir, symlinkPath := + setupE2ESourceTree(t, fs, dataDir, chunkSize) + + ctx := context.Background() + + cfg, storer, snapshotID := runFileStorageBackup( + ctx, t, fs, dataDir, storeDir, dbPath, chunkSize, maxBlobSize) + restoreVaultik := &vaultik.Vaultik{ Config: cfg, Storage: storer, @@ -716,6 +799,19 @@ func TestEndToEndFileStorage(t *testing.T) { Verify: true, })) + verifyE2ERestoredTree(t, fs, restoreDir, testFiles, + restrictedPath, emptyDir, symlinkPath) +} + +// verifyE2ERestoredTree byte-compares every restored file and checks the +// restricted-permission file, empty directory, and symlink special cases. +func verifyE2ERestoredTree( + t *testing.T, fs afero.Fs, restoreDir string, + testFiles map[string][]byte, + restrictedPath, emptyDir, symlinkPath string, +) { + t.Helper() + // Byte-equality compare every original against its restored copy. for origPath, expected := range testFiles { restoredPath := filepath.Join(restoreDir, origPath) @@ -751,22 +847,22 @@ func TestEndToEndFileStorage(t *testing.T) { // regression where snapshot_blobs was populated only for blobs uploaded // during the snapshot, leaving fully-deduplicated snapshots unrestorable // with "chunk X not found in any blob" errors. -func TestDedupOnlySnapshotRestores(t *testing.T) { - log.Initialize(log.Config{}) +// dedupBackupEnv bundles the moving parts of the dedup round-trip test. +type dedupBackupEnv struct { + cfg *config.Config + storer *storage.FileStorer + db *database.DB + repos *database.Repositories + sm *snapshot.SnapshotManager + makeScanner func() *snapshot.Scanner +} - fs := afero.NewOsFs() - tempDir, err := os.MkdirTemp("", "vaultik-dedup-") - require.NoError(t, err) - - defer func() { _ = os.RemoveAll(tempDir) }() - - dataDir := filepath.Join(tempDir, "source") - storeDir := filepath.Join(tempDir, "remote") - restoreDir := filepath.Join(tempDir, "restored") - dbPath := filepath.Join(tempDir, "index.sqlite") - - chunkSize := int64(64 * 1024) - maxBlobSize := int64(512 * 1024) +// writeDedupSourceFiles writes the two deterministic source files used +// by the dedup test and returns their expected contents by path. +func writeDedupSourceFiles( + t *testing.T, fs afero.Fs, dataDir string, chunkSize int64, +) map[string][]byte { + t.Helper() testFiles := map[string][]byte{ filepath.Join(dataDir, "a.bin"): bytesPattern("a-", int(chunkSize*3)), @@ -777,25 +873,30 @@ func TestDedupOnlySnapshotRestores(t *testing.T) { require.NoError(t, afero.WriteFile(fs, path, content, 0o644)) } + return testFiles +} + +// setupDedupBackupEnv creates the storer, config, database, snapshot +// manager, and scanner factory for the dedup round-trip test. +func setupDedupBackupEnv( + ctx context.Context, t *testing.T, fs afero.Fs, + storeDir, dbPath string, chunkSize, maxBlobSize int64, +) *dedupBackupEnv { + t.Helper() + storer, err := storage.NewFileStorer(storeDir) require.NoError(t, err) - agePublicKey := "age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05gl0sjq9q9wjg" - ageSecretKey := "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5" - cfg := &config.Config{ - AgeRecipients: []string{agePublicKey}, - AgeSecretKey: ageSecretKey, + AgeRecipients: []string{testAgePublicKey}, + AgeSecretKey: testAgeSecretKey, CompressionLevel: 3, - Hostname: "test-host", + Hostname: testHostname, } - ctx := context.Background() db, err := database.New(ctx, dbPath) require.NoError(t, err) - defer func() { _ = db.Close() }() - repos := database.NewRepositories(db) makeScanner := func() *snapshot.Scanner { @@ -814,31 +915,85 @@ func TestDedupOnlySnapshotRestores(t *testing.T) { }) sm.SetFilesystem(fs) + return &dedupBackupEnv{ + cfg: cfg, + storer: storer, + db: db, + repos: repos, + sm: sm, + makeScanner: makeScanner, + } +} + +// runDedupSnapshot creates a "dedup" snapshot, scans dataDir into it, +// completes it, and exports its metadata, returning the snapshot ID and +// scan result. +func runDedupSnapshot( + ctx context.Context, t *testing.T, + sm *snapshot.SnapshotManager, scanner *snapshot.Scanner, + hostname, dataDir, dbPath string, +) (string, *snapshot.ScanResult) { + t.Helper() + + id, err := sm.CreateSnapshotWithName(ctx, hostname, "dedup", "v", "g") + require.NoError(t, err) + + result, err := scanner.Scan(ctx, dataDir, id) + require.NoError(t, err) + + require.NoError(t, sm.CompleteSnapshot(ctx, id)) + require.NoError(t, sm.ExportSnapshotMetadata(ctx, dbPath, id)) + + return id, result +} + +func TestDedupOnlySnapshotRestores(t *testing.T) { + log.Initialize(log.Config{}) + t.Parallel() + + fs := afero.NewOsFs() + tempDir := t.TempDir() + + dataDir := filepath.Join(tempDir, "source") + storeDir := filepath.Join(tempDir, "remote") + restoreDir := filepath.Join(tempDir, "restored") + dbPath := filepath.Join(tempDir, "index.sqlite") + + chunkSize := int64(64 * 1024) + maxBlobSize := int64(512 * 1024) + + ctx := context.Background() + + testFiles := writeDedupSourceFiles(t, fs, dataDir, chunkSize) + + env := setupDedupBackupEnv( + ctx, t, fs, storeDir, dbPath, chunkSize, maxBlobSize) + defer func() { _ = env.db.Close() }() + + cfg, storer, repos, sm := env.cfg, env.storer, env.repos, env.sm + makeScanner := env.makeScanner + db := env.db + // First snapshot — uploads all blobs. - id1, err := sm.CreateSnapshotWithName(ctx, cfg.Hostname, "dedup", "v", "g") - require.NoError(t, err) - r1, err := makeScanner().Scan(ctx, dataDir, id1) - require.NoError(t, err) - require.Positive(t, r1.BlobsCreated, "first snapshot should upload at least one blob") - require.NoError(t, sm.CompleteSnapshot(ctx, id1)) - require.NoError(t, sm.ExportSnapshotMetadata(ctx, dbPath, id1)) + _, r1 := runDedupSnapshot(ctx, t, sm, makeScanner(), + cfg.Hostname, dataDir, dbPath) + require.Positive(t, r1.BlobsCreated, + "first snapshot should upload at least one blob") // Second snapshot — same data, every chunk dedups. Sleep past the // second-precision timestamp so the snapshot IDs differ. time.Sleep(1100 * time.Millisecond) - id2, err := sm.CreateSnapshotWithName(ctx, cfg.Hostname, "dedup", "v", "g") - require.NoError(t, err) - r2, err := makeScanner().Scan(ctx, dataDir, id2) - require.NoError(t, err) - require.Equal(t, 0, r2.BlobsCreated, "second snapshot should upload zero new blobs (fully dedup'd)") - require.NoError(t, sm.CompleteSnapshot(ctx, id2)) - require.NoError(t, sm.ExportSnapshotMetadata(ctx, dbPath, id2)) + id2, r2 := runDedupSnapshot(ctx, t, sm, makeScanner(), + cfg.Hostname, dataDir, dbPath) + require.Equal(t, 0, r2.BlobsCreated, + "second snapshot should upload zero new blobs (fully dedup'd)") // snapshot_blobs for id2 must be populated despite no uploads. blobHashes, err := repos.Snapshots.GetBlobHashes(ctx, id2) require.NoError(t, err) - require.NotEmpty(t, blobHashes, "snapshot_blobs for fully-dedup'd snapshot must reference blobs uploaded by prior snapshot") + require.NotEmpty(t, blobHashes, "snapshot_blobs for fully-dedup'd snapshot "+ + "must reference blobs uploaded by prior snapshot") require.NoError(t, db.Close()) @@ -871,7 +1026,7 @@ func TestDedupOnlySnapshotRestores(t *testing.T) { func bytesPattern(tag string, n int) []byte { out := make([]byte, n) for i := range out { - out[i] = byte(tag[i%len(tag)] ^ byte(i&0xff)) + out[i] = tag[i%len(tag)] ^ byte(i&0xff) } return out diff --git a/internal/vaultik/prune.go b/internal/vaultik/prune.go index 42fec89..328b0b2 100644 --- a/internal/vaultik/prune.go +++ b/internal/vaultik/prune.go @@ -6,7 +6,6 @@ import ( "fmt" "strings" - "github.com/dustin/go-humanize" "sneak.berlin/go/vaultik/internal/log" ) @@ -16,6 +15,14 @@ type PruneOptions struct { JSON bool } +// errNukeRequiresForce guards the destructive remote nuke operation. +var errNukeRequiresForce = errors.New( + "nuke requires --force (this deletes ALL remote snapshots and blobs)") + +// metadataDirName is the top-level remote directory holding snapshot +// metadata. +const metadataDirName = "metadata" + // NukeRemote deletes every snapshot's metadata and every blob from remote // storage. After this returns successfully the bucket prefix is empty and // the next backup starts from scratch. @@ -24,29 +31,31 @@ type PruneOptions struct { // confirming with the user. func (v *Vaultik) NukeRemote(force bool) error { if !force { - return errors.New("nuke requires --force (this deletes ALL remote snapshots and blobs)") + return errNukeRequiresForce } - v.UI.Begin("Removing all snapshot metadata from backup destination store.") + v.UI.Beginf("Removing all snapshot metadata from backup destination store.") _, err := v.RemoveAllSnapshots(&RemoveOptions{Force: true}) if err != nil { return fmt.Errorf("removing all snapshots: %w", err) } - v.UI.Begin("Removing any blobs still present in backup destination store.") + v.UI.Beginf("Removing any blobs still present in backup destination store.") err = v.PruneBlobs(&PruneOptions{Force: true}) if err != nil { return fmt.Errorf("pruning blobs: %w", err) } - v.UI.Complete("Backup destination store is now empty.") + v.UI.Completef("Backup destination store is now empty.") return nil } // PruneBlobsResult contains the result of a blob prune operation +// +//nolint:tagliatelle // snake_case is the established JSON output format type PruneBlobsResult struct { BlobsFound int `json:"blobs_found"` BlobsDeleted int `json:"blobs_deleted"` @@ -113,14 +122,16 @@ func (v *Vaultik) PruneBlobs(opts *PruneOptions) error { return nil } - log.Info("Found unreferenced blobs", "count", len(unreferencedBlobs), "total_size", humanize.Bytes(uint64(totalSize))) + log.Info("Found unreferenced blobs", + "count", len(unreferencedBlobs), "total_size", ubytes(totalSize)) if !opts.JSON { - v.printfStdout("Found %d unreferenced blob(s) totaling %s\n", len(unreferencedBlobs), humanize.Bytes(uint64(totalSize))) + v.stdoutf("Found %d unreferenced blob(s) totaling %s\n", + len(unreferencedBlobs), ubytes(totalSize)) } if !opts.Force && !opts.JSON { - v.printfStdout("\nDelete %d unreferenced blob(s)? [y/N] ", len(unreferencedBlobs)) + v.stdoutf("\nDelete %d unreferenced blob(s)? [y/N] ", len(unreferencedBlobs)) var confirm string @@ -128,7 +139,7 @@ func (v *Vaultik) PruneBlobs(opts *PruneOptions) error { if err != nil { v.printlnStdout("Cancelled") - return nil + return nil //nolint:nilerr // read failure means no confirmation } if strings.ToLower(confirm) != "y" { @@ -144,16 +155,18 @@ func (v *Vaultik) PruneBlobs(opts *PruneOptions) error { return v.outputPruneBlobsJSON(result) } - v.printfStdout("\nDeleted %d blob(s) totaling %s\n", result.BlobsDeleted, humanize.Bytes(uint64(result.BytesFreed))) + v.stdoutf("\nDeleted %d blob(s) totaling %s\n", + result.BlobsDeleted, ubytes(result.BytesFreed)) if result.BlobsFailed > 0 { - v.printfStdout("Failed to delete %d blob(s)\n", result.BlobsFailed) + v.stdoutf("Failed to delete %d blob(s)\n", result.BlobsFailed) } return nil } -// collectReferencedBlobs downloads all manifests and returns the set of referenced blob hashes +// collectReferencedBlobs downloads all manifests and returns the set of +// referenced blob hashes. func (v *Vaultik) collectReferencedBlobs() (map[string]bool, error) { log.Info("Listing remote snapshots") // IDs returned by listUniqueSnapshotIDs are remote keys (hashed @@ -185,7 +198,8 @@ func (v *Vaultik) collectReferencedBlobs() (map[string]bool, error) { manifestCount++ } - log.Info("Processed manifests", "count", manifestCount, "unique_blobs_referenced", len(allBlobsReferenced)) + log.Info("Processed manifests", + "count", manifestCount, "unique_blobs_referenced", len(allBlobsReferenced)) return allBlobsReferenced, nil } @@ -203,8 +217,10 @@ func (v *Vaultik) listUniqueSnapshotIDs() ([]string, error) { } parts := strings.Split(object.Key, "/") - if len(parts) >= 2 && parts[0] == "metadata" && parts[1] != "" { - if strings.HasSuffix(object.Key, "/") || strings.Contains(object.Key, "/manifest.json.zst") { + if len(parts) >= minSnapshotIDParts && + parts[0] == metadataDirName && parts[1] != "" { + if strings.HasSuffix(object.Key, "/") || + strings.Contains(object.Key, "/manifest.json.zst") { snapshotID := parts[1] if !seen[snapshotID] { seen[snapshotID] = true @@ -230,7 +246,7 @@ func (v *Vaultik) listAllRemoteBlobs() (map[string]int64, error) { } parts := strings.Split(object.Key, "/") - if len(parts) == 4 && parts[0] == "blobs" { + if len(parts) == blobKeyParts && parts[0] == "blobs" { allBlobs[parts[3]] = object.Size } } @@ -240,8 +256,11 @@ func (v *Vaultik) listAllRemoteBlobs() (map[string]int64, error) { return allBlobs, nil } -// findUnreferencedBlobs returns blob hashes not referenced by any manifest and their total size -func (v *Vaultik) findUnreferencedBlobs(allBlobs map[string]int64, referenced map[string]bool) ([]string, int64) { +// findUnreferencedBlobs returns blob hashes not referenced by any +// manifest and their total size. +func (v *Vaultik) findUnreferencedBlobs( + allBlobs map[string]int64, referenced map[string]bool, +) ([]string, int64) { var ( unreferenced []string totalSize int64 @@ -257,8 +276,11 @@ func (v *Vaultik) findUnreferencedBlobs(allBlobs map[string]int64, referenced ma return unreferenced, totalSize } -// deleteUnreferencedBlobs deletes the given blobs from storage and populates the result -func (v *Vaultik) deleteUnreferencedBlobs(unreferencedBlobs []string, allBlobs map[string]int64, result *PruneBlobsResult) { +// deleteUnreferencedBlobs deletes the given blobs from storage and +// populates the result. +func (v *Vaultik) deleteUnreferencedBlobs( + unreferencedBlobs []string, allBlobs map[string]int64, result *PruneBlobsResult, +) { log.Info("Deleting unreferenced blobs") for i, hash := range unreferencedBlobs { @@ -274,11 +296,12 @@ func (v *Vaultik) deleteUnreferencedBlobs(unreferencedBlobs []string, allBlobs m result.BlobsDeleted++ result.BytesFreed += allBlobs[hash] - if (i+1)%100 == 0 || i == len(unreferencedBlobs)-1 { + if (i+1)%progressLogEvery == 0 || i == len(unreferencedBlobs)-1 { log.Info("Deletion progress", "deleted", i+1, "total", len(unreferencedBlobs), - "percent", fmt.Sprintf("%.1f%%", float64(i+1)/float64(len(unreferencedBlobs))*100), + "percent", fmt.Sprintf("%.1f%%", + float64(i+1)/float64(len(unreferencedBlobs))*percentScale), ) } } @@ -287,7 +310,7 @@ func (v *Vaultik) deleteUnreferencedBlobs(unreferencedBlobs []string, allBlobs m log.Info("Prune complete", "deleted_count", result.BlobsDeleted, - "deleted_size", humanize.Bytes(uint64(result.BytesFreed)), + "deleted_size", ubytes(result.BytesFreed), "failed", result.BlobsFailed, ) } diff --git a/internal/vaultik/purge_per_name_test.go b/internal/vaultik/purge_per_name_test.go index b8c4782..318f27c 100644 --- a/internal/vaultik/purge_per_name_test.go +++ b/internal/vaultik/purge_per_name_test.go @@ -16,12 +16,19 @@ import ( "sneak.berlin/go/vaultik/internal/vaultik" ) +// Snapshot IDs reused across the purge tests. +const ( + snapSystemT0 = "testhost_system_2026-01-01T00:00:00Z" + snapHomeT0 = "testhost_home_2026-01-01T00:00:00Z" + snapHomeT1 = "testhost_home_2026-01-01T01:00:00Z" + snapHomeT3 = "testhost_home_2026-01-01T03:00:00Z" +) + // setupPurgeTest creates a Vaultik instance with an in-memory database and mock // storage pre-populated with the given snapshot IDs. Each snapshot is marked as // completed. Remote metadata stubs are created so syncWithRemote keeps them. func setupPurgeTest(t *testing.T, snapshotIDs []string) *vaultik.Vaultik { t.Helper() - log.Initialize(log.Config{}) ctx := context.Background() db, err := database.New(ctx, ":memory:") @@ -44,7 +51,7 @@ func setupPurgeTest(t *testing.T, snapshotIDs []string) *vaultik.Vaultik { snap := &database.Snapshot{ ID: types.SnapshotID(id), Hostname: "testhost", - VaultikVersion: "test", + VaultikVersion: testLabel, StartedAt: startedAt, CompletedAt: &completedAt, } @@ -96,13 +103,16 @@ func listRemainingSnapshots(t *testing.T, v *vaultik.Vaultik) []string { } func TestPurgeKeepLatest_PerName(t *testing.T) { + log.Initialize(log.Config{}) + t.Parallel() + // Create snapshots for two different names: "home" and "system". // With per-name --keep-latest, the latest of each should be kept. snapshotIDs := []string{ - "testhost_system_2026-01-01T00:00:00Z", - "testhost_home_2026-01-01T01:00:00Z", + snapSystemT0, + snapHomeT1, "testhost_system_2026-01-01T02:00:00Z", - "testhost_home_2026-01-01T03:00:00Z", + snapHomeT3, "testhost_system_2026-01-01T04:00:00Z", } @@ -118,15 +128,19 @@ func TestPurgeKeepLatest_PerName(t *testing.T) { // Should keep the latest of each name assert.Len(t, remaining, 2, "should keep exactly 2 snapshots (one per name)") - assert.Contains(t, remaining, "testhost_system_2026-01-01T04:00:00Z", "should keep latest system") - assert.Contains(t, remaining, "testhost_home_2026-01-01T03:00:00Z", "should keep latest home") + assert.Contains(t, remaining, "testhost_system_2026-01-01T04:00:00Z", + "should keep latest system") + assert.Contains(t, remaining, snapHomeT3, "should keep latest home") } func TestPurgeKeepLatest_SingleName(t *testing.T) { + log.Initialize(log.Config{}) + t.Parallel() + // All snapshots have the same name — keep-latest should keep exactly one. snapshotIDs := []string{ - "testhost_home_2026-01-01T00:00:00Z", - "testhost_home_2026-01-01T01:00:00Z", + snapHomeT0, + snapHomeT1, "testhost_home_2026-01-01T02:00:00Z", } @@ -140,17 +154,21 @@ func TestPurgeKeepLatest_SingleName(t *testing.T) { remaining := listRemainingSnapshots(t, v) assert.Len(t, remaining, 1) - assert.Contains(t, remaining, "testhost_home_2026-01-01T02:00:00Z", "should keep the newest") + assert.Contains(t, remaining, "testhost_home_2026-01-01T02:00:00Z", + "should keep the newest") } func TestPurgeKeepLatest_WithNameFilter(t *testing.T) { + log.Initialize(log.Config{}) + t.Parallel() + // Use --name to filter purge to only "home" snapshots. // "system" snapshots should be untouched. snapshotIDs := []string{ - "testhost_system_2026-01-01T00:00:00Z", - "testhost_home_2026-01-01T01:00:00Z", + snapSystemT0, + snapHomeT1, "testhost_system_2026-01-01T02:00:00Z", - "testhost_home_2026-01-01T03:00:00Z", + snapHomeT3, "testhost_home_2026-01-01T04:00:00Z", } @@ -167,12 +185,15 @@ func TestPurgeKeepLatest_WithNameFilter(t *testing.T) { // 2 system snapshots untouched + 1 latest home = 3 assert.Len(t, remaining, 3) - assert.Contains(t, remaining, "testhost_system_2026-01-01T00:00:00Z") + assert.Contains(t, remaining, snapSystemT0) assert.Contains(t, remaining, "testhost_system_2026-01-01T02:00:00Z") assert.Contains(t, remaining, "testhost_home_2026-01-01T04:00:00Z") } func TestPurgeKeepLatest_NoSnapshots(t *testing.T) { + log.Initialize(log.Config{}) + t.Parallel() + v := setupPurgeTest(t, nil) err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{ @@ -183,8 +204,11 @@ func TestPurgeKeepLatest_NoSnapshots(t *testing.T) { } func TestPurgeKeepLatest_NameFilterNoMatch(t *testing.T) { + log.Initialize(log.Config{}) + t.Parallel() + snapshotIDs := []string{ - "testhost_system_2026-01-01T00:00:00Z", + snapSystemT0, "testhost_system_2026-01-01T01:00:00Z", } @@ -203,13 +227,16 @@ func TestPurgeKeepLatest_NameFilterNoMatch(t *testing.T) { } func TestPurgeOlderThan_WithNameFilter(t *testing.T) { + log.Initialize(log.Config{}) + t.Parallel() + // Snapshots with different names and timestamps. // --older-than should apply only to the named subset when --name is used. snapshotIDs := []string{ "testhost_system_2020-01-01T00:00:00Z", "testhost_home_2020-01-01T00:00:00Z", - "testhost_system_2026-01-01T00:00:00Z", - "testhost_home_2026-01-01T00:00:00Z", + snapSystemT0, + snapHomeT0, } v := setupPurgeTest(t, snapshotIDs) @@ -227,14 +254,17 @@ func TestPurgeOlderThan_WithNameFilter(t *testing.T) { // Old system stays (not filtered by name), old home deleted, recent ones stay assert.Len(t, remaining, 3) assert.Contains(t, remaining, "testhost_system_2020-01-01T00:00:00Z") - assert.Contains(t, remaining, "testhost_system_2026-01-01T00:00:00Z") - assert.Contains(t, remaining, "testhost_home_2026-01-01T00:00:00Z") + assert.Contains(t, remaining, snapSystemT0) + assert.Contains(t, remaining, snapHomeT0) } func TestPurgeKeepLatest_ThreeNames(t *testing.T) { + log.Initialize(log.Config{}) + t.Parallel() + // Three different snapshot names with multiple snapshots each. snapshotIDs := []string{ - "testhost_home_2026-01-01T00:00:00Z", + snapHomeT0, "testhost_system_2026-01-01T01:00:00Z", "testhost_media_2026-01-01T02:00:00Z", "testhost_home_2026-01-01T03:00:00Z", diff --git a/internal/vaultik/remove_snapshot_test.go b/internal/vaultik/remove_snapshot_test.go index 2bbf33f..bed9ace 100644 --- a/internal/vaultik/remove_snapshot_test.go +++ b/internal/vaultik/remove_snapshot_test.go @@ -29,7 +29,7 @@ func newTestStorer() *testStorer { } } -func (s *testStorer) Put(ctx context.Context, key string, reader io.Reader) error { +func (s *testStorer) Put(_ context.Context, key string, reader io.Reader) error { s.mu.Lock() defer s.mu.Unlock() @@ -43,11 +43,14 @@ func (s *testStorer) Put(ctx context.Context, key string, reader io.Reader) erro return nil } -func (s *testStorer) PutWithProgress(ctx context.Context, key string, reader io.Reader, size int64, progress storage.ProgressCallback) error { +func (s *testStorer) PutWithProgress( + ctx context.Context, key string, reader io.Reader, + _ int64, _ storage.ProgressCallback, +) error { return s.Put(ctx, key, reader) } -func (s *testStorer) Get(ctx context.Context, key string) (io.ReadCloser, error) { +func (s *testStorer) Get(_ context.Context, key string) (io.ReadCloser, error) { s.mu.Lock() defer s.mu.Unlock() @@ -59,7 +62,7 @@ func (s *testStorer) Get(ctx context.Context, key string) (io.ReadCloser, error) return io.NopCloser(bytes.NewReader(data)), nil } -func (s *testStorer) Stat(ctx context.Context, key string) (*storage.ObjectInfo, error) { +func (s *testStorer) Stat(_ context.Context, key string) (*storage.ObjectInfo, error) { s.mu.Lock() defer s.mu.Unlock() @@ -74,7 +77,7 @@ func (s *testStorer) Stat(ctx context.Context, key string) (*storage.ObjectInfo, }, nil } -func (s *testStorer) Delete(ctx context.Context, key string) error { +func (s *testStorer) Delete(_ context.Context, key string) error { s.mu.Lock() defer s.mu.Unlock() @@ -83,7 +86,7 @@ func (s *testStorer) Delete(ctx context.Context, key string) error { return nil } -func (s *testStorer) List(ctx context.Context, prefix string) ([]string, error) { +func (s *testStorer) List(_ context.Context, prefix string) ([]string, error) { s.mu.Lock() defer s.mu.Unlock() @@ -98,7 +101,9 @@ func (s *testStorer) List(ctx context.Context, prefix string) ([]string, error) return keys, nil } -func (s *testStorer) ListStream(ctx context.Context, prefix string) <-chan storage.ObjectInfo { +func (s *testStorer) ListStream( + _ context.Context, prefix string, +) <-chan storage.ObjectInfo { ch := make(chan storage.ObjectInfo) go func() { @@ -120,6 +125,13 @@ func (s *testStorer) ListStream(ctx context.Context, prefix string) <-chan stora return ch } +func (s *testStorer) Info() storage.Info { + return storage.Info{ + Type: testLabel, + Location: "memory", + } +} + func (s *testStorer) hasKey(key string) bool { s.mu.Lock() defer s.mu.Unlock() @@ -136,17 +148,16 @@ func (s *testStorer) keyCount() int { return len(s.data) } -func (s *testStorer) Info() storage.StorageInfo { - return storage.StorageInfo{ - Type: "test", - Location: "memory", - } -} +// testBlobHashA is a fixture blob hash reused across the remove tests. +const testBlobHashA = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" // addManifest creates a compressed manifest in storage at the same // hashed path the production code uses. snapshotID is the human ID; // the storage path uses RemoteSnapshotKey(id). -func addManifest(t *testing.T, store *testStorer, snapshotID string, blobHashes []string) { +func addManifest( + t *testing.T, store *testStorer, snapshotID string, blobHashes []string, +) { t.Helper() blobs := make([]snapshot.BlobInfo, len(blobHashes)) @@ -173,11 +184,12 @@ func addManifest(t *testing.T, store *testStorer, snapshotID string, blobHashes } // remoteKeyPath returns the storage-relative path to a snapshot's -// metadata directory or manifest under the hashed remote-key scheme. -// Tests use this in hasKey/asserts to avoid scattering RemoteSnapshotKey -// calls throughout. -func remoteKeyPath(snapshotID, suffix string) string { - return "metadata/" + snapshot.RemoteSnapshotKey(snapshotID) + "/" + suffix +// manifest under the hashed remote-key scheme. Tests use this in +// hasKey/asserts to avoid scattering RemoteSnapshotKey calls +// throughout. +func remoteKeyPath(snapshotID string) string { + return "metadata/" + snapshot.RemoteSnapshotKey(snapshotID) + + "/manifest.json.zst" } // addBlob adds a fake blob to storage @@ -206,10 +218,11 @@ func addBlob(t *testing.T, store *testStorer, hash string) { // untouched. func TestRemoveSnapshot_LocalOnly_PreservesRemote(t *testing.T) { log.Initialize(log.Config{}) + t.Parallel() store := newTestStorer() - blobA := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + blobA := testBlobHashA addManifest(t, store, "snapshot-001", []string{blobA}) addBlob(t, store, blobA) @@ -223,9 +236,10 @@ func TestRemoveSnapshot_LocalOnly_PreservesRemote(t *testing.T) { assert.False(t, result.RemoteRemoved) assert.True(t, store.hasKey("blobs/aa/aa/"+blobA)) - assert.True(t, store.hasKey(remoteKeyPath("snapshot-001", "manifest.json.zst"))) + assert.True(t, store.hasKey(remoteKeyPath("snapshot-001"))) - assert.Contains(t, tv.Stdout.String(), "Removed snapshot 'snapshot-001' from local database") + assert.Contains(t, tv.Stdout.String(), + "Removed snapshot 'snapshot-001' from local database") } // TestRemoveSnapshot_DefaultRemovesMetadataNotBlobs is the canonical @@ -235,10 +249,11 @@ func TestRemoveSnapshot_LocalOnly_PreservesRemote(t *testing.T) { // remaining remote manifest, and the output prints that exact command. func TestRemoveSnapshot_DefaultRemovesMetadataNotBlobs(t *testing.T) { log.Initialize(log.Config{}) + t.Parallel() store := newTestStorer() - blobUnique := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + blobUnique := testBlobHashA blobShared := "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" addManifest(t, store, "snapshot-001", []string{blobUnique, blobShared}) @@ -255,8 +270,8 @@ func TestRemoveSnapshot_DefaultRemovesMetadataNotBlobs(t *testing.T) { assert.Equal(t, "snapshot-001", result.SnapshotID) assert.True(t, result.RemoteRemoved) - assert.False(t, store.hasKey(remoteKeyPath("snapshot-001", "manifest.json.zst"))) - assert.True(t, store.hasKey(remoteKeyPath("snapshot-002", "manifest.json.zst"))) + assert.False(t, store.hasKey(remoteKeyPath("snapshot-001"))) + assert.True(t, store.hasKey(remoteKeyPath("snapshot-002"))) // Blobs are intentionally NOT touched — that's what `vaultik prune` // is for. assert.True(t, store.hasKey("blobs/aa/aa/"+blobUnique)) @@ -270,10 +285,11 @@ func TestRemoveSnapshot_DefaultRemovesMetadataNotBlobs(t *testing.T) { func TestRemoveSnapshot_DryRun(t *testing.T) { log.Initialize(log.Config{}) + t.Parallel() store := newTestStorer() - blobA := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + blobA := testBlobHashA addManifest(t, store, "snapshot-001", []string{blobA}) addBlob(t, store, blobA) @@ -289,13 +305,14 @@ func TestRemoveSnapshot_DryRun(t *testing.T) { assert.Equal(t, initialCount, store.keyCount()) assert.True(t, store.hasKey("blobs/aa/aa/"+blobA)) - assert.True(t, store.hasKey(remoteKeyPath("snapshot-001", "manifest.json.zst"))) + assert.True(t, store.hasKey(remoteKeyPath("snapshot-001"))) assert.Contains(t, tv.Stdout.String(), "[Dry run - no changes made]") } func TestRemoveAllSnapshots_RequiresForce(t *testing.T) { log.Initialize(log.Config{}) + t.Parallel() store := newTestStorer() addManifest(t, store, "snapshot-001", []string{}) @@ -306,16 +323,17 @@ func TestRemoveAllSnapshots_RequiresForce(t *testing.T) { opts := &vaultik.RemoveOptions{} // No Force _, err := tv.RemoveAllSnapshots(opts) - assert.Error(t, err) + require.Error(t, err) assert.Contains(t, err.Error(), "--all requires --force") } func TestRemoveAllSnapshots_WithForce(t *testing.T) { log.Initialize(log.Config{}) + t.Parallel() store := newTestStorer() - blobA := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + blobA := testBlobHashA addManifest(t, store, "snapshot-001", []string{blobA}) addManifest(t, store, "snapshot-002", []string{blobA}) addBlob(t, store, blobA) @@ -331,8 +349,8 @@ func TestRemoveAllSnapshots_WithForce(t *testing.T) { // Blobs intentionally preserved — that's prune's job. assert.True(t, store.hasKey("blobs/aa/aa/"+blobA)) - assert.False(t, store.hasKey(remoteKeyPath("snapshot-001", "manifest.json.zst"))) - assert.False(t, store.hasKey(remoteKeyPath("snapshot-002", "manifest.json.zst"))) + assert.False(t, store.hasKey(remoteKeyPath("snapshot-001"))) + assert.False(t, store.hasKey(remoteKeyPath("snapshot-002"))) out := tv.Stdout.String() assert.Contains(t, out, "Removed 2 snapshot(s)") @@ -342,6 +360,7 @@ func TestRemoveAllSnapshots_WithForce(t *testing.T) { func TestRemoveAllSnapshots_DryRun(t *testing.T) { log.Initialize(log.Config{}) + t.Parallel() store := newTestStorer() addManifest(t, store, "snapshot-001", []string{}) @@ -368,6 +387,7 @@ func TestRemoveAllSnapshots_DryRun(t *testing.T) { func TestRemoveAllSnapshots_NoSnapshots(t *testing.T) { log.Initialize(log.Config{}) + t.Parallel() store := newTestStorer() // No snapshots added diff --git a/internal/vaultik/restore.go b/internal/vaultik/restore.go index 604a475..36a632e 100644 --- a/internal/vaultik/restore.go +++ b/internal/vaultik/restore.go @@ -14,7 +14,6 @@ import ( "time" "filippo.io/age" - "github.com/dustin/go-humanize" "github.com/spf13/afero" "sneak.berlin/go/vaultik/internal/blobgen" "sneak.berlin/go/vaultik/internal/database" @@ -23,6 +22,34 @@ import ( "sneak.berlin/go/vaultik/internal/types" ) +// Sentinel errors for restore failures. +var ( + errFilesFailedRestore = errors.New("file(s) failed to restore") + errFilesFailedVerify = errors.New("files failed verification") + errDecryptionKeyRequired = errors.New( + "decryption key required for restore\n\n" + + "Set the VAULTIK_AGE_SECRET_KEY environment variable to your " + + "age private key:\n" + + " export VAULTIK_AGE_SECRET_KEY='AGE-SECRET-KEY-...'") + errBlobMissingFromIndex = errors.New("blob hash missing from blob index") + errChunkNotInAnyBlob = errors.New("chunk not found in any blob") + errBlobIDNotInHashIndex = errors.New("blob id missing from hash index") + errShortChunkRead = errors.New("short read") +) + +// restoreDirMode is the permission mode for directories created while +// restoring (parent directories and the target root; restored +// directories themselves get their stored mode). +const restoreDirMode = 0o755 + +// sweepIntervalDivisor sets the sweeper threshold to one N-th of the +// configured blob size limit. +const sweepIntervalDivisor = 100 + +// restoreStatusInterval is how often periodic progress lines are +// printed during restore and verify. +const restoreStatusInterval = 15 * time.Second + // RestoreOptions contains options for the restore operation type RestoreOptions struct { SnapshotID string @@ -91,16 +118,17 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error { if len(files) == 0 { log.Warn("No files found to restore") - v.UI.Warning("No files found to restore.") + v.UI.Warningf("No files found to restore.") return nil } log.Info("Found files to restore", "count", len(files)) - v.UI.Info("Found %s files to restore.", v.UI.Count(len(files))) + v.UI.Infof("Found %s files to restore.", v.UI.Count(len(files))) // Step 3: Create target directory - if err := v.Fs.MkdirAll(opts.TargetDir, 0755); err != nil { + err = v.Fs.MkdirAll(opts.TargetDir, restoreDirMode) + if err != nil { return fmt.Errorf("creating target directory: %w", err) } @@ -120,27 +148,40 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error { log.Info("Restore complete", "files_restored", result.FilesRestored, - "bytes_restored", humanize.Bytes(uint64(result.BytesRestored)), + "bytes_restored", ubytes(result.BytesRestored), "blobs_downloaded", result.BlobsDownloaded, - "bytes_downloaded", humanize.Bytes(uint64(result.BytesDownloaded)), + "bytes_downloaded", ubytes(result.BytesDownloaded), "duration", result.Duration, ) - v.UI.Complete("Restored %s files (%s) in %s.", + v.UI.Completef("Restored %s files (%s) in %s.", v.UI.Count(result.FilesRestored), v.UI.Size(result.BytesRestored), v.UI.Duration(result.Duration), ) + return v.finishRestore(repos, files, opts, result) +} + +// finishRestore emits the post-restore warnings, runs optional +// verification, and converts any failed-file count into an error. +func (v *Vaultik) finishRestore( + repos *database.Repositories, + files []*database.File, + opts *RestoreOptions, + result *RestoreResult, +) error { if os.Geteuid() != 0 { - v.UI.Warning("Restore did not preserve file ownership: chown(2) requires root. Re-run as root (e.g. with sudo) if you need original UID/GID preserved.") + v.UI.Warningf("Restore did not preserve file ownership: chown(2) " + + "requires root. Re-run as root (e.g. with sudo) if you need " + + "original UID/GID preserved.") } if result.FilesFailed > 0 { - v.UI.Warning("%d file(s) failed to restore:", result.FilesFailed) + v.UI.Warningf("%d file(s) failed to restore:", result.FilesFailed) for _, path := range result.FailedFiles { - v.UI.Detail("%s", v.UI.Path(path)) + v.UI.Detailf("%s", v.UI.Path(path)) } } @@ -153,16 +194,19 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error { } if result.FilesFailed > 0 { - return fmt.Errorf("%d file(s) failed to restore", result.FilesFailed) + return fmt.Errorf("%d %w", result.FilesFailed, errFilesFailedRestore) } return nil } -// prepareRestoreIdentity validates that an age secret key is configured and parses it +// prepareRestoreIdentity validates that an age secret key is configured +// and parses it. +// +//nolint:ireturn // age.Identity is the decryption abstraction by design func (v *Vaultik) prepareRestoreIdentity() (age.Identity, error) { if v.Config.AgeSecretKey == "" { - return nil, errors.New("decryption key required for restore\n\nSet the VAULTIK_AGE_SECRET_KEY environment variable to your age private key:\n export VAULTIK_AGE_SECRET_KEY='AGE-SECRET-KEY-...'") + return nil, errDecryptionKeyRequired } identity, err := age.ParseX25519Identity(v.Config.AgeSecretKey) @@ -212,22 +256,12 @@ func (v *Vaultik) restoreAllFiles( // Per-restore sweep state: every blob_size_limit/100 bytes written, // scan the cache and delete any blob whose remaining file references // are all already restored. - sweeper := newRestoreSweeper(v.ctx, repos, blobCache, v.Config.BlobSizeLimit.Int64()/100) + sweeper := newRestoreSweeper(v.ctx, repos, blobCache, + v.Config.BlobSizeLimit.Int64()/sweepIntervalDivisor) - // Pre-fetch every blob row once so chunk extraction can map a - // blob_id to its hash without a DB round-trip per chunk. - blobsByID, err := repos.Blobs.GetAll(v.ctx) + blobByHash, blobIDToHash, err := v.buildBlobIndexes(repos) if err != nil { - return nil, fmt.Errorf("fetching blob index: %w", err) - } - - blobIDToHash := make(map[string]string, len(blobsByID)) - - blobByHash := make(map[string]*database.Blob, len(blobsByID)) - for id, blob := range blobsByID { - hash := blob.Hash.String() - blobIDToHash[id] = hash - blobByHash[hash] = blob + return nil, err } plan, err := newRestorePlan(v.ctx, repos, files, chunkToBlobMap, blobIDToHash) @@ -235,20 +269,9 @@ func (v *Vaultik) restoreAllFiles( return nil, fmt.Errorf("building restore plan: %w", err) } - // Index files by ID so the loop can look them up by the IDs the - // plan hands back. - filesByID := make(map[types.FileID]*database.File, len(files)) - for _, f := range files { - filesByID[f.ID] = f - } + filesByID, totalBytesExpected := indexRestoreFiles(files) - // Calculate total bytes expected for percentage / ETA arithmetic. - var totalBytesExpected int64 - for _, file := range files { - totalBytesExpected += file.Size - } - - v.UI.Begin("Restoring %s files (%s) to %s.", + v.UI.Beginf("Restoring %s files (%s) to %s.", v.UI.Count(len(files)), v.UI.Size(totalBytesExpected), v.UI.Path(opts.TargetDir)) @@ -268,52 +291,41 @@ func (v *Vaultik) restoreAllFiles( runningAsRoot: os.Geteuid() == 0, } + err = v.runRestoreLoop(session, plan, filesByID, totalBytesExpected) + if err != nil { + return nil, err + } + + return result, nil +} + +// runRestoreLoop drains the restore plan: restore files as their blobs +// become available, download the next blob set when nothing is ready, +// and emit periodic progress. +func (v *Vaultik) runRestoreLoop( + session *restoreSession, plan *restorePlan, + filesByID map[types.FileID]*database.File, totalBytesExpected int64, +) error { // Periodic progress output, matching the snapshot create cadence. startTime := time.Now() lastStatusTime := startTime - - const statusInterval = 15 * time.Second - processed := 0 + totalFiles := len(filesByID) for plan.hasPending() { if v.ctx.Err() != nil { - return nil, v.ctx.Err() + return v.ctx.Err() } fileID, ready := plan.popReady() if !ready { - // No file is fully cache-served. First free any blobs - // whose file sets are exhausted — without this, the - // blob whose last file we just finished would still be - // cached when we Put the next one, briefly pushing - // peak occupancy from 1 to 2. - sweeper.sweep() - - // Pick the pending file with the smallest uncached - // blob set and download its blobs. After each blob - // lands, the plan moves any pending file whose set - // just emptied onto the ready queue. - next := plan.pickNextDownload() - if next.IsZero() { - break + downloaded, err := session.downloadNextBlobSet(plan) + if err != nil { + return err } - for _, hash := range plan.blobsNeeded(next) { - blob, ok := blobByHash[hash] - if !ok { - return nil, fmt.Errorf("blob hash %s missing from blob index", hash[:16]) - } - - err := session.downloadBlobToCache(hash, blob.CompressedSize) - if err != nil { - return nil, fmt.Errorf("downloading blob %s: %w", hash[:16], err) - } - - result.BlobsDownloaded++ - result.BytesDownloaded += blob.CompressedSize - - plan.markBlobCached(hash) + if !downloaded { + break } continue @@ -323,54 +335,173 @@ func (v *Vaultik) restoreAllFiles( err := session.restoreFile(file) if err != nil { - log.Error("Failed to restore file", "path", file.Path, "error", err) - - if !opts.SkipErrors { - return nil, fmt.Errorf("restoring %s: %w (pass --skip-errors to continue past restore failures)", file.Path, err) + err = v.handleRestoreFileError( + plan, session.opts, session.result, file, fileID, err) + if err != nil { + return err } - v.UI.Error("Failed to restore %s: %v. Skipping (--skip-errors).", v.UI.Path(file.Path.String()), err) - - result.FilesFailed++ - result.FailedFiles = append(result.FailedFiles, file.Path.String()) - - plan.finishFile(fileID) - continue } // Record the file as restored so the sweeper can free blobs // once all referencing files are done, and drop it from the // plan's indexes so future picks ignore it. - sweeper.fileRestored(fileID.String()) + session.sweeper.fileRestored(fileID.String()) plan.finishFile(fileID) processed++ - if time.Since(lastStatusTime) >= statusInterval { - v.printRestoreProgress(processed, len(files), result.BytesRestored, totalBytesExpected, startTime) - - lastStatusTime = time.Now() - } - - // Structured progress log for --verbose / JSON consumers. - if processed%100 == 0 || processed == len(files) { - log.Info("Restore progress", - "files", fmt.Sprintf("%d/%d", processed, len(files)), - "bytes", humanize.Bytes(uint64(result.BytesRestored)), - ) - } + v.restoreProgressTick(processed, totalFiles, + session.result.BytesRestored, + totalBytesExpected, startTime, &lastStatusTime) } - return result, nil + return nil +} + +// downloadNextBlobSet is invoked when no file is fully cache-served. +// It first frees any blobs whose file sets are exhausted — without +// this, the blob whose last file we just finished would still be +// cached when we Put the next one, briefly pushing peak occupancy from +// 1 to 2. It then picks the pending file with the smallest uncached +// blob set and downloads its blobs; after each blob lands, the plan +// moves any pending file whose set just emptied onto the ready queue. +// Returns false when nothing is pending download (the caller stops). +func (s *restoreSession) downloadNextBlobSet(plan *restorePlan) (bool, error) { + s.sweeper.sweep() + + next := plan.pickNextDownload() + if next.IsZero() { + return false, nil + } + + for _, hash := range plan.blobsNeeded(next) { + blob, ok := s.blobByHash[hash] + if !ok { + return false, fmt.Errorf("%w: %s", errBlobMissingFromIndex, hash[:16]) + } + + err := s.downloadBlobToCache(hash, blob.CompressedSize) + if err != nil { + return false, fmt.Errorf("downloading blob %s: %w", hash[:16], err) + } + + s.result.BlobsDownloaded++ + s.result.BytesDownloaded += blob.CompressedSize + + plan.markBlobCached(hash) + } + + return true, nil +} + +// indexRestoreFiles indexes files by ID for plan lookups and sums the +// expected byte total for percentage / ETA arithmetic. +func indexRestoreFiles( + files []*database.File, +) (map[types.FileID]*database.File, int64) { + filesByID := make(map[types.FileID]*database.File, len(files)) + + var totalBytesExpected int64 + + for _, f := range files { + filesByID[f.ID] = f + totalBytesExpected += f.Size + } + + return filesByID, totalBytesExpected +} + +// buildBlobIndexes pre-fetches every blob row once so chunk extraction +// can map a blob_id to its hash without a DB round-trip per chunk. +func (v *Vaultik) buildBlobIndexes( + repos *database.Repositories, +) (map[string]*database.Blob, map[string]string, error) { + blobsByID, err := repos.Blobs.GetAll(v.ctx) + if err != nil { + return nil, nil, fmt.Errorf("fetching blob index: %w", err) + } + + blobIDToHash := make(map[string]string, len(blobsByID)) + + blobByHash := make(map[string]*database.Blob, len(blobsByID)) + for id, blob := range blobsByID { + hash := blob.Hash.String() + blobIDToHash[id] = hash + blobByHash[hash] = blob + } + + return blobByHash, blobIDToHash, nil +} + +// restoreProgressTick emits the periodic UI status line and structured +// progress log during the restore loop. +func (v *Vaultik) restoreProgressTick( + processed, totalFiles int, bytesRestored, totalBytesExpected int64, + startTime time.Time, lastStatusTime *time.Time, +) { + if time.Since(*lastStatusTime) >= restoreStatusInterval { + v.printRestoreProgress( + processed, totalFiles, bytesRestored, + totalBytesExpected, startTime) + + *lastStatusTime = time.Now() + } + + // Structured progress log for --verbose / JSON consumers. + if processed%progressLogEvery == 0 || processed == totalFiles { + log.Info("Restore progress", + "files", fmt.Sprintf("%d/%d", processed, totalFiles), + "bytes", ubytes(bytesRestored), + ) + } +} + +// handleRestoreFileError records a per-file restore failure: fatal unless +// --skip-errors is set, in which case the file is counted as failed and +// dropped from the plan. +func (v *Vaultik) handleRestoreFileError( + plan *restorePlan, opts *RestoreOptions, result *RestoreResult, + file *database.File, fileID types.FileID, err error, +) error { + log.Error("Failed to restore file", "path", file.Path, "error", err) + + if !opts.SkipErrors { + return fmt.Errorf( + "restoring %s: %w (pass --skip-errors to continue past "+ + "restore failures)", file.Path, err) + } + + v.UI.Errorf("Failed to restore %s: %v. Skipping (--skip-errors).", + v.UI.Path(file.Path.String()), err) + + result.FilesFailed++ + result.FailedFiles = append(result.FailedFiles, file.Path.String()) + + plan.finishFile(fileID) + + return nil } // printRestoreProgress emits a periodic restore-phase status line via // the UI writer, mirroring scanner.printProcessingProgress so the two // long-running commands have the same on-screen rhythm. -func (v *Vaultik) printRestoreProgress(filesDone, totalFiles int, bytesDone, totalBytes int64, startTime time.Time) { +func (v *Vaultik) printRestoreProgress( + filesDone, totalFiles int, bytesDone, totalBytes int64, startTime time.Time, +) { + v.printPhaseProgress("Restore", "restore", + filesDone, totalFiles, bytesDone, totalBytes, startTime) +} + +// printPhaseProgress emits a periodic status line for a long-running +// phase (restore or verify) so user-facing pacing is uniform. +func (v *Vaultik) printPhaseProgress( + title, phase string, + filesDone, totalFiles int, bytesDone, totalBytes int64, startTime time.Time, +) { elapsed := time.Since(startTime) - pct := float64(bytesDone) / float64(totalBytes) * 100 + pct := float64(bytesDone) / float64(totalBytes) * percentScale byteRate := float64(bytesDone) / elapsed.Seconds() fileRate := float64(filesDone) / elapsed.Seconds() @@ -382,7 +513,9 @@ func (v *Vaultik) printRestoreProgress(filesDone, totalFiles int, bytesDone, tot } if eta > 0 { - v.UI.Progress("Restore: %s/%s files (%s), %s/%s, %s, %.0f files/sec, restore elapsed: %s, restore ETA: %s (est remain %s).", + v.UI.Progressf("%s: %s/%s files (%s), %s/%s, %s, %.0f files/sec, "+ + "%s elapsed: %s, %s ETA: %s (est remain %s).", + title, v.UI.Count(filesDone), v.UI.Count(totalFiles), v.UI.Percent(pct), @@ -390,14 +523,18 @@ func (v *Vaultik) printRestoreProgress(filesDone, totalFiles int, bytesDone, tot v.UI.Size(totalBytes), v.UI.Speed(byteRate), fileRate, + phase, v.UI.Duration(elapsed), + phase, v.UI.Time(time.Now().Add(eta)), v.UI.Duration(eta)) return } - v.UI.Progress("Restore: %s/%s files (%s), %s/%s, %s, %.0f files/sec, restore elapsed: %s.", + v.UI.Progressf("%s: %s/%s files (%s), %s/%s, %s, %.0f files/sec, "+ + "%s elapsed: %s.", + title, v.UI.Count(filesDone), v.UI.Count(totalFiles), v.UI.Percent(pct), @@ -405,6 +542,7 @@ func (v *Vaultik) printRestoreProgress(filesDone, totalFiles int, bytesDone, tot v.UI.Size(totalBytes), v.UI.Speed(byteRate), fileRate, + phase, v.UI.Duration(elapsed)) } @@ -421,17 +559,17 @@ func (v *Vaultik) handleRestoreVerification( } if result.FilesFailed > 0 { - v.UI.Error("Verification failed: %s files did not match expected checksums.", + v.UI.Errorf("Verification failed: %s files did not match expected checksums.", v.UI.Count(result.FilesFailed)) for _, path := range result.FailedFiles { - v.UI.Detail("%s", v.UI.Path(path)) + v.UI.Detailf("%s", v.UI.Path(path)) } - return fmt.Errorf("%d files failed verification", result.FilesFailed) + return fmt.Errorf("%d %w", result.FilesFailed, errFilesFailedVerify) } - v.UI.Complete("Verified %s files (%s).", + v.UI.Completef("Verified %s files (%s).", v.UI.Count(result.FilesVerified), v.UI.Size(result.BytesVerified)) @@ -441,9 +579,12 @@ func (v *Vaultik) handleRestoreVerification( // downloadSnapshotDB downloads and decrypts the snapshot metadata // database. The snapshotID is the human ID; we hash it to the remote // key for the storage path. -func (v *Vaultik) downloadSnapshotDB(snapshotID string, identity age.Identity) (*database.DB, error) { +func (v *Vaultik) downloadSnapshotDB( + snapshotID string, identity age.Identity, +) (*database.DB, error) { // Download encrypted database from storage - dbKey := fmt.Sprintf("metadata/%s/db.zst.age", snapshot.RemoteSnapshotKey(snapshotID)) + dbKey := fmt.Sprintf("metadata/%s/db.zst.age", + snapshot.RemoteSnapshotKey(snapshotID)) reader, err := v.Storage.Get(v.ctx, dbKey) if err != nil { @@ -457,7 +598,8 @@ func (v *Vaultik) downloadSnapshotDB(snapshotID string, identity age.Identity) ( return nil, fmt.Errorf("reading encrypted data: %w", err) } - log.Debug("Downloaded encrypted database", "size", humanize.Bytes(uint64(len(encryptedData)))) + log.Debug("Downloaded encrypted database", + "size", ubytes(int64(len(encryptedData)))) // Decrypt and decompress using blobgen.Reader blobReader, err := blobgen.NewReader(bytes.NewReader(encryptedData), identity) @@ -472,7 +614,7 @@ func (v *Vaultik) downloadSnapshotDB(snapshotID string, identity age.Identity) ( return nil, fmt.Errorf("decrypting and decompressing: %w", err) } - log.Debug("Decrypted database", "size", humanize.Bytes(uint64(len(dbData)))) + log.Debug("Decrypted database", "size", ubytes(int64(len(dbData)))) // Create a temporary database file and write the binary SQLite data directly tempFile, err := afero.TempFile(v.Fs, "", "vaultik-restore-*.db") @@ -510,7 +652,9 @@ func (v *Vaultik) downloadSnapshotDB(snapshotID string, identity age.Identity) ( } // getFilesToRestore returns the list of files to restore based on path filters -func (v *Vaultik) getFilesToRestore(ctx context.Context, repos *database.Repositories, pathFilters []string) ([]*database.File, error) { +func (v *Vaultik) getFilesToRestore( + ctx context.Context, repos *database.Repositories, pathFilters []string, +) ([]*database.File, error) { // If no filters, get all files if len(pathFilters) == 0 { return repos.Files.ListAll(ctx) @@ -543,7 +687,9 @@ func (v *Vaultik) getFilesToRestore(ctx context.Context, repos *database.Reposit } // buildChunkToBlobMap creates a mapping from chunk hash to blob information -func (v *Vaultik) buildChunkToBlobMap(ctx context.Context, repos *database.Repositories) (map[string]*database.BlobChunk, error) { +func (v *Vaultik) buildChunkToBlobMap( + ctx context.Context, repos *database.Repositories, +) (map[string]*database.BlobChunk, error) { // Query all blob_chunks query := `SELECT blob_id, chunk_hash, offset, length FROM blob_chunks` @@ -588,7 +734,7 @@ func (v *Vaultik) buildChunkToBlobMap(ctx context.Context, repos *database.Repos // readable: restoreFile(file) instead of a ten-argument helper. type restoreSession struct { v *Vaultik - ctx context.Context + ctx context.Context //nolint:containedctx // per-restore state by design repos *database.Repositories opts *RestoreOptions identity age.Identity @@ -613,7 +759,7 @@ func (s *restoreSession) restoreFile(file *database.File) error { parentDir := filepath.Dir(targetPath) - err := s.v.Fs.MkdirAll(parentDir, 0755) + err := s.v.Fs.MkdirAll(parentDir, restoreDirMode) if err != nil { return fmt.Errorf("creating parent directory: %w", err) } @@ -640,7 +786,8 @@ func (s *restoreSession) restoreSymlink(file *database.File, targetPath string) return fmt.Errorf("creating symlink: %w", err) } } else { - log.Debug("Symlink creation not supported on this filesystem", "path", file.Path, "target", file.LinkTarget) + log.Debug("Symlink creation not supported on this filesystem", + "path", file.Path, "target", file.LinkTarget) } s.result.FilesRestored++ @@ -652,34 +799,51 @@ func (s *restoreSession) restoreSymlink(file *database.File, targetPath string) // restoreDirectory restores a directory with its permissions, mtime, // and (on real filesystems, with sufficient privileges) ownership. -func (s *restoreSession) restoreDirectory(file *database.File, targetPath string) error { +func (s *restoreSession) restoreDirectory( + file *database.File, targetPath string, +) error { err := s.v.Fs.MkdirAll(targetPath, os.FileMode(file.Mode)) if err != nil { return fmt.Errorf("creating directory: %w", err) } - err = s.v.Fs.Chmod(targetPath, os.FileMode(file.Mode)) + s.applyFileMetadata(file, targetPath) + + s.result.FilesRestored++ + + return nil +} + +// applyFileMetadata applies stored permissions, ownership (when running +// as root on a real filesystem), and mtime to a restored path. Failures +// are logged at debug level and do not abort the restore. +func (s *restoreSession) applyFileMetadata(file *database.File, targetPath string) { + err := s.v.Fs.Chmod(targetPath, os.FileMode(file.Mode)) if err != nil { - log.Debug("Failed to set directory permissions", "path", targetPath, "error", err) + log.Debug("Failed to set permissions", "path", targetPath, "error", err) } if s.runningAsRoot { if _, ok := s.v.Fs.(*afero.OsFs); ok { - err := os.Chown(targetPath, int(file.UID), int(file.GID)) + err = os.Chown(targetPath, int(file.UID), int(file.GID)) if err != nil { - log.Debug("Failed to set directory ownership", "path", targetPath, "error", err) + log.Debug("Failed to set ownership", "path", targetPath, "error", err) } } } err = s.v.Fs.Chtimes(targetPath, file.MTime, file.MTime) if err != nil { - log.Debug("Failed to set directory mtime", "path", targetPath, "error", err) + log.Debug("Failed to set mtime", "path", targetPath, "error", err) } +} - s.result.FilesRestored++ - - return nil +// chunkWriteTimings accumulates per-phase durations while writing a +// file's chunks out of the blob cache. Debug instrumentation only. +type chunkWriteTimings struct { + readAt time.Duration + write time.Duration + sweeper time.Duration } // restoreRegularFile reconstructs a regular file by reading chunks @@ -687,7 +851,9 @@ func (s *restoreSession) restoreDirectory(file *database.File, targetPath string // method runs is that every blob this file needs is already in the // disk cache — the planner guarantees that by only marking files // "ready" once their full blob set is on disk. -func (s *restoreSession) restoreRegularFile(file *database.File, targetPath string) error { +func (s *restoreSession) restoreRegularFile( + file *database.File, targetPath string, +) error { fileStart := time.Now() t0 := time.Now() @@ -709,49 +875,9 @@ func (s *restoreSession) restoreRegularFile(file *database.File, targetPath stri defer func() { _ = outFile.Close() }() - var ( - readAtDur time.Duration - writeDur time.Duration - sweeperDur time.Duration - bytesWritten int64 - ) - - for _, fc := range fileChunks { - chunkHashStr := fc.ChunkHash.String() - - blobChunk, ok := s.chunkToBlobMap[chunkHashStr] - if !ok { - return fmt.Errorf("chunk %s not found in any blob", chunkHashStr[:16]) - } - - blobHash, ok := s.blobIDToHash[blobChunk.BlobID.String()] - if !ok { - return fmt.Errorf("blob id %s missing from hash index", blobChunk.BlobID) - } - - t0 = time.Now() - chunkData, err := s.blobCache.ReadAt(blobHash, blobChunk.Offset, blobChunk.Length) - readAtDur += time.Since(t0) - - if err != nil { - return fmt.Errorf("reading chunk %s from cached blob %s: %w", fc.ChunkHash[:16], blobHash[:16], err) - } - - t0 = time.Now() - n, err := outFile.Write(chunkData) - writeDur += time.Since(t0) - - if err != nil { - return fmt.Errorf("writing chunk: %w", err) - } - - bytesWritten += int64(n) - - t0 = time.Now() - - s.sweeper.chunkRestored(int64(n)) - - sweeperDur += time.Since(t0) + bytesWritten, timings, err := s.writeFileChunks(outFile, fileChunks) + if err != nil { + return err } log.Debug("Restored regular file (timings)", @@ -761,9 +887,9 @@ func (s *restoreSession) restoreRegularFile(file *database.File, targetPath stri "ms_total", time.Since(fileStart).Milliseconds(), "ms_file_chunks_query", fileChunksQueryDur.Milliseconds(), "ms_create", createDur.Milliseconds(), - "ms_readat", readAtDur.Milliseconds(), - "ms_writes", writeDur.Milliseconds(), - "ms_sweeper", sweeperDur.Milliseconds(), + "ms_readat", timings.readAt.Milliseconds(), + "ms_writes", timings.write.Milliseconds(), + "ms_sweeper", timings.sweeper.Milliseconds(), ) err = outFile.Close() @@ -771,40 +897,82 @@ func (s *restoreSession) restoreRegularFile(file *database.File, targetPath stri return fmt.Errorf("closing output file: %w", err) } - err = s.v.Fs.Chmod(targetPath, os.FileMode(file.Mode)) - if err != nil { - log.Debug("Failed to set file permissions", "path", targetPath, "error", err) - } - - if s.runningAsRoot { - if _, ok := s.v.Fs.(*afero.OsFs); ok { - err := os.Chown(targetPath, int(file.UID), int(file.GID)) - if err != nil { - log.Debug("Failed to set file ownership", "path", targetPath, "error", err) - } - } - } - - err = s.v.Fs.Chtimes(targetPath, file.MTime, file.MTime) - if err != nil { - log.Debug("Failed to set file mtime", "path", targetPath, "error", err) - } + s.applyFileMetadata(file, targetPath) s.result.FilesRestored++ s.result.BytesRestored += bytesWritten - log.Debug("Restored file", "path", file.Path, "size", humanize.Bytes(uint64(bytesWritten))) + log.Debug("Restored file", "path", file.Path, "size", ubytes(bytesWritten)) return nil } +// writeFileChunks streams each of the file's chunks from the blob disk +// cache into outFile, crediting restored bytes to the sweeper as it +// goes. Returns the bytes written plus per-phase timing accumulators. +func (s *restoreSession) writeFileChunks( + outFile afero.File, fileChunks []*database.FileChunk, +) (int64, chunkWriteTimings, error) { + var ( + timings chunkWriteTimings + bytesWritten int64 + ) + + for _, fc := range fileChunks { + chunkHashStr := fc.ChunkHash.String() + + blobChunk, ok := s.chunkToBlobMap[chunkHashStr] + if !ok { + return bytesWritten, timings, fmt.Errorf( + "%w: %s", errChunkNotInAnyBlob, chunkHashStr[:16]) + } + + blobHash, ok := s.blobIDToHash[blobChunk.BlobID.String()] + if !ok { + return bytesWritten, timings, fmt.Errorf( + "%w: %s", errBlobIDNotInHashIndex, blobChunk.BlobID) + } + + t0 := time.Now() + chunkData, err := s.blobCache.ReadAt( + blobHash, blobChunk.Offset, blobChunk.Length) + timings.readAt += time.Since(t0) + + if err != nil { + return bytesWritten, timings, fmt.Errorf( + "reading chunk %s from cached blob %s: %w", + fc.ChunkHash[:16], blobHash[:16], err) + } + + t0 = time.Now() + n, err := outFile.Write(chunkData) + timings.write += time.Since(t0) + + if err != nil { + return bytesWritten, timings, fmt.Errorf("writing chunk: %w", err) + } + + bytesWritten += int64(n) + + t0 = time.Now() + + s.sweeper.chunkRestored(int64(n)) + + timings.sweeper += time.Since(t0) + } + + return bytesWritten, timings, nil +} + // downloadBlobToCache streams a blob from remote storage straight into // the disk cache, decrypting and decompressing on the fly. The // plaintext never lives fully in memory — io.Copy through // blobDiskCache.PutFromReader uses a 32 KiB buffer regardless of blob // size, which is what makes multi-GB blobs tractable on machines with // less RAM than the blob. -func (s *restoreSession) downloadBlobToCache(blobHash string, expectedSize int64) error { +func (s *restoreSession) downloadBlobToCache( + blobHash string, expectedSize int64, +) error { start := time.Now() t0 := time.Now() @@ -840,7 +1008,8 @@ func (s *restoreSession) downloadBlobToCache(blobHash string, expectedSize int64 return nil } -// verifyRestoredFiles verifies that all restored files match their expected chunk hashes +// verifyRestoredFiles verifies that all restored files match their +// expected chunk hashes. func (v *Vaultik) verifyRestoredFiles( ctx context.Context, repos *database.Repositories, @@ -870,17 +1039,15 @@ func (v *Vaultik) verifyRestoredFiles( log.Info("Verifying restored files", "files", len(regularFiles), - "bytes", humanize.Bytes(uint64(totalBytes)), + "bytes", ubytes(totalBytes), ) - v.UI.Begin("Verifying %s files (%s).", + v.UI.Beginf("Verifying %s files (%s).", v.UI.Count(len(regularFiles)), v.UI.Size(totalBytes)) startTime := time.Now() lastStatusTime := startTime - const statusInterval = 15 * time.Second - var bytesProcessed int64 for i, file := range regularFiles { @@ -903,8 +1070,9 @@ func (v *Vaultik) verifyRestoredFiles( bytesProcessed += file.Size - if time.Since(lastStatusTime) >= statusInterval { - v.printVerifyProgress(i+1, len(regularFiles), bytesProcessed, totalBytes, startTime) + if time.Since(lastStatusTime) >= restoreStatusInterval { + v.printVerifyProgress( + i+1, len(regularFiles), bytesProcessed, totalBytes, startTime) lastStatusTime = time.Now() } @@ -912,7 +1080,7 @@ func (v *Vaultik) verifyRestoredFiles( log.Info("Verification complete", "files_verified", result.FilesVerified, - "bytes_verified", humanize.Bytes(uint64(result.BytesVerified)), + "bytes_verified", ubytes(result.BytesVerified), "files_failed", result.FilesFailed, ) @@ -922,44 +1090,11 @@ func (v *Vaultik) verifyRestoredFiles( // printVerifyProgress emits a periodic verify-phase status line. Same // shape as the restore progress line so user-facing pacing is uniform // across the two phases. -func (v *Vaultik) printVerifyProgress(filesDone, totalFiles int, bytesDone, totalBytes int64, startTime time.Time) { - elapsed := time.Since(startTime) - pct := float64(bytesDone) / float64(totalBytes) * 100 - byteRate := float64(bytesDone) / elapsed.Seconds() - fileRate := float64(filesDone) / elapsed.Seconds() - - remainingBytes := totalBytes - bytesDone - - var eta time.Duration - if byteRate > 0 && remainingBytes > 0 { - eta = time.Duration(float64(remainingBytes)/byteRate) * time.Second - } - - if eta > 0 { - v.UI.Progress("Verify: %s/%s files (%s), %s/%s, %s, %.0f files/sec, verify elapsed: %s, verify ETA: %s (est remain %s).", - v.UI.Count(filesDone), - v.UI.Count(totalFiles), - v.UI.Percent(pct), - v.UI.Size(bytesDone), - v.UI.Size(totalBytes), - v.UI.Speed(byteRate), - fileRate, - v.UI.Duration(elapsed), - v.UI.Time(time.Now().Add(eta)), - v.UI.Duration(eta)) - - return - } - - v.UI.Progress("Verify: %s/%s files (%s), %s/%s, %s, %.0f files/sec, verify elapsed: %s.", - v.UI.Count(filesDone), - v.UI.Count(totalFiles), - v.UI.Percent(pct), - v.UI.Size(bytesDone), - v.UI.Size(totalBytes), - v.UI.Speed(byteRate), - fileRate, - v.UI.Duration(elapsed)) +func (v *Vaultik) printVerifyProgress( + filesDone, totalFiles int, bytesDone, totalBytes int64, startTime time.Time, +) { + v.printPhaseProgress("Verify", "verify", + filesDone, totalFiles, bytesDone, totalBytes, startTime) } // verifyFile verifies a single restored file by checking its chunk hashes @@ -989,7 +1124,8 @@ func (v *Vaultik) verifyFile( // Get chunk size from database chunk, err := repos.Chunks.GetByHash(ctx, fc.ChunkHash.String()) if err != nil { - return bytesVerified, fmt.Errorf("getting chunk %s: %w", fc.ChunkHash.String()[:16], err) + return bytesVerified, fmt.Errorf("getting chunk %s: %w", + fc.ChunkHash.String()[:16], err) } // Read chunk data from file @@ -1001,7 +1137,8 @@ func (v *Vaultik) verifyFile( } if int64(n) != chunk.Size { - return bytesVerified, fmt.Errorf("short read: expected %d bytes, got %d", chunk.Size, n) + return bytesVerified, fmt.Errorf("%w: expected %d bytes, got %d", + errShortChunkRead, chunk.Size, n) } // Calculate hash and compare @@ -1010,14 +1147,15 @@ func (v *Vaultik) verifyFile( expectedHash := fc.ChunkHash.String() if actualHash != expectedHash { - return bytesVerified, fmt.Errorf("chunk %d hash mismatch: expected %s, got %s", - fc.Idx, expectedHash[:16], actualHash[:16]) + return bytesVerified, fmt.Errorf("%w: chunk %d: expected %s, got %s", + errChunkHashMismatch, fc.Idx, expectedHash[:16], actualHash[:16]) } bytesVerified += int64(n) } - log.Debug("File verified", "path", file.Path, "bytes", bytesVerified, "chunks", len(fileChunks)) + log.Debug("File verified", + "path", file.Path, "bytes", bytesVerified, "chunks", len(fileChunks)) return bytesVerified, nil } diff --git a/internal/vaultik/restore_locality_test.go b/internal/vaultik/restore_locality_test.go index 0e7c52d..67b1310 100644 --- a/internal/vaultik/restore_locality_test.go +++ b/internal/vaultik/restore_locality_test.go @@ -1,4 +1,4 @@ -package vaultik +package vaultik //nolint:testpackage // inspects unexported cache internals import ( "bytes" @@ -50,33 +50,28 @@ import ( // path-ordered names that interleave the blobs (a1, b1, c1, a2, b2, // c2, …) so naive path-order processing would touch every blob before // finishing any of them. -func TestRestoreLocalityAndReadAt(t *testing.T) { - log.Initialize(log.Config{}) +// localitySource is one 1 MiB fixture file used by the locality test. +type localitySource struct { + path string + data []byte +} - fs := afero.NewOsFs() - tempDir, err := os.MkdirTemp("", "vaultik-locality-") - require.NoError(t, err) +// localityCopy is a byte-for-byte clone of one fixture source with an +// interleaved name. +type localityCopy struct { + path string + data []byte +} - defer func() { _ = os.RemoveAll(tempDir) }() +// buildLocalityFixture writes the adversarial source layout described +// in TestRestoreLocalityAndReadAt: 15 sources of 1 MiB each (which the +// backup packs into 3 blobs of 5 chunks) plus 9 interleaved-name copies +// (3 per blob group). +func buildLocalityFixture( + t *testing.T, fs afero.Fs, dataDir string, +) ([]*localitySource, []localityCopy) { + t.Helper() - dataDir := filepath.Join(tempDir, "source") - storeDir := filepath.Join(tempDir, "remote") - restoreDir := filepath.Join(tempDir, "restored") - dbPath := filepath.Join(tempDir, "index.sqlite") - - require.NoError(t, fs.MkdirAll(dataDir, 0o755)) - - // Layout: 15 source files of exactly 1 MiB each. With - // chunkSize (avg) = 4 MiB the chunker's minSize is 1 MiB, so any - // file of 1 MiB becomes a single chunk. With a 5 MiB blob limit - // the packer fits exactly 5 chunks per blob, producing 3 blobs - // containing src-001..005, src-006..010, src-011..015. - // - // Then add 9 "copy" files — byte-for-byte clones of three of the - // sources (one from each blob group) — with interleaved names - // (cp-001-A, cp-002-B, cp-003-C, cp-004-A, …) so a naive - // path-ordered restore would touch all three blobs before - // finishing any of them. const ( srcBytes = 1024 * 1024 srcCount = 15 @@ -84,35 +79,24 @@ func TestRestoreLocalityAndReadAt(t *testing.T) { perBlob = srcCount / blobsCount ) - type source struct { - path string - data []byte - } - - sources := make([]*source, srcCount) + sources := make([]*localitySource, srcCount) for i := range srcCount { - s := &source{ + s := &localitySource{ path: fmt.Sprintf("src-%03d.bin", i+1), data: randomBytes(t, srcBytes), } sources[i] = s - require.NoError(t, afero.WriteFile(fs, filepath.Join(dataDir, s.path), s.data, 0o644)) + require.NoError(t, afero.WriteFile( + fs, filepath.Join(dataDir, s.path), s.data, 0o644)) } // Pick one representative source per blob group (src-001 → blob // 1, src-006 → blob 2, src-011 → blob 3) and create 3 copies of // each with interleaved alphabetical names. - type copyFile struct { - path string - data []byte - sourceBlob int // 0, 1, or 2 - sourceIndex int // index into sources slice - } - groupReps := []int{0, perBlob, 2 * perBlob} // 0, 5, 10 letters := []byte{'A', 'B', 'C'} - var copies []copyFile + copies := make([]localityCopy, 0, blobsCount*3) for i := range 3 { for j := range blobsCount { @@ -121,10 +105,24 @@ func TestRestoreLocalityAndReadAt(t *testing.T) { path := filepath.Join(dataDir, name) src := sources[groupReps[j]] require.NoError(t, afero.WriteFile(fs, path, src.data, 0o644)) - copies = append(copies, copyFile{path: path, data: src.data, sourceBlob: j, sourceIndex: groupReps[j]}) + + copies = append(copies, localityCopy{path: path, data: src.data}) } } + return sources, copies +} + +// setupLocalityBackup runs the backup half of the locality test: a +// snapshot of dataDir into a file storer with sizes tuned so 15 one-chunk +// files pack into 3 blobs. The index database is closed before returning +// so the restore half runs from remote bytes only. +func setupLocalityBackup( + ctx context.Context, t *testing.T, fs afero.Fs, + dataDir, storeDir, dbPath string, +) (*config.Config, *storage.FileStorer, string) { + t.Helper() + // chunkSize avg = 4 MiB makes minSize = 1 MiB, so a 1 MiB file // becomes one chunk. maxBlobSize = 5 MiB packs exactly 5 chunks // per blob, yielding 3 blobs from 15 source files. @@ -134,8 +132,10 @@ func TestRestoreLocalityAndReadAt(t *testing.T) { storer, err := storage.NewFileStorer(storeDir) require.NoError(t, err) - agePublicKey := "age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05gl0sjq9q9wjg" - ageSecretKey := "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5" + agePublicKey := "age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05g" + + "l0sjq9q9wjg" + ageSecretKey := "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKU" + + "T68TXSFPK7APHXA2QS2NJA5" cfg := &config.Config{ AgeRecipients: []string{agePublicKey}, @@ -145,8 +145,6 @@ func TestRestoreLocalityAndReadAt(t *testing.T) { BlobSizeLimit: config.Size(maxBlobSize), } - ctx := context.Background() - db, err := database.New(ctx, dbPath) require.NoError(t, err) @@ -171,7 +169,8 @@ func TestRestoreLocalityAndReadAt(t *testing.T) { Repositories: repos, }) - snapshotID, err := sm.CreateSnapshotWithName(ctx, cfg.Hostname, "locality", "test-version", "test-git") + snapshotID, err := sm.CreateSnapshotWithName( + ctx, cfg.Hostname, "locality", "test-version", "test-git") require.NoError(t, err) _, err = scanner.Scan(ctx, dataDir, snapshotID) @@ -182,10 +181,35 @@ func TestRestoreLocalityAndReadAt(t *testing.T) { blobsOnDisk := listBlobKeys(t, storeDir) t.Logf("backup produced %d blobs", len(blobsOnDisk)) - require.GreaterOrEqual(t, len(blobsOnDisk), 3, "expected at least 3 blobs from 3 filler groups") + require.GreaterOrEqual(t, len(blobsOnDisk), 3, + "expected at least 3 blobs from 3 filler groups") require.NoError(t, db.Close()) + return cfg, storer, snapshotID +} + +func TestRestoreLocalityAndReadAt(t *testing.T) { + log.Initialize(log.Config{}) + t.Parallel() + + fs := afero.NewOsFs() + tempDir := t.TempDir() + + dataDir := filepath.Join(tempDir, "source") + storeDir := filepath.Join(tempDir, "remote") + restoreDir := filepath.Join(tempDir, "restored") + dbPath := filepath.Join(tempDir, "index.sqlite") + + require.NoError(t, fs.MkdirAll(dataDir, 0o755)) + + sources, copies := buildLocalityFixture(t, fs, dataDir) + + ctx := context.Background() + + cfg, storer, snapshotID := setupLocalityBackup( + ctx, t, fs, dataDir, storeDir, dbPath) + // Wrap the storer so we can count downloads per blob key. counter := newCountingStorer(storer) @@ -215,20 +239,7 @@ func TestRestoreLocalityAndReadAt(t *testing.T) { require.NotNil(t, cacheRef, "restoreCacheObserver must fire during restore") - // Verify restored content matches. - for _, s := range sources { - restored := filepath.Join(restoreDir, dataDir, s.path) - got, err := afero.ReadFile(fs, restored) - require.NoErrorf(t, err, "source missing after restore: %s", s.path) - require.Truef(t, bytes.Equal(got, s.data), "byte mismatch for source %s", s.path) - } - - for _, c := range copies { - restored := filepath.Join(restoreDir, c.path) - got, err := afero.ReadFile(fs, restored) - require.NoErrorf(t, err, "copy missing after restore: %s", c.path) - require.Truef(t, bytes.Equal(got, c.data), "byte mismatch for copy %s", c.path) - } + verifyLocalityRestore(t, fs, restoreDir, dataDir, sources, copies) // (1) Each blob fetched exactly once. for key, n := range counter.snapshot() { @@ -242,18 +253,48 @@ func TestRestoreLocalityAndReadAt(t *testing.T) { // (2) Peak cache size ≤ 1. The sweeper plus locality-aware // ordering should free each blob before the next one downloads. assert.LessOrEqualf(t, cacheRef.PeakLen(), 1, - "peak cached blobs was %d; expected ≤ 1 with locality-ordered restore", cacheRef.PeakLen()) + "peak cached blobs was %d; expected ≤ 1 with locality-ordered restore", + cacheRef.PeakLen()) // (3) Cache.Get must never be called during restore — chunk // extraction has to go through ReadAt so we never read the whole // blob from disk to grab a few KB slice. assert.Equalf(t, 0, cacheRef.GetCalls(), - "blobDiskCache.Get was called %d times during restore; restore must use ReadAt exclusively", cacheRef.GetCalls()) + "blobDiskCache.Get was called %d times during restore; "+ + "restore must use ReadAt exclusively", cacheRef.GetCalls()) t.Logf("blob cache stats: peak_len=%d get_calls=%d readat_calls=%d", cacheRef.PeakLen(), cacheRef.GetCalls(), cacheRef.ReadAtCalls()) } +// verifyLocalityRestore byte-compares every restored source and copy +// against its original content. +func verifyLocalityRestore( + t *testing.T, + fs afero.Fs, + restoreDir, dataDir string, + sources []*localitySource, + copies []localityCopy, +) { + t.Helper() + + for _, s := range sources { + restored := filepath.Join(restoreDir, dataDir, s.path) + got, err := afero.ReadFile(fs, restored) + require.NoErrorf(t, err, "source missing after restore: %s", s.path) + require.Truef(t, bytes.Equal(got, s.data), + "byte mismatch for source %s", s.path) + } + + for _, c := range copies { + restored := filepath.Join(restoreDir, c.path) + got, err := afero.ReadFile(fs, restored) + require.NoErrorf(t, err, "copy missing after restore: %s", c.path) + require.Truef(t, bytes.Equal(got, c.data), + "byte mismatch for copy %s", c.path) + } +} + // randomBytes returns n bytes of random data. Used to make sure the // chunker picks non-degenerate FastCDC boundaries. func randomBytes(t *testing.T, n int) []byte { @@ -316,7 +357,9 @@ func newCountingStorer(inner storage.Storer) *countingStorerInternal { return &countingStorerInternal{Storer: inner, counts: make(map[string]int)} } -func (c *countingStorerInternal) Get(ctx context.Context, key string) (io.ReadCloser, error) { +func (c *countingStorerInternal) Get( + ctx context.Context, key string, +) (io.ReadCloser, error) { c.mu.Lock() c.counts[key]++ c.mu.Unlock() diff --git a/internal/vaultik/restore_plan.go b/internal/vaultik/restore_plan.go index c6cf5cb..e40afde 100644 --- a/internal/vaultik/restore_plan.go +++ b/internal/vaultik/restore_plan.go @@ -2,6 +2,7 @@ package vaultik import ( "context" + "errors" "fmt" "math" "os" @@ -10,6 +11,12 @@ import ( "sneak.berlin/go/vaultik/internal/types" ) +// Sentinel errors for restore planning index lookups. +var ( + errPlanChunkMissing = errors.New("chunk missing from blob map") + errPlanBlobIDMissing = errors.New("blob id missing from id-to-hash map") +) + // restorePlan orders restore-time file processing by blob locality. The // goal is to keep the blob disk cache occupancy as small as possible: // download one blob, drain every file referencing only that blob, let @@ -67,14 +74,14 @@ func newRestorePlan( for _, fc := range fileChunks { bc, ok := chunkToBlobMap[fc.ChunkHash.String()] if !ok { - return nil, fmt.Errorf("planning %s: chunk %s missing from blob map", - f.Path, fc.ChunkHash.String()[:16]) + return nil, fmt.Errorf("planning %s: %w: %s", + f.Path, errPlanChunkMissing, fc.ChunkHash.String()[:16]) } hash, ok := blobIDToHash[bc.BlobID.String()] if !ok { - return nil, fmt.Errorf("planning %s: blob id %s missing from id-to-hash map", - f.Path, bc.BlobID) + return nil, fmt.Errorf("planning %s: %w: %s", + f.Path, errPlanBlobIDMissing, bc.BlobID) } blobs[hash] = struct{}{} diff --git a/internal/vaultik/restore_sweeper.go b/internal/vaultik/restore_sweeper.go index 9382dcb..e363d60 100644 --- a/internal/vaultik/restore_sweeper.go +++ b/internal/vaultik/restore_sweeper.go @@ -28,7 +28,7 @@ import ( // which is local, indexed, and not under contention — the queries are // cheap and run at most once per blob per sweep interval. type restoreSweeper struct { - ctx context.Context + ctx context.Context //nolint:containedctx // ctx bound at construction by design repos *database.Repositories cache *blobDiskCache threshold int64 @@ -38,7 +38,12 @@ type restoreSweeper struct { // newRestoreSweeper returns a sweeper that triggers eviction every // `threshold` bytes restored. Callers should pass blob_size_limit/100. -func newRestoreSweeper(ctx context.Context, repos *database.Repositories, cache *blobDiskCache, threshold int64) *restoreSweeper { +func newRestoreSweeper( + ctx context.Context, + repos *database.Repositories, + cache *blobDiskCache, + threshold int64, +) *restoreSweeper { if threshold <= 0 { threshold = 1 } @@ -78,7 +83,8 @@ func (s *restoreSweeper) sweep() { for _, blobHash := range s.cache.Keys() { needed, err := s.blobStillNeeded(blobHash) if err != nil { - log.Debug("sweeper referencing-files query failed", "blob_hash", blobHash[:16], "error", err) + log.Debug("sweeper referencing-files query failed", + "blob_hash", blobHash[:16], "error", err) continue } diff --git a/internal/vaultik/restore_sweeper_integration_test.go b/internal/vaultik/restore_sweeper_integration_test.go index d5b21e0..13cc5cc 100644 --- a/internal/vaultik/restore_sweeper_integration_test.go +++ b/internal/vaultik/restore_sweeper_integration_test.go @@ -40,33 +40,20 @@ import ( // originals; the sweeper must keep each blob alive until BOTH the // original AND every duplicate referencing its chunks have been // restored. -func TestRestoreSweeperEvictsBlobs(t *testing.T) { - log.Initialize(log.Config{}) +// buildSweeperFixture writes the source layout for the sweeper test: 30 +// unique 1 MB random files plus 10 duplicates of a random subset. The +// PRNG seed is fixed so failures are reproducible; the entropy is what +// matters here — the FastCDC chunker needs realistic-looking data to +// pick chunk boundaries naturally. Returns the expected content by +// path. +func buildSweeperFixture( + t *testing.T, fs afero.Fs, dataDir string, uniqueFiles, duplicateFiles int, +) map[string][]byte { + t.Helper() - fs := afero.NewOsFs() - tempDir, err := os.MkdirTemp("", "vaultik-sweeper-") - require.NoError(t, err) + const fileSize = 1 * 1024 * 1024 - defer func() { _ = os.RemoveAll(tempDir) }() - - dataDir := filepath.Join(tempDir, "source") - storeDir := filepath.Join(tempDir, "remote") - restoreDir := filepath.Join(tempDir, "restored") - dbPath := filepath.Join(tempDir, "index.sqlite") - - require.NoError(t, fs.MkdirAll(dataDir, 0o755)) - - // Generate 30 unique 1 MB random files. The PRNG seed is fixed so - // failures are reproducible; the entropy is what matters here — the - // FastCDC chunker needs realistic-looking data to pick chunk - // boundaries naturally. - const ( - uniqueFiles = 30 - duplicateFiles = 10 - fileSize = 1 * 1024 * 1024 - ) - - rng := rand.New(rand.NewSource(42)) + rng := rand.New(rand.NewSource(42)) //nolint:gosec // G404: fixture data only type sourceFile struct { path string @@ -87,33 +74,58 @@ func TestRestoreSweeperEvictsBlobs(t *testing.T) { expected[path] = data } - // Pick 10 of the originals and copy each to a fresh path so the - // chunker dedups them against the originals' blobs. + // Copy a subset of the originals to fresh paths so the chunker + // dedups them against the originals' blobs. for i, idx := range rng.Perm(uniqueFiles)[:duplicateFiles] { src := uniques[idx] dstPath := filepath.Join(dataDir, fmt.Sprintf("dup-%02d.bin", i)) require.NoError(t, afero.WriteFile(fs, dstPath, src.data, 0o644)) + expected[dstPath] = src.data } - chunkSize := int64(64 * 1024) - maxBlobSize := int64(10 * 1024 * 1024) + return expected +} - storer, err := storage.NewFileStorer(storeDir) - require.NoError(t, err) +// verifySingleDownloadPerBlob asserts each blob on disk was fetched +// exactly once. >1 means the sweeper evicted a still-needed blob; 0 +// means the cache silently stopped being consulted. +func verifySingleDownloadPerBlob( + t *testing.T, counter *countingStorer, storeDir string, +) { + t.Helper() - agePublicKey := "age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05gl0sjq9q9wjg" - ageSecretKey := "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5" + blobDownloads := 0 - cfg := &config.Config{ - AgeRecipients: []string{agePublicKey}, - AgeSecretKey: ageSecretKey, - CompressionLevel: 3, - Hostname: "test-host", - BlobSizeLimit: config.Size(maxBlobSize), + for key, count := range counter.snapshot() { + if !strings.HasPrefix(key, "blobs/") { + continue + } + + assert.Equalf(t, 1, count, + "blob %s should have been downloaded exactly once during "+ + "restore, got %d", key, count) + + blobDownloads++ } - ctx := context.Background() + blobCount := countBlobsOnDisk(t, storeDir) + assert.Equal(t, blobCount, blobDownloads, + "every blob on disk should have been fetched exactly once during restore") + t.Logf("restore downloaded %d blobs, each exactly once", blobDownloads) +} + +// runSweeperBackup performs the backup half of the sweeper test and +// closes the index database so the restore half runs from remote bytes +// only. Returns the snapshot ID. +func runSweeperBackup( + ctx context.Context, t *testing.T, fs afero.Fs, + storer *storage.FileStorer, cfg *config.Config, + dataDir, storeDir, dbPath string, + chunkSize, maxBlobSize int64, + uniqueFiles, duplicateFiles int, +) string { + t.Helper() db, err := database.New(ctx, dbPath) require.NoError(t, err) @@ -139,13 +151,15 @@ func TestRestoreSweeperEvictsBlobs(t *testing.T) { Repositories: repos, }) - snapshotID, err := sm.CreateSnapshotWithName(ctx, cfg.Hostname, "sweeper", "test-version", "test-git") + snapshotID, err := sm.CreateSnapshotWithName( + ctx, cfg.Hostname, "sweeper", "test-version", "test-git") require.NoError(t, err) scanResult, err := scanner.Scan(ctx, dataDir, snapshotID) require.NoError(t, err) require.Equal(t, uniqueFiles+duplicateFiles, scanResult.FilesScanned) - require.Greater(t, scanResult.BlobsCreated, 1, "30 MB of unique data at 10 MB blob size should yield multiple blobs") + require.Greater(t, scanResult.BlobsCreated, 1, + "30 MB of unique data at 10 MB blob size should yield multiple blobs") require.NoError(t, sm.CompleteSnapshot(ctx, snapshotID)) require.NoError(t, sm.ExportSnapshotMetadata(ctx, dbPath, snapshotID)) @@ -161,6 +175,50 @@ func TestRestoreSweeperEvictsBlobs(t *testing.T) { // as a real restore on a fresh machine would. require.NoError(t, db.Close()) + return snapshotID +} + +func TestRestoreSweeperEvictsBlobs(t *testing.T) { + log.Initialize(log.Config{}) + t.Parallel() + + fs := afero.NewOsFs() + tempDir := t.TempDir() + + dataDir := filepath.Join(tempDir, "source") + storeDir := filepath.Join(tempDir, "remote") + restoreDir := filepath.Join(tempDir, "restored") + dbPath := filepath.Join(tempDir, "index.sqlite") + + require.NoError(t, fs.MkdirAll(dataDir, 0o755)) + + const ( + uniqueFiles = 30 + duplicateFiles = 10 + ) + + expected := buildSweeperFixture(t, fs, dataDir, uniqueFiles, duplicateFiles) + + chunkSize := int64(64 * 1024) + maxBlobSize := int64(10 * 1024 * 1024) + + storer, err := storage.NewFileStorer(storeDir) + require.NoError(t, err) + + cfg := &config.Config{ + AgeRecipients: []string{testAgePublicKey}, + AgeSecretKey: testAgeSecretKey, + CompressionLevel: 3, + Hostname: testHostname, + BlobSizeLimit: config.Size(maxBlobSize), + } + + ctx := context.Background() + + snapshotID := runSweeperBackup(ctx, t, fs, storer, cfg, + dataDir, storeDir, dbPath, chunkSize, maxBlobSize, + uniqueFiles, duplicateFiles) + counter := newCountingStorer(storer) restoreVaultik := &vaultik.Vaultik{ @@ -186,25 +244,7 @@ func TestRestoreSweeperEvictsBlobs(t *testing.T) { require.Equalf(t, want, got, "byte mismatch for %s", origPath) } - // Each blob must have been downloaded exactly once. >1 means the - // sweeper evicted a still-needed blob; 0 means the cache silently - // stopped being consulted. - blobDownloads := 0 - - for key, count := range counter.snapshot() { - if !strings.HasPrefix(key, "blobs/") { - continue - } - - assert.Equalf(t, 1, count, - "blob %s should have been downloaded exactly once during restore, got %d", key, count) - - blobDownloads++ - } - - assert.Equal(t, blobCount, blobDownloads, - "every blob on disk should have been fetched exactly once during restore") - t.Logf("restore downloaded %d blobs, each exactly once", blobDownloads) + verifySingleDownloadPerBlob(t, counter, storeDir) } // countingStorer wraps a Storer and records the number of Get calls per @@ -221,7 +261,9 @@ func newCountingStorer(inner storage.Storer) *countingStorer { return &countingStorer{Storer: inner, counts: make(map[string]int)} } -func (c *countingStorer) Get(ctx context.Context, key string) (io.ReadCloser, error) { +func (c *countingStorer) Get( + ctx context.Context, key string, +) (io.ReadCloser, error) { c.mu.Lock() c.counts[key]++ c.mu.Unlock() diff --git a/internal/vaultik/snapshot.go b/internal/vaultik/snapshot.go index a83f4e0..4e6b98c 100644 --- a/internal/vaultik/snapshot.go +++ b/internal/vaultik/snapshot.go @@ -12,17 +12,32 @@ import ( "text/tabwriter" "time" - "github.com/dustin/go-humanize" "sneak.berlin/go/vaultik/internal/database" "sneak.berlin/go/vaultik/internal/log" "sneak.berlin/go/vaultik/internal/snapshot" ) +// Sentinel errors for snapshot management. +var ( + errSnapshotNotInConfig = errors.New("snapshot not found in config") + errNoSnapshotsInConfig = errors.New("no snapshots configured") + errBlobsMissing = errors.New("blobs are missing") + errSnapshotVerifyFailed = errors.New("verification failed") + errRemoveAllNeedsForce = errors.New("--all requires --force") + errInvalidTableName = errors.New("invalid table name") +) + +// listRecentLimit caps how many snapshot rows are fetched from the +// local index for listing and purge operations. +const listRecentLimit = 10000 + // SnapshotCreateOptions contains options for the snapshot create command type SnapshotCreateOptions struct { - Cron bool - Prune bool - KeepNewerThan string // With --prune: keep snapshots newer than this duration (e.g. "4w"); default: keep only latest + Cron bool + Prune bool + // KeepNewerThan is used with --prune: keep snapshots newer than this + // duration (e.g. "4w"); default: keep only latest. + KeepNewerThan string SkipErrors bool // Skip file read errors (log them loudly but continue) Snapshots []string // Optional list of snapshot names to process (empty = all) } @@ -69,18 +84,19 @@ func (v *Vaultik) CreateSnapshot(opts *SnapshotCreateOptions) error { // Validate requested snapshot names exist for _, name := range snapshotNames { if _, ok := v.Config.Snapshots[name]; !ok { - return fmt.Errorf("snapshot %q not found in config", name) + return fmt.Errorf("%w: %q", errSnapshotNotInConfig, name) } } } if len(snapshotNames) == 0 { - return errors.New("no snapshots configured") + return errNoSnapshotsInConfig } // Process each named snapshot for snapIdx, snapName := range snapshotNames { - err := v.createNamedSnapshot(opts, hostname, snapName, snapIdx+1, len(snapshotNames)) + err = v.createNamedSnapshot( + opts, hostname, snapName, snapIdx+1, len(snapshotNames)) if err != nil { return err } @@ -88,11 +104,12 @@ func (v *Vaultik) CreateSnapshot(opts *SnapshotCreateOptions) error { // Print overall summary if multiple snapshots if len(snapshotNames) > 1 { - v.UI.Complete("All %d snapshots completed in %s.", len(snapshotNames), v.UI.Duration(time.Since(overallStartTime))) + v.UI.Completef("All %d snapshots completed in %s.", + len(snapshotNames), v.UI.Duration(time.Since(overallStartTime))) } if opts.Prune { - err := v.runPostBackupPrune(snapshotNames, opts.KeepNewerThan) + err = v.runPostBackupPrune(snapshotNames, opts.KeepNewerThan) if err != nil { return fmt.Errorf("post-backup prune: %w", err) } @@ -104,9 +121,10 @@ func (v *Vaultik) CreateSnapshot(opts *SnapshotCreateOptions) error { // emit the summary via UI.Warning so cron actually delivers // something for the user to look at. if v.UI.WarningCount() > 0 { - v.UI.Warning("Finished with %d warning(s) — review the output above.", v.UI.WarningCount()) + v.UI.Warningf("Finished with %d warning(s) — review the output above.", + v.UI.WarningCount()) } else { - v.UI.Complete("Finished successfully.") + v.UI.Completef("Finished successfully.") } return nil @@ -116,9 +134,12 @@ func (v *Vaultik) CreateSnapshot(opts *SnapshotCreateOptions) error { // orphan blobs from remote storage. If keepNewerThan is set (e.g. "4w"), // snapshots newer than that duration are kept. Otherwise only the latest // snapshot of each name is kept. -func (v *Vaultik) runPostBackupPrune(snapshotNames []string, keepNewerThan string) error { - log.Info("Running post-backup prune", "snapshots", snapshotNames, "keep_newer_than", keepNewerThan) - v.UI.Begin("Running post-backup prune.") +func (v *Vaultik) runPostBackupPrune( + snapshotNames []string, keepNewerThan string, +) error { + log.Info("Running post-backup prune", + "snapshots", snapshotNames, "keep_newer_than", keepNewerThan) + v.UI.Beginf("Running post-backup prune.") purgeOpts := &SnapshotPurgeOptions{ Force: true, @@ -161,11 +182,13 @@ type snapshotStats struct { } // createNamedSnapshot creates a single named snapshot -func (v *Vaultik) createNamedSnapshot(opts *SnapshotCreateOptions, hostname, snapName string, idx, total int) error { +func (v *Vaultik) createNamedSnapshot( + opts *SnapshotCreateOptions, hostname, snapName string, idx, total int, +) error { snapshotStartTime := time.Now() if total > 1 { - v.UI.Info("Snapshot %d/%d: %s.", idx, total, snapName) + v.UI.Infof("Snapshot %d/%d: %s.", idx, total, snapName) } resolvedDirs, err := v.resolveSnapshotPaths(snapName) @@ -181,13 +204,14 @@ func (v *Vaultik) createNamedSnapshot(opts *SnapshotCreateOptions, hostname, sna SkipErrors: opts.SkipErrors, }) - snapshotID, err := v.SnapshotManager.CreateSnapshotWithName(v.ctx, hostname, snapName, v.Globals.Version, v.Globals.Commit) + snapshotID, err := v.SnapshotManager.CreateSnapshotWithName( + v.ctx, hostname, snapName, v.Globals.Version, v.Globals.Commit) if err != nil { return fmt.Errorf("creating snapshot: %w", err) } log.Info("Beginning snapshot", "snapshot_id", snapshotID, "name", snapName) - v.UI.Begin("Creating snapshot %s.", v.UI.Snapshot(snapshotID)) + v.UI.Beginf("Creating snapshot %s.", v.UI.Snapshot(snapshotID)) stats, err := v.scanAllDirectories(scanner, resolvedDirs, snapshotID) if err != nil { @@ -214,7 +238,8 @@ func (v *Vaultik) createNamedSnapshot(opts *SnapshotCreateOptions, hostname, sna return nil } -// resolveSnapshotPaths resolves source directories to absolute paths with symlink resolution +// resolveSnapshotPaths resolves source directories to absolute paths +// with symlink resolution. func (v *Vaultik) resolveSnapshotPaths(snapName string) ([]string, error) { snapConfig := v.Config.Snapshots[snapName] resolvedDirs := make([]string, 0, len(snapConfig.Paths)) @@ -230,7 +255,8 @@ func (v *Vaultik) resolveSnapshotPaths(snapName string) ([]string, error) { if os.IsNotExist(err) { resolvedPath = absPath } else { - return nil, fmt.Errorf("failed to resolve symlinks for %s: %w", absPath, err) + return nil, fmt.Errorf( + "failed to resolve symlinks for %s: %w", absPath, err) } } @@ -240,8 +266,11 @@ func (v *Vaultik) resolveSnapshotPaths(snapName string) ([]string, error) { return resolvedDirs, nil } -// scanAllDirectories runs the scanner on each resolved directory and accumulates stats -func (v *Vaultik) scanAllDirectories(scanner *snapshot.Scanner, resolvedDirs []string, snapshotID string) (*snapshotStats, error) { +// scanAllDirectories runs the scanner on each resolved directory and +// accumulates stats. +func (v *Vaultik) scanAllDirectories( + scanner *snapshot.Scanner, resolvedDirs []string, snapshotID string, +) (*snapshotStats, error) { stats := &snapshotStats{} for i, dir := range resolvedDirs { @@ -254,7 +283,8 @@ func (v *Vaultik) scanAllDirectories(scanner *snapshot.Scanner, resolvedDirs []s } log.Info("Scanning directory", "path", dir) - v.UI.Begin("Enumerating snapshot source files in %s (%d of %d).", v.UI.Path(dir), i+1, len(resolvedDirs)) + v.UI.Beginf("Enumerating snapshot source files in %s (%d of %d).", + v.UI.Path(dir), i+1, len(resolvedDirs)) result, err := scanner.Scan(v.ctx, dir, snapshotID) if err != nil { @@ -284,18 +314,22 @@ func (v *Vaultik) scanAllDirectories(scanner *snapshot.Scanner, resolvedDirs []s return stats, nil } -// collectUploadStats gathers upload statistics from the scanner's progress reporter +// collectUploadStats gathers upload statistics from the scanner's +// progress reporter. func (v *Vaultik) collectUploadStats(scanner *snapshot.Scanner, stats *snapshotStats) { if s := scanner.GetProgress(); s != nil { progressStats := s.GetStats() stats.totalBytesUploaded = progressStats.BytesUploaded.Load() stats.totalBlobsUploaded = int(progressStats.BlobsUploaded.Load()) - stats.uploadDuration = time.Duration(progressStats.UploadDurationMs.Load()) * time.Millisecond + stats.uploadDuration = time.Duration( + progressStats.UploadDurationMs.Load()) * time.Millisecond } } // finalizeSnapshotMetadata updates stats, marks complete, and exports metadata -func (v *Vaultik) finalizeSnapshotMetadata(snapshotID string, stats *snapshotStats) error { +func (v *Vaultik) finalizeSnapshotMetadata( + snapshotID string, stats *snapshotStats, +) error { extStats := snapshot.ExtendedBackupStats{ BackupStats: snapshot.BackupStats{ FilesScanned: stats.totalFiles, @@ -319,7 +353,8 @@ func (v *Vaultik) finalizeSnapshotMetadata(snapshotID string, stats *snapshotSta return fmt.Errorf("completing snapshot: %w", err) } - err = v.SnapshotManager.ExportSnapshotMetadata(v.ctx, v.Config.IndexPath, snapshotID) + err = v.SnapshotManager.ExportSnapshotMetadata( + v.ctx, v.Config.IndexPath, snapshotID) if err != nil { return fmt.Errorf("exporting snapshot metadata: %w", err) } @@ -338,22 +373,24 @@ func (v *Vaultik) uploadSpeed(bytesUploaded int64, duration time.Duration) strin } // printSnapshotSummary prints the comprehensive snapshot completion summary -func (v *Vaultik) printSnapshotSummary(snapshotID string, startTime time.Time, stats *snapshotStats) { +func (v *Vaultik) printSnapshotSummary( + snapshotID string, startTime time.Time, stats *snapshotStats, +) { snapshotDuration := time.Since(startTime) totalFilesChanged := stats.totalFiles - stats.totalFilesSkipped totalBytesAll := stats.totalBytes + stats.totalBytesSkipped // Get total blob sizes from database - totalBlobSizeCompressed, totalBlobSizeUncompressed := v.getSnapshotBlobSizes(snapshotID) + compressedSize, uncompressedSize := v.getSnapshotBlobSizes(snapshotID) var compressionRatio float64 - if totalBlobSizeUncompressed > 0 { - compressionRatio = float64(totalBlobSizeCompressed) / float64(totalBlobSizeUncompressed) + if uncompressedSize > 0 { + compressionRatio = float64(compressedSize) / float64(uncompressedSize) } else { compressionRatio = 1.0 } - v.UI.Complete("Created snapshot %s.", v.UI.Snapshot(snapshotID)) + v.UI.Completef("Created snapshot %s.", v.UI.Snapshot(snapshotID)) filesMsg := fmt.Sprintf("Files: %s examined, %s backed up, %s unchanged", v.UI.Count(stats.totalFiles), @@ -363,7 +400,7 @@ func (v *Vaultik) printSnapshotSummary(snapshotID string, startTime time.Time, s filesMsg += fmt.Sprintf(", %s deleted", v.UI.Count(stats.totalFilesDeleted)) } - v.UI.Detail("%s.", filesMsg) + v.UI.Detailf("%s.", filesMsg) dataMsg := fmt.Sprintf("Data: %s total (%s backed up)", v.UI.Size(totalBytesAll), @@ -372,25 +409,28 @@ func (v *Vaultik) printSnapshotSummary(snapshotID string, startTime time.Time, s dataMsg += fmt.Sprintf(", %s deleted", v.UI.Size(stats.totalBytesDeleted)) } - v.UI.Detail("%s.", dataMsg) + v.UI.Detailf("%s.", dataMsg) if stats.totalBlobsUploaded > 0 { - v.UI.Detail("Storage: %s compressed from %s (%.2fx ratio).", - v.UI.Size(totalBlobSizeCompressed), - v.UI.Size(totalBlobSizeUncompressed), + v.UI.Detailf("Storage: %s compressed from %s (%.2fx ratio).", + v.UI.Size(compressedSize), + v.UI.Size(uncompressedSize), compressionRatio) - v.UI.Detail("Upload: %d blobs, %s in %s (%s).", + v.UI.Detailf("Upload: %d blobs, %s in %s (%s).", stats.totalBlobsUploaded, v.UI.Size(stats.totalBytesUploaded), v.UI.Duration(stats.uploadDuration), v.uploadSpeed(stats.totalBytesUploaded, stats.uploadDuration)) } - v.UI.Detail("Snapshot create duration: %s.", v.UI.Duration(snapshotDuration)) + v.UI.Detailf("Snapshot create duration: %s.", v.UI.Duration(snapshotDuration)) } -// getSnapshotBlobSizes returns total compressed and uncompressed blob sizes for a snapshot -func (v *Vaultik) getSnapshotBlobSizes(snapshotID string) (compressed int64, uncompressed int64) { +// getSnapshotBlobSizes returns total compressed and uncompressed blob +// sizes for a snapshot. +func (v *Vaultik) getSnapshotBlobSizes(snapshotID string) (int64, int64) { + var compressed, uncompressed int64 + blobHashes, err := v.Repositories.Snapshots.GetBlobHashes(v.ctx, snapshotID) if err != nil { return 0, 0 @@ -431,7 +471,7 @@ func (v *Vaultik) getSnapshotBlobSizes(snapshotID string) (compressed int64, unc func (v *Vaultik) ListSnapshots(jsonOutput bool) error { log.Info("Listing snapshots") - localSnaps, err := v.Repositories.Snapshots.ListRecent(v.ctx, 10000) + localSnaps, err := v.Repositories.Snapshots.ListRecent(v.ctx, listRecentLimit) if err != nil { return fmt.Errorf("listing local snapshots: %w", err) } @@ -465,11 +505,20 @@ func (v *Vaultik) ListSnapshots(jsonOutput bool) error { return nil } + v.reportRemoteDrift(localSnaps) + + return nil +} + +// reportRemoteDrift cross-references local snapshot records against the +// remote metadata keys and reports local-only records and unknown +// remote keys. Never fails: remote listing errors degrade to a warning. +func (v *Vaultik) reportRemoteDrift(localSnaps []*database.Snapshot) { remoteKeys, err := v.listAllRemoteSnapshotKeys() if err != nil { - v.UI.Warning("Could not list backup destination store: %v.", err) + v.UI.Warningf("Could not list backup destination store: %v.", err) - return nil + return } localKeys := make(map[string]string, len(localSnaps)) @@ -503,20 +552,20 @@ func (v *Vaultik) ListSnapshots(jsonOutput bool) error { } if len(localOnly) > 0 { - v.UI.Warning("%d local snapshot record(s) not found in backup destination store:", len(localOnly)) + v.UI.Warningf("%d local snapshot record(s) not found in backup "+ + "destination store:", len(localOnly)) for _, id := range localOnly { - v.UI.Info("%s", v.UI.Snapshot(id)) + v.UI.Infof("%s", v.UI.Snapshot(id)) } - v.UI.Info("Run 'vaultik snapshot cleanup' to remove stale local records.") + v.UI.Infof("Run 'vaultik snapshot cleanup' to remove stale local records.") } if remoteOnlyCount > 0 { - v.UI.Notice("NOTE: %d remote snapshot(s) found in backup destination store but not in local database.", remoteOnlyCount) + v.UI.Noticef("NOTE: %d remote snapshot(s) found in backup destination "+ + "store but not in local database.", remoteOnlyCount) } - - return nil } // snapshotInfoFromLocal builds a SnapshotInfo row from a local snapshot @@ -525,14 +574,16 @@ func (v *Vaultik) ListSnapshots(jsonOutput bool) error { func (v *Vaultik) snapshotInfoFromLocal(ls *database.Snapshot) SnapshotInfo { idStr := ls.ID.String() - totalSize, err := v.Repositories.Snapshots.GetSnapshotTotalCompressedSize(v.ctx, idStr) + totalSize, err := v.Repositories.Snapshots.GetSnapshotTotalCompressedSize( + v.ctx, idStr) if err != nil { log.Warn("Failed to get total compressed size", "id", idStr, "error", err) totalSize = ls.BlobSize } - uncompressedSize, err := v.Repositories.Snapshots.GetSnapshotUncompressedChunkSize(v.ctx, idStr) + uncompressedSize, err := v.Repositories.Snapshots.GetSnapshotUncompressedChunkSize( + v.ctx, idStr) if err != nil { log.Warn("Failed to get uncompressed chunk size", "id", idStr, "error", err) } @@ -552,9 +603,12 @@ func (v *Vaultik) snapshotInfoFromLocal(ls *database.Snapshot) SnapshotInfo { } } +// tabPadding is the tabwriter cell padding for the snapshot table. +const tabPadding = 3 + // printSnapshotTable renders the snapshot list as a formatted table func (v *Vaultik) printSnapshotTable(snapshots []SnapshotInfo) error { - w := tabwriter.NewWriter(v.Stdout, 0, 0, 3, ' ', 0) + w := tabwriter.NewWriter(v.Stdout, 0, 0, tabPadding, ' ', 0) _, err := fmt.Fprintln(w, "CONFIGURED SNAPSHOTS:") if err != nil { @@ -592,11 +646,17 @@ func (v *Vaultik) printSnapshotTable(snapshots []SnapshotInfo) error { return err } - if _, err := fmt.Fprintln(w, "SNAPSHOT ID\tTIMESTAMP\tCOMPRESSED SIZE\tUNCOMPRESSED SIZE\tNEW CHUNK SIZE"); err != nil { + _, err = fmt.Fprintln(w, + "SNAPSHOT ID\tTIMESTAMP\tCOMPRESSED SIZE\t"+ + "UNCOMPRESSED SIZE\tNEW CHUNK SIZE") + if err != nil { return err } - if _, err := fmt.Fprintln(w, "───────────\t─────────\t───────────────\t─────────────────\t──────────────"); err != nil { + _, err = fmt.Fprintln(w, + "───────────\t─────────\t───────────────\t"+ + "─────────────────\t──────────────") + if err != nil { return err } @@ -626,11 +686,15 @@ func (v *Vaultik) printSnapshotTable(snapshots []SnapshotInfo) error { // SnapshotPurgeOptions contains options for the snapshot purge command. type SnapshotPurgeOptions struct { - KeepLatest bool // Keep only the most recent snapshot per name - OlderThan string // Drop snapshots older than this duration (e.g. "30d", "6m", "1y") - Force bool // Skip confirmation prompt - Names []string // If non-empty, only operate on snapshots with one of these names - Quiet bool // Suppress informational output (used by --prune flag) + KeepLatest bool // Keep only the most recent snapshot per name + // OlderThan drops snapshots older than this duration (e.g. "30d", + // "6m", "1y"). + OlderThan string + Force bool // Skip confirmation prompt + // Names restricts the operation to snapshots with one of these + // names when non-empty. + Names []string + Quiet bool // Suppress informational output (used by --prune flag) } // PurgeSnapshotsWithOptions removes old snapshots based on criteria. @@ -649,7 +713,7 @@ func (v *Vaultik) PurgeSnapshotsWithOptions(opts *SnapshotPurgeOptions) error { } // Get snapshots from local database - dbSnapshots, err := v.Repositories.Snapshots.ListRecent(v.ctx, 10000) + dbSnapshots, err := v.Repositories.Snapshots.ListRecent(v.ctx, listRecentLimit) if err != nil { return fmt.Errorf("listing snapshots: %w", err) } @@ -685,9 +749,32 @@ func (v *Vaultik) PurgeSnapshotsWithOptions(opts *SnapshotPurgeOptions) error { return snapshots[i].Timestamp.After(snapshots[j].Timestamp) }) + toDelete, err := selectSnapshotsToPurge(snapshots, opts) + if err != nil { + return err + } + + if len(toDelete) == 0 { + if !opts.Quiet { + v.printlnStdout("No snapshots to delete") + } + + return nil + } + + return v.confirmAndExecutePurge(toDelete, opts.Force, opts.Quiet) +} + +// selectSnapshotsToPurge applies the purge retention criteria to the +// newest-first sorted snapshot list and returns the deletion +// candidates. +func selectSnapshotsToPurge( + snapshots []SnapshotInfo, opts *SnapshotPurgeOptions, +) ([]SnapshotInfo, error) { var toDelete []SnapshotInfo - if opts.KeepLatest { + switch { + case opts.KeepLatest: // Keep the latest snapshot per snapshot name. Snapshots are sorted // newest-first, so the first occurrence of each name is kept. seen := make(map[string]bool) @@ -702,10 +789,10 @@ func (v *Vaultik) PurgeSnapshotsWithOptions(opts *SnapshotPurgeOptions) error { seen[name] = true } - } else if opts.OlderThan != "" { + case opts.OlderThan != "": duration, err := parseDuration(opts.OlderThan) if err != nil { - return fmt.Errorf("invalid duration: %w", err) + return nil, fmt.Errorf("invalid duration: %w", err) } cutoff := time.Now().UTC().Add(-duration) @@ -716,24 +803,19 @@ func (v *Vaultik) PurgeSnapshotsWithOptions(opts *SnapshotPurgeOptions) error { } } - if len(toDelete) == 0 { - if !opts.Quiet { - v.printlnStdout("No snapshots to delete") - } - - return nil - } - - return v.confirmAndExecutePurge(toDelete, opts.Force, opts.Quiet) + return toDelete, nil } -// confirmAndExecutePurge shows deletion candidates, confirms with user, and deletes snapshots -func (v *Vaultik) confirmAndExecutePurge(toDelete []SnapshotInfo, force, quiet bool) error { +// confirmAndExecutePurge shows deletion candidates, confirms with the +// user, and deletes snapshots. +func (v *Vaultik) confirmAndExecutePurge( + toDelete []SnapshotInfo, force, quiet bool, +) error { if !quiet { - v.printfStdout("The following snapshots will be deleted:\n\n") + v.stdoutf("The following snapshots will be deleted:\n\n") for _, snap := range toDelete { - v.printfStdout(" %s (%s, %s)\n", + v.stdoutf(" %s (%s, %s)\n", snap.ID, snap.Timestamp.Format("2006-01-02 15:04:05"), formatBytes(snap.CompressedSize)) @@ -742,16 +824,15 @@ func (v *Vaultik) confirmAndExecutePurge(toDelete []SnapshotInfo, force, quiet b // Confirm unless --force is used if !force { - v.printfStdout("\nDelete %d snapshot(s)? [y/N] ", len(toDelete)) + v.stdoutf("\nDelete %d snapshot(s)? [y/N] ", len(toDelete)) var confirm string _, err := v.scanStdin(&confirm) if err != nil { - // Treat EOF or error as "no" v.printlnStdout("Cancelled") - return nil + return nil //nolint:nilerr // treat EOF or read error as "no" } if strings.ToLower(confirm) != "y" { @@ -760,7 +841,7 @@ func (v *Vaultik) confirmAndExecutePurge(toDelete []SnapshotInfo, force, quiet b return nil } } else if !quiet { - v.printfStdout("\nDeleting %d snapshot(s) (--force specified)\n", len(toDelete)) + v.stdoutf("\nDeleting %d snapshot(s) (--force specified)\n", len(toDelete)) } // Delete snapshots (both local and remote) @@ -770,7 +851,8 @@ func (v *Vaultik) confirmAndExecutePurge(toDelete []SnapshotInfo, force, quiet b err := v.deleteSnapshotFromLocalDB(snapshotID) if err != nil { - log.Error("Failed to delete from local database", "snapshot_id", snapshotID, "error", err) + log.Error("Failed to delete from local database", + "snapshot_id", snapshotID, "error", err) } err = v.deleteRemoteSnapshotByKey(snapshot.RemoteSnapshotKey(snapshotID)) @@ -785,13 +867,15 @@ func (v *Vaultik) confirmAndExecutePurge(toDelete []SnapshotInfo, force, quiet b if v.SnapshotManager != nil { err := v.SnapshotManager.CleanupOrphanedData(v.ctx) if err != nil { - log.Warn("Failed to clean up orphaned local data after purge", "error", err) + log.Warn("Failed to clean up orphaned local data after purge", + "error", err) } } if !quiet { - v.printfStdout("Deleted %d snapshot(s)\n", len(toDelete)) - v.printlnStdout("\nNote: Run 'vaultik prune' to clean up unreferenced remote blobs.") + v.stdoutf("Deleted %d snapshot(s)\n", len(toDelete)) + v.printlnStdout( + "\nNote: Run 'vaultik prune' to clean up unreferenced remote blobs.") } return nil @@ -810,7 +894,9 @@ func (v *Vaultik) VerifySnapshot(snapshotID string, deep bool) error { // VerifySnapshotWithOptions checks snapshot integrity with full options. // Deep verification is delegated to RunDeepVerify so this function only // implements the shallow (existence-only) path. -func (v *Vaultik) VerifySnapshotWithOptions(snapshotID string, opts *VerifyOptions) error { +func (v *Vaultik) VerifySnapshotWithOptions( + snapshotID string, opts *VerifyOptions, +) error { if opts.Deep { return v.RunDeepVerify(snapshotID, opts) } @@ -827,7 +913,7 @@ func (v *Vaultik) VerifySnapshotWithOptions(snapshotID string, opts *VerifyOptio manifest, err := v.downloadManifestByKey(snapshot.RemoteSnapshotKey(snapshotID)) if err != nil { if opts.JSON { - result.Status = "failed" + result.Status = verifyStatusFailed result.ErrorMessage = fmt.Sprintf("downloading manifest: %v", err) return v.outputVerifyJSON(result) @@ -840,30 +926,32 @@ func (v *Vaultik) VerifySnapshotWithOptions(snapshotID string, opts *VerifyOptio result.TotalSize = manifest.TotalCompressedSize if !opts.JSON { - v.printfStdout("Snapshot information:\n") - v.printfStdout(" Blob count: %d\n", manifest.BlobCount) - v.printfStdout(" Total size: %s\n", humanize.Bytes(uint64(manifest.TotalCompressedSize))) + v.stdoutf("Snapshot information:\n") + v.stdoutf(" Blob count: %d\n", manifest.BlobCount) + v.stdoutf(" Total size: %s\n", ubytes(manifest.TotalCompressedSize)) if manifest.Timestamp != "" { - t, err := time.Parse(time.RFC3339, manifest.Timestamp) - if err == nil { - v.printfStdout(" Created: %s\n", t.Format("2006-01-02 15:04:05 MST")) + t, terr := time.Parse(time.RFC3339, manifest.Timestamp) + if terr == nil { + v.stdoutf(" Created: %s\n", + t.Format("2006-01-02 15:04:05 MST")) } } v.printlnStdout() // Check each blob exists - v.printfStdout("Checking blob existence...\n") + v.stdoutf("Checking blob existence...\n") } - result.Verified, result.Missing, result.MissingSize = v.verifyManifestBlobsExist(manifest, opts) + result.Verified, result.Missing, result.MissingSize = + v.verifyManifestBlobsExist(manifest, opts) return v.formatVerifyResult(result, manifest, opts) } -// printVerifyHeader prints the snapshot ID and parsed timestamp for verification output. -// Snapshot ID format: hostname[_name]_ +// printVerifyHeader prints the snapshot ID and parsed timestamp for +// verification output. Snapshot ID format: hostname[_name]_ func (v *Vaultik) printVerifyHeader(snapshotID string, opts *VerifyOptions) { var snapshotTime time.Time @@ -873,26 +961,39 @@ func (v *Vaultik) printVerifyHeader(snapshotID string, opts *VerifyOptions) { } if !opts.JSON { - v.printfStdout("Verifying snapshot %s\n", snapshotID) + v.stdoutf("Verifying snapshot %s\n", snapshotID) if !snapshotTime.IsZero() { - v.printfStdout("Snapshot time: %s\n", snapshotTime.Format("2006-01-02 15:04:05 MST")) + v.stdoutf("Snapshot time: %s\n", + snapshotTime.Format("2006-01-02 15:04:05 MST")) } v.printlnStdout() } } -// verifyManifestBlobsExist checks that each blob in the manifest exists in storage -func (v *Vaultik) verifyManifestBlobsExist(manifest *snapshot.Manifest, opts *VerifyOptions) (verified, missing int, missingSize int64) { - for _, blob := range manifest.Blobs { - blobPath := fmt.Sprintf("blobs/%s/%s/%s", blob.Hash[:2], blob.Hash[2:4], blob.Hash) +// verifyManifestBlobsExist checks that each blob in the manifest exists +// in storage, returning the verified count, missing count, and total +// missing bytes. +func (v *Vaultik) verifyManifestBlobsExist( + manifest *snapshot.Manifest, opts *VerifyOptions, +) (int, int, int64) { + var ( + verified, missing int + missingSize int64 + ) - // Shallow: just check existence (deep verification is handled by RunDeepVerify) + for _, blob := range manifest.Blobs { + blobPath := fmt.Sprintf("blobs/%s/%s/%s", + blob.Hash[:2], blob.Hash[2:4], blob.Hash) + + // Shallow: check existence only (deep verification is handled + // by RunDeepVerify). _, err := v.Storage.Stat(v.ctx, blobPath) if err != nil { if !opts.JSON { - v.printfStdout(" Missing: %s (%s)\n", blob.Hash, humanize.Bytes(uint64(blob.CompressedSize))) + v.stdoutf(" Missing: %s (%s)\n", + blob.Hash, ubytes(blob.CompressedSize)) } missing++ @@ -905,11 +1006,14 @@ func (v *Vaultik) verifyManifestBlobsExist(manifest *snapshot.Manifest, opts *Ve return verified, missing, missingSize } -// formatVerifyResult outputs the final verification results as JSON or human-readable text -func (v *Vaultik) formatVerifyResult(result *VerifyResult, manifest *snapshot.Manifest, opts *VerifyOptions) error { +// formatVerifyResult outputs the final verification results as JSON or +// human-readable text. +func (v *Vaultik) formatVerifyResult( + result *VerifyResult, manifest *snapshot.Manifest, opts *VerifyOptions, +) error { if opts.JSON { if result.Missing > 0 { - result.Status = "failed" + result.Status = verifyStatusFailed result.ErrorMessage = fmt.Sprintf("%d blobs are missing", result.Missing) } else { result.Status = "ok" @@ -918,25 +1022,26 @@ func (v *Vaultik) formatVerifyResult(result *VerifyResult, manifest *snapshot.Ma return v.outputVerifyJSON(result) } - v.printfStdout("\nVerification complete:\n") - v.printfStdout(" Verified: %d blobs (%s)\n", result.Verified, - humanize.Bytes(uint64(manifest.TotalCompressedSize-result.MissingSize))) + v.stdoutf("\nVerification complete:\n") + v.stdoutf(" Verified: %d blobs (%s)\n", result.Verified, + ubytes(manifest.TotalCompressedSize-result.MissingSize)) if result.Missing > 0 { - v.printfStdout(" Missing: %d blobs (%s)\n", result.Missing, humanize.Bytes(uint64(result.MissingSize))) + v.stdoutf(" Missing: %d blobs (%s)\n", + result.Missing, ubytes(result.MissingSize)) } else { - v.printfStdout(" Missing: 0 blobs\n") + v.stdoutf(" Missing: 0 blobs\n") } - v.printfStdout(" Status: ") + v.stdoutf(" Status: ") if result.Missing > 0 { - v.printfStdout("FAILED - %d blobs are missing\n", result.Missing) + v.stdoutf("FAILED - %d blobs are missing\n", result.Missing) - return fmt.Errorf("%d blobs are missing", result.Missing) + return fmt.Errorf("%d %w", result.Missing, errBlobsMissing) } - v.printfStdout("OK - All blobs verified\n") + v.stdoutf("OK - All blobs verified\n") return nil } @@ -951,8 +1056,8 @@ func (v *Vaultik) outputVerifyJSON(result *VerifyResult) error { return fmt.Errorf("encoding JSON: %w", err) } - if result.Status == "failed" { - return fmt.Errorf("verification failed: %s", result.ErrorMessage) + if result.Status == verifyStatusFailed { + return fmt.Errorf("%w: %s", errSnapshotVerifyFailed, result.ErrorMessage) } return nil @@ -979,7 +1084,7 @@ func (v *Vaultik) CleanupLocalSnapshots() error { remoteSet[k] = true } - localSnapshots, err := v.Repositories.Snapshots.ListRecent(v.ctx, 10000) + localSnapshots, err := v.Repositories.Snapshots.ListRecent(v.ctx, listRecentLimit) if err != nil { return fmt.Errorf("listing local snapshots: %w", err) } @@ -989,11 +1094,12 @@ func (v *Vaultik) CleanupLocalSnapshots() error { for _, snap := range localSnapshots { id := snap.ID.String() if !remoteSet[snapshot.RemoteSnapshotKey(id)] { - v.printfStdout("Removing stale local record: %s\n", id) + v.stdoutf("Removing stale local record: %s\n", id) - err := v.deleteSnapshotFromLocalDB(id) + err = v.deleteSnapshotFromLocalDB(id) if err != nil { - log.Error("Failed to delete local snapshot", "snapshot_id", id, "error", err) + log.Error("Failed to delete local snapshot", + "snapshot_id", id, "error", err) continue } @@ -1005,7 +1111,7 @@ func (v *Vaultik) CleanupLocalSnapshots() error { if removed == 0 { v.printlnStdout("No stale local snapshots found.") } else { - v.printfStdout("Removed %d stale local snapshot record(s).\n", removed) + v.stdoutf("Removed %d stale local snapshot record(s).\n", removed) } return nil @@ -1048,7 +1154,8 @@ func (v *Vaultik) syncWithRemote() error { // Extract snapshot ID from paths like metadata/hostname-20240115-143052Z/ parts := strings.Split(object.Key, "/") - if len(parts) >= 2 && parts[0] == "metadata" && parts[1] != "" { + if len(parts) >= minSnapshotIDParts && + parts[0] == metadataDirName && parts[1] != "" { // Skip macOS resource fork files (._*) and other hidden files if strings.HasPrefix(parts[1], ".") { continue @@ -1061,7 +1168,7 @@ func (v *Vaultik) syncWithRemote() error { log.Debug("Found remote snapshots", "count", len(remoteSnapshots)) // Get all local snapshots (use a high limit to get all) - localSnapshots, err := v.Repositories.Snapshots.ListRecent(v.ctx, 10000) + localSnapshots, err := v.Repositories.Snapshots.ListRecent(v.ctx, listRecentLimit) if err != nil { return fmt.Errorf("listing local snapshots: %w", err) } @@ -1069,14 +1176,16 @@ func (v *Vaultik) syncWithRemote() error { // Remove local snapshots that don't exist remotely removedCount := 0 - for _, snapshot := range localSnapshots { - snapshotIDStr := snapshot.ID.String() + for _, snap := range localSnapshots { + snapshotIDStr := snap.ID.String() if !remoteSnapshots[snapshotIDStr] { - log.Info("Removing local snapshot not found in remote", "snapshot_id", snapshot.ID) + log.Info("Removing local snapshot not found in remote", + "snapshot_id", snap.ID) - err := v.deleteSnapshotFromLocalDB(snapshotIDStr) + err = v.deleteSnapshotFromLocalDB(snapshotIDStr) if err != nil { - log.Error("Failed to delete local snapshot", "snapshot_id", snapshot.ID, "error", err) + log.Error("Failed to delete local snapshot", + "snapshot_id", snap.ID, "error", err) } else { removedCount++ } @@ -1099,6 +1208,8 @@ type RemoveOptions struct { } // RemoveResult contains the result of a snapshot removal +// +//nolint:tagliatelle // snake_case is the established JSON output format type RemoveResult struct { SnapshotID string `json:"snapshot_id,omitempty"` SnapshotsRemoved []string `json:"snapshots_removed,omitempty"` @@ -1119,7 +1230,9 @@ const pruneCommandHint = "vaultik prune" // `vaultik prune` is for — the command prints the prune invocation to // run as a follow-up. When the remote is unreachable the command still // completes the local-DB removal and warns. -func (v *Vaultik) RemoveSnapshot(snapshotID string, opts *RemoveOptions) (*RemoveResult, error) { +func (v *Vaultik) RemoveSnapshot( + snapshotID string, opts *RemoveOptions, +) (*RemoveResult, error) { result := &RemoveResult{ SnapshotID: snapshotID, } @@ -1130,46 +1243,11 @@ func (v *Vaultik) RemoveSnapshot(snapshotID string, opts *RemoveOptions) (*Remov } if opts.DryRun { - result.DryRun = true - - if !opts.JSON { - v.printfStdout("Would remove snapshot: %s\n", snapshotID) - - if !opts.LocalOnly { - v.printlnStdout("Would also remove snapshot metadata from remote storage (blobs untouched)") - } - - v.printlnStdout("[Dry run - no changes made]") - } - - if opts.JSON { - return result, v.outputRemoveJSON(result) - } - - return result, nil + return v.removeSnapshotDryRun(result, snapshotID, opts) } - if !opts.Force && !opts.JSON { - if opts.LocalOnly { - v.printfStdout("Remove snapshot '%s' from local database (remote untouched)? [y/N] ", snapshotID) - } else { - v.printfStdout("Remove snapshot '%s' from local database AND its metadata from remote storage? [y/N] ", snapshotID) - } - - var confirm string - - _, err = v.scanStdin(&confirm) - if err != nil { - v.printlnStdout("Cancelled") - - return result, nil - } - - if strings.ToLower(confirm) != "y" { - v.printlnStdout("Cancelled") - - return result, nil - } + if !opts.Force && !opts.JSON && !v.confirmRemoveSnapshot(snapshotID, opts) { + return result, nil } log.Info("Removing snapshot from local database", "snapshot_id", snapshotID) @@ -1180,30 +1258,14 @@ func (v *Vaultik) RemoveSnapshot(snapshotID string, opts *RemoveOptions) (*Remov } if !opts.LocalOnly { - log.Info("Removing snapshot metadata from remote storage", "snapshot_id", snapshotID) - - remoteKey := snapshot.RemoteSnapshotKey(snapshotID) - - err := v.deleteRemoteSnapshotByKey(remoteKey) - if err != nil { - // Warn-and-proceed: the local-DB removal has already - // happened, so let the user know the remote half didn't - // finish and they can retry with `vaultik prune` once the - // destination store is reachable. - log.Warn("Could not remove snapshot metadata from remote storage", "error", err) - - if v.UI != nil { - v.UI.Warning("Could not remove snapshot metadata from remote: %v. Run '%s' once the remote is reachable to finish cleanup.", err, pruneCommandHint) - } - } else { - result.RemoteRemoved = true - } + result.RemoteRemoved = v.removeSnapshotRemote(snapshotID) } if v.SnapshotManager != nil { - err := v.SnapshotManager.CleanupOrphanedData(v.ctx) + err = v.SnapshotManager.CleanupOrphanedData(v.ctx) if err != nil { - log.Warn("Failed to clean up orphaned local data after removal", "error", err) + log.Warn("Failed to clean up orphaned local data after removal", + "error", err) } } @@ -1211,16 +1273,99 @@ func (v *Vaultik) RemoveSnapshot(snapshotID string, opts *RemoveOptions) (*Remov return result, v.outputRemoveJSON(result) } - v.printfStdout("Removed snapshot '%s' from local database\n", snapshotID) + v.stdoutf("Removed snapshot '%s' from local database\n", snapshotID) if !opts.LocalOnly && result.RemoteRemoved { v.printlnStdout("Removed snapshot metadata from remote storage") - v.printfStdout("\nNote: The removed snapshot's blobs remain on the remote. Run '%s' to delete any blobs no longer referenced by any remaining remote snapshot.\n", pruneCommandHint) + v.stdoutf("\nNote: The removed snapshot's blobs remain on the remote. "+ + "Run '%s' to delete any blobs no longer referenced by any "+ + "remaining remote snapshot.\n", pruneCommandHint) } return result, nil } +// removeSnapshotDryRun reports what RemoveSnapshot would do without +// making changes. +func (v *Vaultik) removeSnapshotDryRun( + result *RemoveResult, snapshotID string, opts *RemoveOptions, +) (*RemoveResult, error) { + result.DryRun = true + + if opts.JSON { + return result, v.outputRemoveJSON(result) + } + + v.stdoutf("Would remove snapshot: %s\n", snapshotID) + + if !opts.LocalOnly { + v.printlnStdout("Would also remove snapshot metadata from " + + "remote storage (blobs untouched)") + } + + v.printlnStdout("[Dry run - no changes made]") + + return result, nil +} + +// confirmRemoveSnapshot prompts for confirmation before removing a +// snapshot. Returns false (treating EOF/read errors as "no") unless the +// user answers "y". +func (v *Vaultik) confirmRemoveSnapshot(snapshotID string, opts *RemoveOptions) bool { + if opts.LocalOnly { + v.stdoutf("Remove snapshot '%s' from local database "+ + "(remote untouched)? [y/N] ", snapshotID) + } else { + v.stdoutf("Remove snapshot '%s' from local database AND its metadata "+ + "from remote storage? [y/N] ", snapshotID) + } + + var confirm string + + _, err := v.scanStdin(&confirm) + if err != nil { + v.printlnStdout("Cancelled") + + return false + } + + if strings.ToLower(confirm) != "y" { + v.printlnStdout("Cancelled") + + return false + } + + return true +} + +// removeSnapshotRemote strips the snapshot's metadata from the +// destination store, warning and proceeding on failure: the local-DB +// removal has already happened, so the user is told the remote half +// didn't finish and can retry with `vaultik prune` once the destination +// store is reachable. Returns true when the remote removal succeeded. +func (v *Vaultik) removeSnapshotRemote(snapshotID string) bool { + log.Info("Removing snapshot metadata from remote storage", + "snapshot_id", snapshotID) + + remoteKey := snapshot.RemoteSnapshotKey(snapshotID) + + err := v.deleteRemoteSnapshotByKey(remoteKey) + if err != nil { + log.Warn("Could not remove snapshot metadata from remote storage", + "error", err) + + if v.UI != nil { + v.UI.Warningf("Could not remove snapshot metadata from remote: "+ + "%v. Run '%s' once the remote is reachable to finish cleanup.", + err, pruneCommandHint) + } + + return false + } + + return true +} + // RemoveAllSnapshots removes every snapshot known to the local // database from the local index, and (with --remote) every snapshot // metadata directory in remote storage. Both sides are processed so a @@ -1244,7 +1389,7 @@ func (v *Vaultik) RemoveAllSnapshots(opts *RemoveOptions) (*RemoveResult, error) remoteKeys, remoteErr := v.listAllRemoteSnapshotKeys() if remoteErr != nil { log.Warn("Could not list remote snapshots", "error", remoteErr) - v.UI.Warning("Could not list remote snapshots: %v.", remoteErr) + v.UI.Warningf("Could not list remote snapshots: %v.", remoteErr) } // Anything visible on the remote that doesn't correspond to a @@ -1286,7 +1431,7 @@ func (v *Vaultik) localSnapshotIDs() ([]string, error) { return nil, nil } - snaps, err := v.Repositories.Snapshots.ListRecent(v.ctx, 100000) + snaps, err := v.Repositories.Snapshots.ListRecent(v.ctx, listRecentLimit) if err != nil { return nil, err } @@ -1321,13 +1466,15 @@ func (v *Vaultik) listAllRemoteSnapshotKeys() ([]string, error) { } parts := strings.Split(object.Key, "/") - if len(parts) >= 2 && parts[0] == "metadata" && parts[1] != "" { + if len(parts) >= minSnapshotIDParts && + parts[0] == metadataDirName && parts[1] != "" { // Skip macOS resource fork files (._*) and other hidden files if strings.HasPrefix(parts[1], ".") { continue } - if strings.HasSuffix(object.Key, "/") || strings.Contains(object.Key, "/manifest.json.zst") { + if strings.HasSuffix(object.Key, "/") || + strings.Contains(object.Key, "/manifest.json.zst") { key := parts[1] if !seen[key] { seen[key] = true @@ -1341,30 +1488,35 @@ func (v *Vaultik) listAllRemoteSnapshotKeys() ([]string, error) { } // handleRemoveAllDryRun handles the dry-run mode for removing all snapshots -func (v *Vaultik) handleRemoveAllDryRun(localSnaps, orphanRemoteKeys []string, opts *RemoveOptions) (*RemoveResult, error) { +func (v *Vaultik) handleRemoveAllDryRun( + localSnaps, orphanRemoteKeys []string, opts *RemoveOptions, +) (*RemoveResult, error) { result := &RemoveResult{DryRun: true} result.SnapshotsRemoved = append(result.SnapshotsRemoved, localSnaps...) if !opts.LocalOnly { - result.SnapshotsRemoved = append(result.SnapshotsRemoved, orphanRemoteKeys...) + result.SnapshotsRemoved = append( + result.SnapshotsRemoved, orphanRemoteKeys...) } if !opts.JSON { - v.printfStdout("Would remove %d local snapshot(s):\n", len(localSnaps)) + v.stdoutf("Would remove %d local snapshot(s):\n", len(localSnaps)) for _, id := range localSnaps { - v.printfStdout(" %s\n", id) + v.stdoutf(" %s\n", id) } if !opts.LocalOnly { if len(orphanRemoteKeys) > 0 { - v.printfStdout("Would also remove %d orphan remote snapshot key(s):\n", len(orphanRemoteKeys)) + v.stdoutf("Would also remove %d orphan remote snapshot key(s):\n", + len(orphanRemoteKeys)) for _, key := range orphanRemoteKeys { - v.printfStdout(" %s\n", key) + v.stdoutf(" %s\n", key) } } else { - v.printlnStdout("Would also remove snapshot metadata from remote storage (blobs untouched)") + v.printlnStdout("Would also remove snapshot metadata from " + + "remote storage (blobs untouched)") } } @@ -1382,58 +1534,30 @@ func (v *Vaultik) handleRemoveAllDryRun(localSnaps, orphanRemoteKeys []string, o // is set, every corresponding remote metadata directory plus any // orphan remote keys. Blobs are not touched; the printed prune-command // hint is the next step. -func (v *Vaultik) executeRemoveAll(localSnaps, orphanRemoteKeys []string, opts *RemoveOptions) (*RemoveResult, error) { +func (v *Vaultik) executeRemoveAll( + localSnaps, orphanRemoteKeys []string, opts *RemoveOptions, +) (*RemoveResult, error) { if !opts.Force { - return nil, errors.New("--all requires --force") + return nil, errRemoveAllNeedsForce } - log.Info("Removing all snapshots", "local_count", len(localSnaps), "orphan_remote_count", len(orphanRemoteKeys)) + log.Info("Removing all snapshots", + "local_count", len(localSnaps), + "orphan_remote_count", len(orphanRemoteKeys)) result := &RemoveResult{} - remoteErrors := 0 - for _, snapshotID := range localSnaps { - log.Info("Removing snapshot", "snapshot_id", snapshotID) - - err := v.deleteSnapshotFromLocalDB(snapshotID) - if err != nil { - log.Error("Failed to remove from local database", "snapshot_id", snapshotID, "error", err) - - continue - } - - if !opts.LocalOnly { - err := v.deleteRemoteSnapshotByKey(snapshot.RemoteSnapshotKey(snapshotID)) - if err != nil { - log.Warn("Failed to remove snapshot metadata from remote", "snapshot_id", snapshotID, "error", err) - - remoteErrors++ - } - } - - result.SnapshotsRemoved = append(result.SnapshotsRemoved, snapshotID) - } + remoteErrors := v.removeAllLocalSnapshots(localSnaps, opts, result) if !opts.LocalOnly { - for _, key := range orphanRemoteKeys { - log.Info("Removing orphan remote snapshot", "remote_key", key) - - err := v.deleteRemoteSnapshotByKey(key) - if err != nil { - log.Warn("Failed to remove orphan from remote", "remote_key", key, "error", err) - - remoteErrors++ - - continue - } - - result.SnapshotsRemoved = append(result.SnapshotsRemoved, key) - } + remoteErrors += v.removeAllOrphanRemotes(orphanRemoteKeys, result) if remoteErrors == 0 { result.RemoteRemoved = true } else if v.UI != nil { - v.UI.Warning("Some remote metadata deletions failed. Run '%s' once the remote is healthy to clean up unreferenced blobs.", pruneCommandHint) + v.UI.Warningf("Some remote metadata deletions failed. Run '%s' "+ + "once the remote is healthy to clean up unreferenced blobs.", + pruneCommandHint) } } @@ -1448,16 +1572,116 @@ func (v *Vaultik) executeRemoveAll(localSnaps, orphanRemoteKeys []string, opts * return result, v.outputRemoveJSON(result) } - v.printfStdout("Removed %d snapshot(s)\n", len(result.SnapshotsRemoved)) + v.stdoutf("Removed %d snapshot(s)\n", len(result.SnapshotsRemoved)) if !opts.LocalOnly && result.RemoteRemoved { v.printlnStdout("Removed snapshot metadata from remote storage") - v.printfStdout("\nNote: Removed snapshots' blobs remain on the remote. Run '%s' to delete any blobs no longer referenced by any remaining remote snapshot.\n", pruneCommandHint) + v.stdoutf("\nNote: Removed snapshots' blobs remain on the remote. "+ + "Run '%s' to delete any blobs no longer referenced by any "+ + "remaining remote snapshot.\n", pruneCommandHint) } return result, nil } +// deleteIncompleteSnapshot deletes one incomplete snapshot and its +// association rows, reporting whether the snapshot row itself was +// deleted. +func (v *Vaultik) deleteIncompleteSnapshot(snapshotIDStr string, id any) bool { + log.Info("Deleting incomplete snapshot", "snapshot_id", id) + + // Delete related records first + err := v.Repositories.Snapshots.DeleteSnapshotFiles(v.ctx, snapshotIDStr) + if err != nil { + log.Error("Failed to delete snapshot files", + "snapshot_id", id, "error", err) + } + + err = v.Repositories.Snapshots.DeleteSnapshotBlobs(v.ctx, snapshotIDStr) + if err != nil { + log.Error("Failed to delete snapshot blobs", + "snapshot_id", id, "error", err) + } + + err = v.Repositories.Snapshots.DeleteSnapshotUploads(v.ctx, snapshotIDStr) + if err != nil { + log.Error("Failed to delete snapshot uploads", + "snapshot_id", id, "error", err) + } + + err = v.Repositories.Snapshots.Delete(v.ctx, snapshotIDStr) + if err != nil { + log.Error("Failed to delete snapshot", + "snapshot_id", id, "error", err) + + return false + } + + return true +} + +// removeAllLocalSnapshots removes each known local snapshot (and, unless +// --local-only, its remote metadata), returning the number of remote +// deletion failures. +func (v *Vaultik) removeAllLocalSnapshots( + localSnaps []string, opts *RemoveOptions, result *RemoveResult, +) int { + remoteErrors := 0 + + for _, snapshotID := range localSnaps { + log.Info("Removing snapshot", "snapshot_id", snapshotID) + + err := v.deleteSnapshotFromLocalDB(snapshotID) + if err != nil { + log.Error("Failed to remove from local database", + "snapshot_id", snapshotID, "error", err) + + continue + } + + if !opts.LocalOnly { + err = v.deleteRemoteSnapshotByKey( + snapshot.RemoteSnapshotKey(snapshotID)) + if err != nil { + log.Warn("Failed to remove snapshot metadata from remote", + "snapshot_id", snapshotID, "error", err) + + remoteErrors++ + } + } + + result.SnapshotsRemoved = append(result.SnapshotsRemoved, snapshotID) + } + + return remoteErrors +} + +// removeAllOrphanRemotes deletes remote snapshot metadata that has no +// local counterpart, returning the number of deletion failures. +func (v *Vaultik) removeAllOrphanRemotes( + orphanRemoteKeys []string, result *RemoveResult, +) int { + remoteErrors := 0 + + for _, key := range orphanRemoteKeys { + log.Info("Removing orphan remote snapshot", "remote_key", key) + + err := v.deleteRemoteSnapshotByKey(key) + if err != nil { + log.Warn("Failed to remove orphan from remote", + "remote_key", key, "error", err) + + remoteErrors++ + + continue + } + + result.SnapshotsRemoved = append(result.SnapshotsRemoved, key) + } + + return remoteErrors +} + // deleteSnapshotFromLocalDB removes a snapshot from the local database only func (v *Vaultik) deleteSnapshotFromLocalDB(snapshotID string) error { if v.Repositories == nil { @@ -1539,8 +1763,10 @@ type PruneResult struct { // and blobs from the local database. This ensures database consistency // before starting a new backup or on-demand via the prune command. func (v *Vaultik) PruneDatabase() (*PruneResult, error) { - log.Info("Pruning local database: removing incomplete snapshots and orphaned data") - v.UI.Begin("Pruning local index database (removing incomplete snapshots and orphaned data).") + log.Info("Pruning local database: " + + "removing incomplete snapshots and orphaned data") + v.UI.Beginf("Pruning local index database " + + "(removing incomplete snapshots and orphaned data).") result := &PruneResult{} @@ -1553,29 +1779,8 @@ func (v *Vaultik) PruneDatabase() (*PruneResult, error) { return nil, fmt.Errorf("getting incomplete snapshots: %w", err) } - for _, snapshot := range incompleteSnapshots { - snapshotIDStr := snapshot.ID.String() - log.Info("Deleting incomplete snapshot", "snapshot_id", snapshot.ID) - // Delete related records first - err := v.Repositories.Snapshots.DeleteSnapshotFiles(v.ctx, snapshotIDStr) - if err != nil { - log.Error("Failed to delete snapshot files", "snapshot_id", snapshot.ID, "error", err) - } - - err = v.Repositories.Snapshots.DeleteSnapshotBlobs(v.ctx, snapshotIDStr) - if err != nil { - log.Error("Failed to delete snapshot blobs", "snapshot_id", snapshot.ID, "error", err) - } - - err = v.Repositories.Snapshots.DeleteSnapshotUploads(v.ctx, snapshotIDStr) - if err != nil { - log.Error("Failed to delete snapshot uploads", "snapshot_id", snapshot.ID, "error", err) - } - - err = v.Repositories.Snapshots.Delete(v.ctx, snapshotIDStr) - if err != nil { - log.Error("Failed to delete snapshot", "snapshot_id", snapshot.ID, "error", err) - } else { + for _, snap := range incompleteSnapshots { + if v.deleteIncompleteSnapshot(snap.ID.String(), snap.ID) { result.SnapshotsDeleted++ } } @@ -1609,27 +1814,33 @@ func (v *Vaultik) PruneDatabase() (*PruneResult, error) { snapshotCountAfter := snapshotCountBefore - result.SnapshotsDeleted - v.UI.Complete("Pruned local index database.") - v.UI.Detail("Incomplete snapshots: %d removed (%d remain).", result.SnapshotsDeleted, snapshotCountAfter) - v.UI.Detail("Orphaned files: %d removed (%d remain).", result.FilesDeleted, fileCountAfter) - v.UI.Detail("Orphaned chunks: %d removed (%d remain).", result.ChunksDeleted, chunkCountAfter) - v.UI.Detail("Orphaned blobs: %d removed (%d remain).", result.BlobsDeleted, blobCountAfter) + v.UI.Completef("Pruned local index database.") + v.UI.Detailf("Incomplete snapshots: %d removed (%d remain).", + result.SnapshotsDeleted, snapshotCountAfter) + v.UI.Detailf("Orphaned files: %d removed (%d remain).", + result.FilesDeleted, fileCountAfter) + v.UI.Detailf("Orphaned chunks: %d removed (%d remain).", + result.ChunksDeleted, chunkCountAfter) + v.UI.Detailf("Orphaned blobs: %d removed (%d remain).", + result.BlobsDeleted, blobCountAfter) return result, nil } -// validTableNameRe matches table names containing only lowercase alphanumeric characters and underscores. +// validTableNameRe matches table names containing only lowercase +// alphanumeric characters and underscores. var validTableNameRe = regexp.MustCompile(`^[a-z0-9_]+$`) -// getTableCount returns the count of rows in a table. -// The tableName is sanitized to only allow [a-z0-9_] characters to prevent SQL injection. +// getTableCount returns the count of rows in a table. The tableName is +// sanitized to only allow [a-z0-9_] characters to prevent SQL +// injection. func (v *Vaultik) getTableCount(tableName string) (int64, error) { if v.DB == nil { return 0, nil } if !validTableNameRe.MatchString(tableName) { - return 0, fmt.Errorf("invalid table name: %q", tableName) + return 0, fmt.Errorf("%w: %q", errInvalidTableName, tableName) } var count int64 diff --git a/internal/vaultik/snapshot_prune_test.go b/internal/vaultik/snapshot_prune_test.go index dbff412..8b76a93 100644 --- a/internal/vaultik/snapshot_prune_test.go +++ b/internal/vaultik/snapshot_prune_test.go @@ -1,20 +1,24 @@ -package vaultik +package vaultik_test import ( "testing" + + "sneak.berlin/go/vaultik/internal/vaultik" ) // TestSnapshotCreateOptions_PruneFlag verifies the Prune field exists on // SnapshotCreateOptions and can be set. func TestSnapshotCreateOptions_PruneFlag(t *testing.T) { - opts := &SnapshotCreateOptions{ + t.Parallel() + + opts := &vaultik.SnapshotCreateOptions{ Prune: true, } if !opts.Prune { t.Error("Expected Prune to be true") } - opts2 := &SnapshotCreateOptions{ + opts2 := &vaultik.SnapshotCreateOptions{ Prune: false, } if opts2.Prune { diff --git a/internal/vaultik/storage_bind.go b/internal/vaultik/storage_bind.go index 27f79c2..2570d48 100644 --- a/internal/vaultik/storage_bind.go +++ b/internal/vaultik/storage_bind.go @@ -1,12 +1,18 @@ package vaultik import ( + "errors" "fmt" "sneak.berlin/go/vaultik/internal/database" "sneak.berlin/go/vaultik/internal/log" ) +// errStorageBindingMismatch is returned when the local index database is +// bound to a different destination than the configured storage_url. +var errStorageBindingMismatch = errors.New( + "local index is bound to a different backup destination") + // EnsureStorageBinding guarantees that the local index database is // bound to the currently-configured storage destination. Every mutating // command must call this before touching either the local index or the @@ -54,7 +60,8 @@ func (v *Vaultik) EnsureStorageBinding() error { } if stored == "" { - err := v.Repositories.LocalMeta.Set(v.ctx, database.LocalMetaKeyStorageURL, configured) + err = v.Repositories.LocalMeta.Set( + v.ctx, database.LocalMetaKeyStorageURL, configured) if err != nil { return fmt.Errorf("recording local storage binding: %w", err) } @@ -68,18 +75,19 @@ func (v *Vaultik) EnsureStorageBinding() error { return nil } - return fmt.Errorf("%s", buildBindingMismatchMessage(stored, configured)) + return fmt.Errorf("%w\n%s", + errStorageBindingMismatch, buildBindingMismatchMessage(stored, configured)) } // buildBindingMismatchMessage assembles the multi-line explanation // shown when the local index is bound to a different destination than -// the currently-configured one. Kept as a separate function so the -// lint-flagged multi-line format string is expressed as a plain string -// literal rather than a fmt.Errorf argument (staticcheck ST1005 -// disallows trailing punctuation on error format strings). +// the currently-configured one (the first line lives in the +// errStorageBindingMismatch sentinel). Kept as a separate function so +// the multi-line text is expressed as a plain string literal rather +// than a fmt.Errorf argument (staticcheck ST1005 disallows trailing +// punctuation on error format strings). func buildBindingMismatchMessage(stored, configured string) string { - return "local index is bound to a different backup destination\n" + - " local index bound to: " + stored + "\n" + + return " local index bound to: " + stored + "\n" + " currently configured: " + configured + "\n" + "\n" + "The local index database tracks which chunks and blobs already exist at the\n" + diff --git a/internal/vaultik/storage_bind_test.go b/internal/vaultik/storage_bind_test.go index 087031b..a2f8272 100644 --- a/internal/vaultik/storage_bind_test.go +++ b/internal/vaultik/storage_bind_test.go @@ -15,7 +15,9 @@ import ( // buildBindTestVaultik returns a minimal Vaultik wired with a real // in-memory DB and a config carrying the given StorageURL. Enough to // exercise EnsureStorageBinding without spinning up storage or fx. -func buildBindTestVaultik(t *testing.T, storageURL string) (*vaultik.TestVaultik, *database.Repositories) { +func buildBindTestVaultik( + t *testing.T, storageURL string, +) (*vaultik.TestVaultik, *database.Repositories) { t.Helper() db, err := database.NewTestDB() @@ -35,6 +37,7 @@ func buildBindTestVaultik(t *testing.T, storageURL string) (*vaultik.TestVaultik func TestEnsureStorageBinding_FreshDBRecordsURL(t *testing.T) { log.Initialize(log.Config{}) + t.Parallel() tv, repos := buildBindTestVaultik(t, "file:///mnt/backups/new") @@ -42,11 +45,13 @@ func TestEnsureStorageBinding_FreshDBRecordsURL(t *testing.T) { got, err := repos.LocalMeta.Get(context.Background(), database.LocalMetaKeyStorageURL) require.NoError(t, err) - assert.Equal(t, "file:///mnt/backups/new", got, "first call must record the configured URL") + assert.Equal(t, "file:///mnt/backups/new", got, + "first call must record the configured URL") } func TestEnsureStorageBinding_MatchingURLPasses(t *testing.T) { log.Initialize(log.Config{}) + t.Parallel() tv, repos := buildBindTestVaultik(t, "s3://bucket/prefix") @@ -58,6 +63,7 @@ func TestEnsureStorageBinding_MatchingURLPasses(t *testing.T) { func TestEnsureStorageBinding_MismatchRefuses(t *testing.T) { log.Initialize(log.Config{}) + t.Parallel() tv, repos := buildBindTestVaultik(t, "file:///mnt/backups/new") diff --git a/internal/vaultik/vaultik.go b/internal/vaultik/vaultik.go index 29f4274..8ac75d0 100644 --- a/internal/vaultik/vaultik.go +++ b/internal/vaultik/vaultik.go @@ -1,3 +1,5 @@ +// Package vaultik implements the core backup, restore, verify, prune, +// and snapshot-management operations behind the vaultik CLI. package vaultik import ( @@ -19,6 +21,12 @@ import ( "sneak.berlin/go/vaultik/internal/ui" ) +// Sentinel errors for misconfigured encryption settings. +var ( + errNoAgeRecipients = errors.New("no age recipients configured") + errNoAgeSecretKey = errors.New("no age secret key configured") +) + // Vaultik contains all dependencies needed for vaultik operations type Vaultik struct { Globals *globals.Globals @@ -32,7 +40,7 @@ type Vaultik struct { Fs afero.Fs // Context management - ctx context.Context + ctx context.Context //nolint:containedctx // ctx bound at construction by design cancel context.CancelFunc // IO @@ -54,8 +62,8 @@ type Vaultik struct { restoreCacheObserver func(*blobDiskCache) } -// VaultikParams contains all parameters for New that can be provided by fx -type VaultikParams struct { +// Params contains all parameters for New that can be provided by fx +type Params struct { fx.In Globals *globals.Globals @@ -71,7 +79,7 @@ type VaultikParams struct { // New creates a new Vaultik instance with proper context management // It automatically includes crypto capabilities if age_secret_key is configured -func New(params VaultikParams) *Vaultik { +func New(params Params) *Vaultik { ctx, cancel := context.WithCancel(context.Background()) // Use provided filesystem or default to OS filesystem @@ -126,7 +134,7 @@ func (v *Vaultik) CanDecrypt() bool { // Returns an error if no recipients are configured func (v *Vaultik) GetEncryptor() (*crypto.Encryptor, error) { if len(v.Config.AgeRecipients) == 0 { - return nil, errors.New("no age recipients configured") + return nil, errNoAgeRecipients } return crypto.NewEncryptor(v.Config.AgeRecipients) @@ -136,19 +144,21 @@ func (v *Vaultik) GetEncryptor() (*crypto.Encryptor, error) { // Returns an error if no secret key is configured func (v *Vaultik) GetDecryptor() (*crypto.Decryptor, error) { if v.Config.AgeSecretKey == "" { - return nil, errors.New("no age secret key configured") + return nil, errNoAgeSecretKey } return crypto.NewDecryptor(v.Config.AgeSecretKey) } // GetFilesystem returns the filesystem instance used by Vaultik +// +//nolint:ireturn // afero.Fs is the filesystem abstraction by design func (v *Vaultik) GetFilesystem() afero.Fs { return v.Fs } -// printfStdout writes formatted output to stdout. -func (v *Vaultik) printfStdout(format string, args ...any) { +// stdoutf writes formatted output to stdout. +func (v *Vaultik) stdoutf(format string, args ...any) { _, _ = fmt.Fprintf(v.Stdout, format, args...) } diff --git a/internal/vaultik/verify.go b/internal/vaultik/verify.go index 8a17526..f5a7365 100644 --- a/internal/vaultik/verify.go +++ b/internal/vaultik/verify.go @@ -4,19 +4,37 @@ import ( "crypto/sha256" "database/sql" "encoding/hex" + "errors" "fmt" "hash" "io" "os" "time" - "github.com/dustin/go-humanize" "github.com/klauspost/compress/zstd" + + // Blank import registers the pure-Go sqlite driver for database/sql. _ "modernc.org/sqlite" "sneak.berlin/go/vaultik/internal/log" "sneak.berlin/go/vaultik/internal/snapshot" ) +// Sentinel errors for snapshot verification failures. +var ( + errVerificationFailed = errors.New("verification failed") + errSecretKeyRequired = errors.New( + "VAULTIK_AGE_SECRET_KEY not set; required for deep verification") + errChunksOutOfOrder = errors.New("chunks out of order") + errChunkHashMismatch = errors.New("chunk hash mismatch") + errTrailingBlobData = errors.New( + "blob has unexpected trailing bytes not covered by chunk list") + errManifestExtraBlob = errors.New("manifest contains blob not in database") + errBlobSizeMismatch = errors.New("blob size mismatch") +) + +// verifyStatusFailed is the JSON status value for a failed verification. +const verifyStatusFailed = "failed" + // VerifyOptions contains options for the verify command type VerifyOptions struct { Deep bool @@ -24,6 +42,8 @@ type VerifyOptions struct { } // VerifyResult contains the result of a snapshot verification +// +//nolint:tagliatelle // snake_case is the established JSON output format type VerifyResult struct { SnapshotID string `json:"snapshot_id"` Status string `json:"status"` // "ok" or "failed" @@ -37,8 +57,10 @@ type VerifyResult struct { } // deepVerifyFailure records a failure in the result and returns it appropriately -func (v *Vaultik) deepVerifyFailure(result *VerifyResult, opts *VerifyOptions, msg string, err error) error { - result.Status = "failed" +func (v *Vaultik) deepVerifyFailure( + result *VerifyResult, opts *VerifyOptions, msg string, err error, +) error { + result.Status = verifyStatusFailed result.ErrorMessage = msg if opts.JSON { @@ -49,7 +71,7 @@ func (v *Vaultik) deepVerifyFailure(result *VerifyResult, opts *VerifyOptions, m return err } - return fmt.Errorf("%s", msg) + return fmt.Errorf("%w: %s", errVerificationFailed, msg) } // RunDeepVerify executes deep verification operation @@ -60,15 +82,14 @@ func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error { } if !v.CanDecrypt() { - msg := "VAULTIK_AGE_SECRET_KEY not set; required for deep verification" - - return v.deepVerifyFailure(result, opts, msg, fmt.Errorf("%s", msg)) + return v.deepVerifyFailure(result, opts, + errSecretKeyRequired.Error(), errSecretKeyRequired) } log.Info("Starting snapshot verification", "snapshot_id", snapshotID, "mode", "deep") if !opts.JSON { - v.printfStdout("Deep verification of snapshot: %s\n\n", snapshotID) + v.stdoutf("Deep verification of snapshot: %s\n\n", snapshotID) } manifest, tempDB, dbBlobs, err := v.loadVerificationData(snapshotID, opts, result) @@ -104,16 +125,18 @@ func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error { log.Info("✓ Verification completed successfully", "snapshot_id", snapshotID, "mode", "deep", "blobs_verified", len(dbBlobs)) - v.printfStdout("\n✓ Verification completed successfully\n") - v.printfStdout(" Snapshot: %s\n", snapshotID) - v.printfStdout(" Blobs verified: %d\n", len(dbBlobs)) - v.printfStdout(" Total size: %s\n", humanize.Bytes(uint64(totalSize))) + v.stdoutf("\n✓ Verification completed successfully\n") + v.stdoutf(" Snapshot: %s\n", snapshotID) + v.stdoutf(" Blobs verified: %d\n", len(dbBlobs)) + v.stdoutf(" Total size: %s\n", ubytes(totalSize)) return nil } // loadVerificationData downloads manifest, database, and blob list for verification -func (v *Vaultik) loadVerificationData(snapshotID string, opts *VerifyOptions, result *VerifyResult) (*snapshot.Manifest, *tempDB, []snapshot.BlobInfo, error) { +func (v *Vaultik) loadVerificationData( + snapshotID string, opts *VerifyOptions, result *VerifyResult, +) (*snapshot.Manifest, *tempDB, []snapshot.BlobInfo, error) { // All remote paths use the hashed key derived from the human ID. remoteKey := snapshot.RemoteSnapshotKey(snapshotID) @@ -122,7 +145,7 @@ func (v *Vaultik) loadVerificationData(snapshotID string, opts *VerifyOptions, r log.Info("Downloading manifest", "path", manifestPath) if !opts.JSON { - v.printfStdout("Downloading manifest...\n") + v.stdoutf("Downloading manifest...\n") } manifestReader, err := v.Storage.Get(v.ctx, manifestPath) @@ -143,11 +166,12 @@ func (v *Vaultik) loadVerificationData(snapshotID string, opts *VerifyOptions, r log.Info("Manifest loaded", "manifest_blob_count", manifest.BlobCount, - "manifest_total_size", humanize.Bytes(uint64(manifest.TotalCompressedSize))) + "manifest_total_size", ubytes(manifest.TotalCompressedSize)) if !opts.JSON { - v.printfStdout("Manifest loaded: %d blobs (%s)\n", manifest.BlobCount, humanize.Bytes(uint64(manifest.TotalCompressedSize))) - v.printfStdout("Downloading and decrypting database...\n") + v.stdoutf("Manifest loaded: %d blobs (%s)\n", + manifest.BlobCount, ubytes(manifest.TotalCompressedSize)) + v.stdoutf("Downloading and decrypting database...\n") } // Download and decrypt database @@ -163,7 +187,7 @@ func (v *Vaultik) loadVerificationData(snapshotID string, opts *VerifyOptions, r defer func() { _ = dbReader.Close() }() - tdb, err := v.decryptAndLoadDatabase(dbReader, v.Config.AgeSecretKey) + tdb, err := v.decryptAndLoadDatabase(dbReader) if err != nil { return nil, nil, nil, v.deepVerifyFailure(result, opts, fmt.Sprintf("failed to decrypt database: %v", err), @@ -186,19 +210,28 @@ func (v *Vaultik) loadVerificationData(snapshotID string, opts *VerifyOptions, r log.Info("Database loaded", "db_blob_count", len(dbBlobs), - "db_total_size", humanize.Bytes(uint64(dbTotalSize))) + "db_total_size", ubytes(dbTotalSize)) if !opts.JSON { - v.printfStdout("Database loaded: %d blobs (%s)\n", len(dbBlobs), humanize.Bytes(uint64(dbTotalSize))) + v.stdoutf("Database loaded: %d blobs (%s)\n", + len(dbBlobs), ubytes(dbTotalSize)) } return manifest, tdb, dbBlobs, nil } -// runVerificationSteps executes manifest verification, blob existence check, and deep content verification -func (v *Vaultik) runVerificationSteps(manifest *snapshot.Manifest, dbBlobs []snapshot.BlobInfo, tdb *tempDB, opts *VerifyOptions, result *VerifyResult, totalSize int64) error { +// runVerificationSteps executes manifest verification, blob existence +// check, and deep content verification. +func (v *Vaultik) runVerificationSteps( + manifest *snapshot.Manifest, + dbBlobs []snapshot.BlobInfo, + tdb *tempDB, + opts *VerifyOptions, + result *VerifyResult, + totalSize int64, +) error { if !opts.JSON { - v.printfStdout("Verifying manifest against database...\n") + v.stdoutf("Verifying manifest against database...\n") } err := v.verifyManifestAgainstDatabase(manifest, dbBlobs) @@ -207,8 +240,8 @@ func (v *Vaultik) runVerificationSteps(manifest *snapshot.Manifest, dbBlobs []sn } if !opts.JSON { - v.printfStdout("Manifest verified.\n") - v.printfStdout("Checking blob existence in remote storage...\n") + v.stdoutf("Manifest verified.\n") + v.stdoutf("Checking blob existence in remote storage...\n") } err = v.verifyBlobExistenceFromDB(dbBlobs) @@ -217,8 +250,9 @@ func (v *Vaultik) runVerificationSteps(manifest *snapshot.Manifest, dbBlobs []sn } if !opts.JSON { - v.printfStdout("All blobs exist.\n") - v.printfStdout("Downloading and verifying blob contents (%d blobs, %s)...\n", len(dbBlobs), humanize.Bytes(uint64(totalSize))) + v.stdoutf("All blobs exist.\n") + v.stdoutf("Downloading and verifying blob contents (%d blobs, %s)...\n", + len(dbBlobs), ubytes(totalSize)) } err = v.performDeepVerificationFromDB(dbBlobs, tdb.DB, opts) @@ -243,8 +277,9 @@ func (t *tempDB) Close() error { return err } -// decryptAndLoadDatabase decrypts and loads the binary SQLite database from the encrypted stream -func (v *Vaultik) decryptAndLoadDatabase(reader io.ReadCloser, secretKey string) (*tempDB, error) { +// decryptAndLoadDatabase decrypts and loads the binary SQLite database +// from the encrypted stream. +func (v *Vaultik) decryptAndLoadDatabase(reader io.ReadCloser) (*tempDB, error) { // Get decryptor decryptor, err := v.GetDecryptor() if err != nil { @@ -285,7 +320,7 @@ func (v *Vaultik) decryptAndLoadDatabase(reader io.ReadCloser, secretKey string) _ = tempFile.Close() - log.Info("Database decompressed", "size", humanize.Bytes(uint64(written))) + log.Info("Database decompressed", "size", ubytes(written)) // Open the database db, err := sql.Open("sqlite", tempPath) @@ -346,15 +381,17 @@ func (v *Vaultik) verifyBlob(blobInfo snapshot.BlobInfo, db *sql.DB) error { log.Info("Blob verified", "hash", blobInfo.Hash[:16]+"...", "chunks", chunkCount, - "size", humanize.Bytes(uint64(blobInfo.CompressedSize)), + "size", ubytes(blobInfo.CompressedSize), ) return nil } -// verifyBlobChunks queries blob chunks from the database and verifies each chunk's hash -// against the decompressed blob stream -func (v *Vaultik) verifyBlobChunks(db *sql.DB, blobHash string, decompressor io.Reader) (int, error) { +// verifyBlobChunks queries blob chunks from the database and verifies +// each chunk's hash against the decompressed blob stream. +func (v *Vaultik) verifyBlobChunks( + db *sql.DB, blobHash string, decompressor io.Reader, +) (int, error) { query := ` SELECT bc.chunk_hash, bc.offset, bc.length FROM blob_chunks bc @@ -389,7 +426,8 @@ func (v *Vaultik) verifyBlobChunks(db *sql.DB, blobHash string, decompressor io. // Verify chunk ordering if offset <= lastOffset { - return 0, fmt.Errorf("chunks out of order: offset %d after %d", offset, lastOffset) + return 0, fmt.Errorf("%w: offset %d after %d", + errChunksOutOfOrder, offset, lastOffset) } lastOffset = offset @@ -423,8 +461,8 @@ func (v *Vaultik) verifyBlobChunks(db *sql.DB, blobHash string, decompressor io. calculatedHash := hex.EncodeToString(hasher.Sum(nil)) if calculatedHash != chunkHash { - return 0, fmt.Errorf("chunk hash mismatch at offset %d: calculated %s, expected %s", - offset, calculatedHash, chunkHash) + return 0, fmt.Errorf("%w at offset %d: calculated %s, expected %s", + errChunkHashMismatch, offset, calculatedHash, chunkHash) } chunkCount++ @@ -438,31 +476,37 @@ func (v *Vaultik) verifyBlobChunks(db *sql.DB, blobHash string, decompressor io. return chunkCount, nil } -// verifyBlobFinalIntegrity checks that no trailing data exists in the decompressed stream -// and that the encrypted blob hash matches the expected value -func (v *Vaultik) verifyBlobFinalIntegrity(decompressor io.Reader, blobHasher hash.Hash, expectedHash string) error { - // Verify no remaining data in blob - if chunk list is accurate, blob should be fully consumed +// verifyBlobFinalIntegrity checks that no trailing data exists in the +// decompressed stream and that the encrypted blob hash matches the +// expected value. +func (v *Vaultik) verifyBlobFinalIntegrity( + decompressor io.Reader, blobHasher hash.Hash, expectedHash string, +) error { + // Verify no remaining data in blob - if the chunk list is accurate, + // the blob should be fully consumed. remaining, err := io.Copy(io.Discard, decompressor) if err != nil { return fmt.Errorf("failed to check for remaining blob data: %w", err) } if remaining > 0 { - return fmt.Errorf("blob has %d unexpected trailing bytes not covered by chunk list", remaining) + return fmt.Errorf("%w: %d bytes", errTrailingBlobData, remaining) } // Verify blob hash matches the encrypted data we downloaded calculatedBlobHash := hex.EncodeToString(blobHasher.Sum(nil)) if calculatedBlobHash != expectedHash { - return fmt.Errorf("blob hash mismatch: calculated %s, expected %s", - calculatedBlobHash, expectedHash) + return fmt.Errorf("%w: calculated %s, expected %s", + errBlobHashMismatch, calculatedBlobHash, expectedHash) } return nil } // getBlobsFromDatabase gets all blobs for the snapshot from the database -func (v *Vaultik) getBlobsFromDatabase(snapshotID string, db *sql.DB) ([]snapshot.BlobInfo, error) { +func (v *Vaultik) getBlobsFromDatabase( + snapshotID string, db *sql.DB, +) ([]snapshot.BlobInfo, error) { query := ` SELECT b.blob_hash, b.compressed_size FROM snapshot_blobs sb @@ -505,8 +549,11 @@ func (v *Vaultik) getBlobsFromDatabase(snapshotID string, db *sql.DB) ([]snapsho return blobs, nil } -// verifyManifestAgainstDatabase verifies the manifest matches the authoritative database -func (v *Vaultik) verifyManifestAgainstDatabase(manifest *snapshot.Manifest, dbBlobs []snapshot.BlobInfo) error { +// verifyManifestAgainstDatabase verifies the manifest matches the +// authoritative database. +func (v *Vaultik) verifyManifestAgainstDatabase( + manifest *snapshot.Manifest, dbBlobs []snapshot.BlobInfo, +) error { log.Info("Verifying manifest against database") // Build map of database blobs @@ -534,12 +581,13 @@ func (v *Vaultik) verifyManifestAgainstDatabase(manifest *snapshot.Manifest, dbB for hash, manifestSize := range manifestBlobMap { dbSize, exists := dbBlobMap[hash] if !exists { - return fmt.Errorf("manifest contains blob %s not in database", hash) + return fmt.Errorf("%w: %s", errManifestExtraBlob, hash) } if dbSize != manifestSize { - return fmt.Errorf("blob %s size mismatch: database has %d bytes, manifest has %d bytes", - hash, dbSize, manifestSize) + return fmt.Errorf( + "%w: blob %s: database has %d bytes, manifest has %d bytes", + errBlobSizeMismatch, hash, dbSize, manifestSize) } } @@ -567,16 +615,18 @@ func (v *Vaultik) verifyBlobExistenceFromDB(blobs []snapshot.BlobInfo) error { // Verify size matches if stat.Size != blob.CompressedSize { - return fmt.Errorf("blob %s size mismatch: S3 has %d bytes, database has %d bytes", - blob.Hash, stat.Size, blob.CompressedSize) + return fmt.Errorf( + "%w: blob %s: S3 has %d bytes, database has %d bytes", + errBlobSizeMismatch, blob.Hash, stat.Size, blob.CompressedSize) } // Progress update every 100 blobs - if (i+1)%100 == 0 || i == len(blobs)-1 { + if (i+1)%progressLogEvery == 0 || i == len(blobs)-1 { log.Info("Blob existence check progress", "checked", i+1, "total", len(blobs), - "percent", fmt.Sprintf("%.1f%%", float64(i+1)/float64(len(blobs))*100), + "percent", fmt.Sprintf("%.1f%%", + float64(i+1)/float64(len(blobs))*percentScale), ) } } @@ -586,8 +636,11 @@ func (v *Vaultik) verifyBlobExistenceFromDB(blobs []snapshot.BlobInfo) error { return nil } -// performDeepVerificationFromDB downloads and verifies the content of each blob using database as source -func (v *Vaultik) performDeepVerificationFromDB(blobs []snapshot.BlobInfo, db *sql.DB, opts *VerifyOptions) error { +// performDeepVerificationFromDB downloads and verifies the content of +// each blob using the database as source. +func (v *Vaultik) performDeepVerificationFromDB( + blobs []snapshot.BlobInfo, db *sql.DB, opts *VerifyOptions, +) error { // Calculate total bytes for ETA var totalBytesExpected int64 for _, b := range blobs { @@ -596,7 +649,7 @@ func (v *Vaultik) performDeepVerificationFromDB(blobs []snapshot.BlobInfo, db *s log.Info("Starting deep verification - downloading and verifying all blobs", "blob_count", len(blobs), - "total_size", humanize.Bytes(uint64(totalBytesExpected)), + "total_size", ubytes(totalBytesExpected), ) startTime := time.Now() @@ -630,18 +683,18 @@ func (v *Vaultik) performDeepVerificationFromDB(blobs []snapshot.BlobInfo, db *s "blobs_total", len(blobs), "blobs_remaining", remaining, "bytes_done", bytesProcessed, - "bytes_done_human", humanize.Bytes(uint64(bytesProcessed)), + "bytes_done_human", ubytes(bytesProcessed), "bytes_total", totalBytesExpected, - "bytes_total_human", humanize.Bytes(uint64(totalBytesExpected)), + "bytes_total_human", ubytes(totalBytesExpected), "elapsed", elapsed.Round(time.Second), "eta", eta.Round(time.Second), ) if !opts.JSON { - v.printfStdout(" Verified %d/%d blobs (%d remaining) - %s/%s - elapsed %s, eta %s\n", + v.stdoutf(" Verified %d/%d blobs (%d remaining) - %s/%s - elapsed %s, eta %s\n", i+1, len(blobs), remaining, - humanize.Bytes(uint64(bytesProcessed)), - humanize.Bytes(uint64(totalBytesExpected)), + ubytes(bytesProcessed), + ubytes(totalBytesExpected), elapsed.Round(time.Second), eta.Round(time.Second)) } @@ -651,7 +704,7 @@ func (v *Vaultik) performDeepVerificationFromDB(blobs []snapshot.BlobInfo, db *s log.Info("✓ Deep verification completed successfully", "blobs_verified", len(blobs), "total_bytes", bytesProcessed, - "total_bytes_human", humanize.Bytes(uint64(bytesProcessed)), + "total_bytes_human", ubytes(bytesProcessed), "duration", totalElapsed.Round(time.Second), ) diff --git a/internal/vaultik/verify_test.go b/internal/vaultik/verify_test.go index c1fc1c3..75aab5f 100644 --- a/internal/vaultik/verify_test.go +++ b/internal/vaultik/verify_test.go @@ -18,6 +18,8 @@ import ( // bytes when streaming through age decryption and zstd decompression. // This validates the verification path: hash encrypted blob -> decrypt -> decompress. func TestTeeReaderWithDecryption(t *testing.T) { + t.Parallel() + // Test data - use random data that doesn't compress well (5MB) testData := make([]byte, 5*1024*1024) _, err := rand.Read(testData) @@ -26,7 +28,8 @@ func TestTeeReaderWithDecryption(t *testing.T) { // Compress the data var compressedBuf bytes.Buffer - compressor, err := zstd.NewWriter(&compressedBuf, zstd.WithEncoderLevel(zstd.SpeedDefault)) + compressor, err := zstd.NewWriter(&compressedBuf, + zstd.WithEncoderLevel(zstd.SpeedDefault)) require.NoError(t, err) _, err = compressor.Write(testData) require.NoError(t, err) @@ -34,8 +37,10 @@ func TestTeeReaderWithDecryption(t *testing.T) { require.NoError(t, err) // Encrypt the compressed data - testRecipient := "age1cplgrwj77ta54dnmydvvmzn64ltk83ankxl5sww04mrtmu62kv3s89gmvv" - testSecretKey := "AGE-SECRET-KEY-1C77PYNTHXSHNNC6EYR2W52UWYXACXA5JT00J9CCW9986M3XY87PSGP89AQ" + testRecipient := "age1cplgrwj77ta54dnmydvvmzn64ltk83ankxl5sww04mrt" + + "mu62kv3s89gmvv" + testSecretKey := "AGE-SECRET-KEY-1C77PYNTHXSHNNC6EYR2W52UWYXACXA5J" + + "T00J9CCW9986M3XY87PSGP89AQ" encryptor, err := crypto.NewEncryptor([]string{testRecipient}) require.NoError(t, err)