diff --git a/.golangci.yml b/.golangci.yml index 34a8e31..26b1610 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,5 +1,9 @@ version: "2" +# Config schema uses the golangci-lint v2 layout (settings live under +# linters.settings, not top-level linters-settings) so that the +# thresholds below are actually applied by golangci-lint >= v2. + run: timeout: 5m modules-download-mode: readonly @@ -14,19 +18,17 @@ linters: - wsl # Deprecated, replaced by wsl_v5 - wrapcheck # Too verbose for internal packages - varnamelen # Short names like db, id are idiomatic Go - -linters-settings: - lll: - line-length: 88 - funlen: - lines: 80 - statements: 50 - cyclop: - max-complexity: 15 - dupl: - threshold: 100 + settings: + lll: + line-length: 88 + funlen: + lines: 80 + statements: 50 + cyclop: + max-complexity: 15 + dupl: + threshold: 100 issues: - exclude-use-default: false max-issues-per-linter: 0 max-same-issues: 0 diff --git a/Dockerfile b/Dockerfile index 72ea9b2..cebd9d9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ # Lint stage -# golangci/golangci-lint:v2.11.3-alpine, 2026-03-17 -FROM golangci/golangci-lint:v2.11.3-alpine@sha256:b1c3de5862ad0a95b4e45a993b0f00415835d687e4f12c845c7493b86c13414e AS lint +# golangci/golangci-lint:v2.12.2-alpine, 2026-08-07 +FROM golangci/golangci-lint:v2.12.2-alpine@sha256:91b27804074a0bacea298707f016911e60cf0cdbc6c7bf5ccacb5f0606d18d60 AS lint RUN apk add --no-cache make build-base diff --git a/Makefile b/Makefile index c026ebf..e70ab9e 100644 --- a/Makefile +++ b/Makefile @@ -55,7 +55,7 @@ clean: # Install dependencies. deps: go mod download - go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest + go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.12.2 # Run tests with coverage. test-coverage: diff --git a/cmd/vaultik/main.go b/cmd/vaultik/main.go index 70e7c19..234de24 100644 --- a/cmd/vaultik/main.go +++ b/cmd/vaultik/main.go @@ -16,9 +16,11 @@ func main() { panic("could not create CPU profile: " + err.Error()) } defer func() { _ = f.Close() }() + if err := pprof.StartCPUProfile(f); err != nil { panic("could not start CPU profile: " + err.Error()) } + defer pprof.StopCPUProfile() } @@ -30,7 +32,9 @@ func main() { panic("could not create memory profile: " + err.Error()) } defer func() { _ = f.Close() }() + runtime.GC() // get up-to-date statistics + if err := pprof.WriteHeapProfile(f); err != nil { panic("could not write memory profile: " + err.Error()) } diff --git a/internal/blob/packer.go b/internal/blob/packer.go index 116c365..1e02096 100644 --- a/internal/blob/packer.go +++ b/internal/blob/packer.go @@ -18,6 +18,7 @@ import ( "context" "database/sql" "encoding/hex" + "errors" "fmt" "io" "sync" @@ -124,6 +125,7 @@ type BlobChunkRef struct { // BlobWithReader wraps a FinishedBlob with its data reader type BlobWithReader struct { *FinishedBlob + Reader io.ReadSeeker TempFile afero.File // Optional, only set for disk-based blobs InsertedChunkHashes []string // Chunk hashes that were inserted to DB with this blob @@ -134,14 +136,17 @@ type BlobWithReader struct { // Returns an error if required configuration fields are missing or invalid. func NewPacker(cfg PackerConfig) (*Packer, error) { if len(cfg.Recipients) == 0 { - return nil, fmt.Errorf("recipients are required - blobs must be encrypted") + return nil, errors.New("recipients are required - blobs must be encrypted") } + if cfg.MaxBlobSize <= 0 { - return nil, fmt.Errorf("max blob size must be positive") + return nil, errors.New("max blob size must be positive") } + if cfg.Fs == nil { - return nil, fmt.Errorf("filesystem is required") + return nil, errors.New("filesystem is required") } + return &Packer{ maxBlobSize: cfg.MaxBlobSize, compressionLevel: cfg.CompressionLevel, @@ -160,6 +165,7 @@ func NewPacker(cfg PackerConfig) (*Packer, error) { func (p *Packer) SetBlobHandler(handler BlobHandler) { p.mu.Lock() defer p.mu.Unlock() + p.blobHandler = handler } @@ -169,6 +175,7 @@ func (p *Packer) SetBlobHandler(handler BlobHandler) { func (p *Packer) AddPendingChunk(hash string, size int64) { p.mu.Lock() defer p.mu.Unlock() + p.pendingChunks = append(p.pendingChunks, PendingChunk{Hash: hash, Size: size}) } @@ -183,7 +190,8 @@ func (p *Packer) AddChunk(chunk *ChunkRef) error { // Initialize new blob if needed if p.currentBlob == nil { - if err := p.startNewBlob(); err != nil { + err := p.startNewBlob() + if err != nil { return fmt.Errorf("starting new blob: %w", err) } } @@ -202,7 +210,8 @@ func (p *Packer) AddChunk(chunk *ChunkRef) error { } // Add chunk to current blob - if err := p.addChunkToCurrentBlob(chunk); err != nil { + err := p.addChunkToCurrentBlob(chunk) + if err != nil { return err } @@ -218,7 +227,8 @@ func (p *Packer) Flush() error { defer p.mu.Unlock() if p.currentBlob != nil && len(p.currentBlob.chunks) > 0 { - if err := p.finalizeCurrentBlob(); err != nil { + err := p.finalizeCurrentBlob() + if err != nil { return fmt.Errorf("finalizing blob: %w", err) } } @@ -253,6 +263,7 @@ func (p *Packer) GetFinishedBlobs() []*FinishedBlob { blobs := p.finishedBlobs p.finishedBlobs = make([]*FinishedBlob, 0) + return blobs } @@ -267,6 +278,7 @@ func (p *Packer) startNewBlob() error { if err != nil { return fmt.Errorf("parsing blob ID: %w", err) } + blob := &database.Blob{ ID: blobIDTyped, Hash: types.BlobHash("temp-placeholder-" + blobID), // Temporary placeholder until finalized @@ -276,6 +288,7 @@ func (p *Packer) startNewBlob() error { CompressedSize: 0, UploadedTS: nil, } + if err := p.repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error { return p.repos.Blobs.Create(ctx, tx, blob) }); err != nil { @@ -294,6 +307,7 @@ func (p *Packer) startNewBlob() error { if err != nil { _ = tempFile.Close() _ = p.fs.Remove(tempFile.Name()) + return fmt.Errorf("creating blobgen writer: %w", err) } @@ -308,6 +322,7 @@ func (p *Packer) startNewBlob() error { } log.Debug("Created new blob container", "blob_id", blobID, "temp_file", tempFile.Name()) + return nil } @@ -316,6 +331,7 @@ func (p *Packer) addChunkToCurrentBlob(chunk *ChunkRef) error { // Skip if chunk already in current blob if p.currentBlob.chunkSet[chunk.Hash] { log.Debug("Skipping duplicate chunk already in current blob", "chunk_hash", chunk.Hash) + return nil } @@ -403,24 +419,31 @@ func (p *Packer) finalizeCurrentBlob() error { func (p *Packer) closeBlobWriter() (string, int64, error) { if err := p.currentBlob.writer.Close(); err != nil { p.cleanupTempFile() + return "", 0, fmt.Errorf("closing blobgen writer: %w", err) } + if err := p.currentBlob.tempFile.Sync(); err != nil { p.cleanupTempFile() + return "", 0, fmt.Errorf("syncing temp file: %w", err) } finalSize, err := p.currentBlob.tempFile.Seek(0, io.SeekCurrent) if err != nil { p.cleanupTempFile() + return "", 0, fmt.Errorf("getting file size: %w", err) } + if _, err := p.currentBlob.tempFile.Seek(0, io.SeekStart); err != nil { p.cleanupTempFile() + return "", 0, fmt.Errorf("seeking to start: %w", err) } finalHash := p.currentBlob.writer.Sum256() + return hex.EncodeToString(finalHash), finalSize, nil } @@ -432,6 +455,7 @@ func (p *Packer) buildChunkRefs() []*BlobChunkRef { ChunkHash: chunk.Hash, Offset: chunk.Offset, Length: chunk.Size, }) } + return refs } @@ -444,13 +468,16 @@ func (p *Packer) commitBlobToDatabase(blobHash string, finalSize int64, chunksTo blobIDTyped, parseErr := types.ParseBlobID(p.currentBlob.id) if parseErr != nil { p.cleanupTempFile() + return fmt.Errorf("parsing blob ID: %w", parseErr) } err := p.repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error { for _, chunk := range chunksToInsert { dbChunk := &database.Chunk{ChunkHash: types.ChunkHash(chunk.Hash), Size: chunk.Size} - if err := p.repos.Chunks.Create(ctx, tx, dbChunk); err != nil { + + err := p.repos.Chunks.Create(ctx, tx, dbChunk) + if err != nil { return fmt.Errorf("creating chunk: %w", err) } } @@ -460,7 +487,9 @@ func (p *Packer) commitBlobToDatabase(blobHash string, finalSize int64, chunksTo BlobID: blobIDTyped, ChunkHash: types.ChunkHash(chunk.Hash), Offset: chunk.Offset, Length: chunk.Size, } - if err := p.repos.BlobChunks.Create(ctx, tx, blobChunk); err != nil { + + err := p.repos.BlobChunks.Create(ctx, tx, blobChunk) + if err != nil { return fmt.Errorf("creating blob_chunk: %w", err) } } @@ -469,11 +498,13 @@ func (p *Packer) commitBlobToDatabase(blobHash string, finalSize int64, chunksTo }) if err != nil { p.cleanupTempFile() + return fmt.Errorf("finalizing blob transaction: %w", err) } log.Debug("Committed blob transaction", "chunks_inserted", len(chunksToInsert), "blob_chunks_inserted", len(p.currentBlob.chunks)) + return nil } @@ -482,6 +513,7 @@ func (p *Packer) deliverFinishedBlob(finished *FinishedBlob, insertedChunkHashes if p.blobHandler != nil { if _, err := p.currentBlob.tempFile.Seek(0, io.SeekStart); err != nil { p.cleanupTempFile() + return fmt.Errorf("seeking for handler: %w", err) } @@ -492,30 +524,39 @@ func (p *Packer) deliverFinishedBlob(finished *FinishedBlob, insertedChunkHashes InsertedChunkHashes: insertedChunkHashes, } - if err := p.blobHandler(blobWithReader); err != nil { + err := p.blobHandler(blobWithReader) + if err != nil { p.cleanupTempFile() + return fmt.Errorf("blob handler failed: %w", err) } + p.currentBlob = nil + return nil } // No handler - read data for legacy behavior log.Debug("No blob handler callback configured", "blob_hash", finished.Hash[:8]+"...") + if _, err := p.currentBlob.tempFile.Seek(0, io.SeekStart); err != nil { p.cleanupTempFile() + return fmt.Errorf("seeking to read data: %w", err) } data, err := io.ReadAll(p.currentBlob.tempFile) if err != nil { p.cleanupTempFile() + return fmt.Errorf("reading blob data: %w", err) } + finished.Data = data p.finishedBlobs = append(p.finishedBlobs, finished) p.cleanupTempFile() p.currentBlob = nil + return nil } @@ -532,13 +573,15 @@ func (p *Packer) cleanupTempFile() { func (p *Packer) PackChunks(chunks []*ChunkRef) error { for _, chunk := range chunks { err := p.AddChunk(chunk) - if err == ErrBlobSizeLimitExceeded { + if errors.Is(err, ErrBlobSizeLimitExceeded) { // Finalize current blob and retry - if err := p.FinalizeBlob(); err != nil { + err := p.FinalizeBlob() + if err != nil { return fmt.Errorf("finalizing blob before retry: %w", err) } // Retry the chunk - if err := p.AddChunk(chunk); err != nil { + err = p.AddChunk(chunk) + if err != nil { return fmt.Errorf("adding chunk %s after finalize: %w", chunk.Hash, err) } } else if err != nil { diff --git a/internal/blob/packer_test.go b/internal/blob/packer_test.go index dba194b..d44bc22 100644 --- a/internal/blob/packer_test.go +++ b/internal/blob/packer_test.go @@ -6,6 +6,7 @@ import ( "crypto/sha256" "database/sql" "encoding/hex" + "errors" "io" "testing" @@ -40,6 +41,7 @@ func TestPacker(t *testing.T) { t.Fatalf("failed to create test db: %v", err) } defer func() { _ = db.Close() }() + repos := database.NewRepositories(db) cfg := PackerConfig{ @@ -49,6 +51,7 @@ func TestPacker(t *testing.T) { Repositories: repos, Fs: afero.NewMemMapFs(), } + packer, err := NewPacker(cfg) if err != nil { t.Fatalf("failed to create packer: %v", err) @@ -64,6 +67,7 @@ func TestPacker(t *testing.T) { ChunkHash: types.ChunkHash(hashStr), Size: int64(len(data)), } + err = repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error { return repos.Chunks.Create(ctx, tx, dbChunk) }) @@ -130,6 +134,7 @@ func TestPacker(t *testing.T) { t.Fatalf("failed to create test db: %v", err) } defer func() { _ = db.Close() }() + repos := database.NewRepositories(db) cfg := PackerConfig{ @@ -139,6 +144,7 @@ func TestPacker(t *testing.T) { Repositories: repos, Fs: afero.NewMemMapFs(), } + packer, err := NewPacker(cfg) if err != nil { t.Fatalf("failed to create packer: %v", err) @@ -146,7 +152,8 @@ func TestPacker(t *testing.T) { // Create multiple small chunks chunks := make([]*ChunkRef, 10) - for i := 0; i < 10; i++ { + + for i := range 10 { data := bytes.Repeat([]byte{byte(i)}, 1000) hash := sha256.Sum256(data) hashStr := hex.EncodeToString(hash[:]) @@ -156,6 +163,7 @@ func TestPacker(t *testing.T) { ChunkHash: types.ChunkHash(hashStr), Size: int64(len(data)), } + err = repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error { return repos.Chunks.Create(ctx, tx, dbChunk) }) @@ -198,9 +206,11 @@ func TestPacker(t *testing.T) { if chunkRef.Offset != expectedOffset { t.Errorf("chunk %d: expected offset %d, got %d", i, expectedOffset, chunkRef.Offset) } + if chunkRef.Length != 1000 { t.Errorf("chunk %d: expected length 1000, got %d", i, chunkRef.Length) } + expectedOffset += chunkRef.Length } }) @@ -212,6 +222,7 @@ func TestPacker(t *testing.T) { t.Fatalf("failed to create test db: %v", err) } defer func() { _ = db.Close() }() + repos := database.NewRepositories(db) // Small blob size limit to force multiple blobs @@ -222,6 +233,7 @@ func TestPacker(t *testing.T) { Repositories: repos, Fs: afero.NewMemMapFs(), } + packer, err := NewPacker(cfg) if err != nil { t.Fatalf("failed to create packer: %v", err) @@ -229,7 +241,8 @@ func TestPacker(t *testing.T) { // Create chunks that will exceed the limit chunks := make([]*ChunkRef, 10) - for i := 0; i < 10; i++ { + + for i := range 10 { data := bytes.Repeat([]byte{byte(i)}, 1000) // 1KB each hash := sha256.Sum256(data) hashStr := hex.EncodeToString(hash[:]) @@ -239,6 +252,7 @@ func TestPacker(t *testing.T) { ChunkHash: types.ChunkHash(hashStr), Size: int64(len(data)), } + err = repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error { return repos.Chunks.Create(ctx, tx, dbChunk) }) @@ -257,14 +271,17 @@ func TestPacker(t *testing.T) { // Add chunks and handle size limit errors for _, chunk := range chunks { err := packer.AddChunk(chunk) - if err == ErrBlobSizeLimitExceeded { + if errors.Is(err, ErrBlobSizeLimitExceeded) { // Finalize current blob - if err := packer.FinalizeBlob(); err != nil { + err := packer.FinalizeBlob() + if err != nil { t.Fatalf("failed to finalize blob: %v", err) } + blobCount++ // Retry adding the chunk - if err := packer.AddChunk(chunk); err != nil { + err = packer.AddChunk(chunk) + if err != nil { t.Fatalf("failed to add chunk after finalize: %v", err) } } else if err != nil { @@ -301,6 +318,7 @@ func TestPacker(t *testing.T) { t.Fatalf("failed to create test db: %v", err) } defer func() { _ = db.Close() }() + repos := database.NewRepositories(db) // Generate test identity (using the one from parent test) @@ -311,6 +329,7 @@ func TestPacker(t *testing.T) { Repositories: repos, Fs: afero.NewMemMapFs(), } + packer, err := NewPacker(cfg) if err != nil { t.Fatalf("failed to create packer: %v", err) @@ -326,6 +345,7 @@ func TestPacker(t *testing.T) { ChunkHash: types.ChunkHash(hashStr), Size: int64(len(data)), } + err = repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error { return repos.Chunks.Create(ctx, tx, dbChunk) }) @@ -342,6 +362,7 @@ func TestPacker(t *testing.T) { if err := packer.AddChunk(chunk); err != nil { t.Fatalf("failed to add chunk: %v", err) } + if err := packer.Flush(); err != nil { t.Fatalf("failed to flush: %v", err) } diff --git a/internal/blobgen/compress.go b/internal/blobgen/compress.go index e8a8799..20f8571 100644 --- a/internal/blobgen/compress.go +++ b/internal/blobgen/compress.go @@ -28,6 +28,7 @@ func CompressData(data []byte, compressionLevel int, recipients []string) (*Comp // Write data if _, err := w.Write(data); err != nil { _ = w.Close() + return nil, fmt.Errorf("writing data: %w", err) } @@ -68,6 +69,7 @@ func CompressStream(dst io.Writer, src io.Reader, compressionLevel int, recipien if err := w.Close(); err != nil { return 0, "", fmt.Errorf("closing writer: %w", err) } + closed = true return w.BytesWritten(), hex.EncodeToString(w.Sum256()), nil diff --git a/internal/blobgen/compress_test.go b/internal/blobgen/compress_test.go index 6d1240c..37c41e9 100644 --- a/internal/blobgen/compress_test.go +++ b/internal/blobgen/compress_test.go @@ -20,13 +20,14 @@ const testRecipient = "age1cplgrwj77ta54dnmydvvmzn64ltk83ankxl5sww04mrtmu62kv3s8 // cause a double close. func TestCompressStreamNoDoubleClose(t *testing.T) { input := []byte("regression test data for issue #28 double-close fix") + var buf bytes.Buffer written, hash, err := CompressStream(&buf, bytes.NewReader(input), 3, []string{testRecipient}) require.NoError(t, err, "CompressStream should not return an error") - assert.True(t, written > 0, "expected bytes written > 0") + assert.Positive(t, written, "expected bytes written > 0") assert.NotEmpty(t, hash, "expected non-empty hash") - assert.True(t, buf.Len() > 0, "expected non-empty output") + assert.Positive(t, buf.Len(), "expected non-empty output") } // TestCompressStreamLargeInput exercises CompressStream with a larger payload @@ -37,9 +38,10 @@ func TestCompressStreamLargeInput(t *testing.T) { require.NoError(t, err) var buf bytes.Buffer + written, hash, err := CompressStream(&buf, bytes.NewReader(data), 3, []string{testRecipient}) require.NoError(t, err) - assert.True(t, written > 0) + assert.Positive(t, written) assert.NotEmpty(t, hash) } @@ -47,6 +49,7 @@ func TestCompressStreamLargeInput(t *testing.T) { // without double-close issues. func TestCompressStreamEmptyInput(t *testing.T) { var buf bytes.Buffer + _, hash, err := CompressStream(&buf, strings.NewReader(""), 3, []string{testRecipient}) require.NoError(t, err) assert.NotEmpty(t, hash) @@ -58,7 +61,7 @@ func TestCompressDataNoDoubleClose(t *testing.T) { input := []byte("CompressData regression test for double-close") result, err := CompressData(input, 3, []string{testRecipient}) require.NoError(t, err) - assert.True(t, result.CompressedSize > 0) - assert.True(t, result.UncompressedSize == int64(len(input))) + assert.Positive(t, result.CompressedSize) + assert.Equal(t, result.UncompressedSize, int64(len(input))) assert.NotEmpty(t, result.SHA256) } diff --git a/internal/blobgen/reader.go b/internal/blobgen/reader.go index cc3edd4..6d7f2f8 100644 --- a/internal/blobgen/reader.go +++ b/internal/blobgen/reader.go @@ -53,12 +53,14 @@ func NewReader(r io.Reader, identity age.Identity) (*Reader, error) { func (r *Reader) Read(p []byte) (n int, err error) { n, err = r.teeReader.Read(p) r.bytesRead += int64(n) + return n, err } // Close closes the decompressor func (r *Reader) Close() error { r.decompressor.Close() + return nil } diff --git a/internal/blobgen/writer.go b/internal/blobgen/writer.go index 920a572..a14574c 100644 --- a/internal/blobgen/writer.go +++ b/internal/blobgen/writer.go @@ -36,11 +36,13 @@ func NewWriter(w io.Writer, compressionLevel int, recipients []string) (*Writer, // Parse recipients var ageRecipients []age.Recipient + for _, recipient := range recipients { r, err := age.ParseX25519Recipient(recipient) if err != nil { return nil, fmt.Errorf("parsing recipient %s: %w", recipient, err) } + ageRecipients = append(ageRecipients, r) } @@ -51,10 +53,7 @@ func NewWriter(w io.Writer, compressionLevel int, recipients []string) (*Writer, } // Calculate compression concurrency: CPUs - 2, minimum 1 - concurrency := runtime.NumCPU() - 2 - if concurrency < 1 { - concurrency = 1 - } + concurrency := max(runtime.NumCPU()-2, 1) // Create compression writer with encryption as destination compressor, err := zstd.NewWriter(encWriter, @@ -63,6 +62,7 @@ func NewWriter(w io.Writer, compressionLevel int, recipients []string) (*Writer, ) if err != nil { _ = encWriter.Close() + return nil, fmt.Errorf("creating compression writer: %w", err) } @@ -82,18 +82,21 @@ func NewWriter(w io.Writer, compressionLevel int, recipients []string) (*Writer, func (w *Writer) Write(p []byte) (n int, err error) { n, err = w.teeWriter.Write(p) w.bytesWritten += int64(n) + return n, err } // Close closes all layers and returns any errors func (w *Writer) Close() error { // Close compressor first - if err := w.compressor.Close(); err != nil { + err := w.compressor.Close() + if err != nil { return fmt.Errorf("closing compressor: %w", err) } // Then close encryptor - if err := w.encryptor.Close(); err != nil { + err = w.encryptor.Close() + if err != nil { return fmt.Errorf("closing encryptor: %w", err) } @@ -109,6 +112,7 @@ func (w *Writer) Sum256() []byte { firstHash := w.hasher.Sum(nil) // Second hash: SHA256(firstHash) - this is the blob ID secondHash := sha256.Sum256(firstHash) + return secondHash[:] } @@ -123,5 +127,6 @@ func validateCompressionLevel(level int) error { if level < 1 || level > 19 { return fmt.Errorf("invalid compression level %d: must be between 1 and 19", level) } + return nil } diff --git a/internal/chunker/chunker.go b/internal/chunker/chunker.go index 3995990..1615899 100644 --- a/internal/chunker/chunker.go +++ b/internal/chunker/chunker.go @@ -3,6 +3,7 @@ package chunker import ( "crypto/sha256" "encoding/hex" + "errors" "fmt" "io" "os" @@ -50,13 +51,15 @@ func (c *Chunker) ChunkReader(r io.Reader) ([]Chunk, error) { defer chunker.Release() var chunks []Chunk + offset := int64(0) for { chunk, err := chunker.Next() - if err == io.EOF { + if errors.Is(err, io.EOF) { break } + if err != nil { return nil, fmt.Errorf("reading chunk: %w", err) } @@ -104,9 +107,10 @@ func (c *Chunker) ChunkReaderStreaming(r io.Reader, callback ChunkCallback) (str for { chunk, err := chunker.Next() - if err == io.EOF { + if errors.Is(err, io.EOF) { break } + if err != nil { return "", fmt.Errorf("reading chunk: %w", err) } @@ -143,7 +147,8 @@ func (c *Chunker) ChunkFile(path string) ([]Chunk, error) { return nil, fmt.Errorf("opening file: %w", err) } defer func() { - if err := file.Close(); err != nil && err.Error() != "invalid argument" { + err := file.Close() + if err != nil && err.Error() != "invalid argument" { // Log error or handle as needed _ = err } diff --git a/internal/chunker/chunker_isolated_test.go b/internal/chunker/chunker_isolated_test.go index 8e3fb0f..b5f86bf 100644 --- a/internal/chunker/chunker_isolated_test.go +++ b/internal/chunker/chunker_isolated_test.go @@ -42,7 +42,7 @@ func TestChunkerExpectedChunkCount(t *testing.T) { // Create data with some variation to trigger chunk boundaries data := make([]byte, tt.fileSize) - for i := 0; i < len(data); i++ { + for i := range data { // Use a pattern that should create boundaries data[i] = byte((i * 17) ^ (i >> 5)) } @@ -59,6 +59,7 @@ func TestChunkerExpectedChunkCount(t *testing.T) { t.Errorf("too few chunks: got %d, expected at least %d", len(chunks), tt.minExpected) } + if len(chunks) > tt.maxExpected { t.Errorf("too many chunks: got %d, expected at most %d", len(chunks), tt.maxExpected) @@ -69,6 +70,7 @@ func TestChunkerExpectedChunkCount(t *testing.T) { for _, chunk := range chunks { reconstructed = append(reconstructed, chunk.Data...) } + if !bytes.Equal(data, reconstructed) { t.Error("reconstructed data doesn't match original") } diff --git a/internal/chunker/chunker_test.go b/internal/chunker/chunker_test.go index a13e143..fca59f2 100644 --- a/internal/chunker/chunker_test.go +++ b/internal/chunker/chunker_test.go @@ -60,6 +60,7 @@ func TestChunker(t *testing.T) { if chunk.Offset != expectedOffset { t.Errorf("chunk %d: expected offset %d, got %d", i, expectedOffset, chunk.Offset) } + expectedOffset += chunk.Size } }) @@ -90,6 +91,7 @@ func TestChunker(t *testing.T) { if chunks1[i].Hash != chunks2[i].Hash { t.Errorf("chunk %d: different hashes", i) } + if chunks1[i].Size != chunks2[i].Size { t.Errorf("chunk %d: different sizes", i) } @@ -121,6 +123,7 @@ func TestChunkBoundaries(t *testing.T) { if i < len(chunks)-1 && chunk.Size < minSize { t.Errorf("chunk %d size %d is below minimum %d", i, chunk.Size, minSize) } + if chunk.Size > maxSize { t.Errorf("chunk %d size %d exceeds maximum %d", i, chunk.Size, maxSize) } diff --git a/internal/chunker/fastcdc.go b/internal/chunker/fastcdc.go index 54c87c7..223eb2b 100644 --- a/internal/chunker/fastcdc.go +++ b/internal/chunker/fastcdc.go @@ -28,7 +28,7 @@ type ReusableChunker struct { // reusableChunkerPool pools ReusableChunker instances to avoid allocations. var reusableChunkerPool = sync.Pool{ - New: func() interface{} { + New: func() any { return &ReusableChunker{} }, } @@ -39,17 +39,20 @@ var bufferPools = sync.Map{} func getBuffer(size int) []byte { poolI, _ := bufferPools.LoadOrStore(size, &sync.Pool{ - New: func() interface{} { + New: func() any { buf := make([]byte, size) + return &buf }, }) pool := poolI.(*sync.Pool) + return *pool.Get().(*[]byte) } func putBuffer(buf []byte) { size := cap(buf) + poolI, ok := bufferPools.Load(size) if ok { pool := poolI.(*sync.Pool) @@ -77,6 +80,7 @@ func AcquireReusableChunker(rd io.Reader, minSize, avgSize, maxSize int) *Reusab if c.buf != nil { putBuffer(c.buf) } + c.buf = getBuffer(bufSize) } else { // Restore buffer to full capacity (may have been truncated by previous EOF) @@ -120,6 +124,7 @@ func (c *ReusableChunker) fillBuffer() error { if c.eof { c.buf = c.buf[:n] + return nil } @@ -134,15 +139,18 @@ func (c *ReusableChunker) fillBuffer() error { } else if err != nil { return err } + return nil } // Next returns the next chunk or io.EOF when done. // The returned Data slice is only valid until the next call to Next. func (c *ReusableChunker) Next() (FastCDCChunk, error) { - if err := c.fillBuffer(); err != nil { + err := c.fillBuffer() + if err != nil { return FastCDCChunk{}, err } + if len(c.buf) == 0 { return FastCDCChunk{}, io.EOF } @@ -189,13 +197,6 @@ func (c *ReusableChunker) nextChunk(data []byte) (int, uint64) { return i, fp } -func min(a, b int) int { - if a < b { - return a - } - return b -} - // 256 random uint64s for the rolling hash function (from FastCDC paper) var table = [256]uint64{ 0xe80e8d55032474b3, 0x11b25b61f5924e15, 0x03aa5bd82a9eb669, 0xc45a153ef107a38c, diff --git a/internal/cli/app.go b/internal/cli/app.go index c1da377..36ae64f 100644 --- a/internal/cli/app.go +++ b/internal/cli/app.go @@ -44,9 +44,11 @@ func setupGlobals(lc fx.Lifecycle, g *globals.Globals, v *vaultik.Vaultik, opts lc.Append(fx.Hook{ OnStart: func(ctx context.Context) error { g.StartTime = time.Now().UTC() + if opts.Cron || opts.Quiet { v.UI.SetQuiet(true) } + return nil }, }) @@ -56,12 +58,12 @@ func setupGlobals(lc fx.Lifecycle, g *globals.Globals, v *vaultik.Vaultik, opts // blank line. Used both from the fx hook (for subcommand invocations) and // from the root cobra Run handler (for `vaultik` with no subcommand). func writeStartupBanner(w *ui.Writer, startTime time.Time, shortCommit string) { - w.Banner("%s %s by %s (commit %s, built on %s) starting up at %s.", + w.Bannerf("%s %s by %s (commit %s, built on %s) starting up at %s.", globals.Appname, globals.Version, globals.Author, shortCommit, globals.CommitDate, startTime.Format(time.RFC3339)) - w.Banner("%s", globals.Homepage) - w.Banner("") + w.Bannerf("%s", globals.Homepage) + w.Bannerf("") } // NewApp creates a new fx application with common modules. @@ -105,6 +107,7 @@ func cleanStartupError(err error) error { if idx := strings.LastIndex(msg, "): "); idx >= 0 { msg = msg[idx+3:] } + return errors.New(msg) } @@ -122,7 +125,8 @@ func RunApp(ctx context.Context, app *fx.App) error { defer cancel() // Start the app - if err := app.Start(ctx); err != nil { + err := app.Start(ctx) + if err != nil { return cleanStartupError(err) } @@ -130,6 +134,7 @@ func RunApp(ctx context.Context, app *fx.App) error { shutdownComplete := make(chan struct{}) go func() { defer close(shutdownComplete) + <-sigChan log.Notice("Received interrupt signal, shutting down gracefully...") @@ -137,7 +142,8 @@ func RunApp(ctx context.Context, app *fx.App) error { shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 30*time.Second) defer shutdownCancel() - if err := app.Stop(shutdownCtx); err != nil { + err := app.Stop(shutdownCtx) + if err != nil { log.Error("Error during shutdown", "error", err) } }() @@ -149,9 +155,11 @@ func RunApp(ctx context.Context, app *fx.App) error { return nil case <-ctx.Done(): // Context cancelled (shouldn't happen in normal operation) - if err := app.Stop(context.Background()); err != nil { + err := app.Stop(context.Background()) + if err != nil { log.Error("Error stopping app", "error", err) } + return ctx.Err() case <-app.Done(): // App finished running (e.g., backup completed) @@ -166,19 +174,24 @@ func RunApp(ctx context.Context, app *fx.App) error { func RunWithApp(ctx context.Context, opts AppOptions) error { // Acquire PID lock to prevent concurrent instances lockDir := filepath.Join(xdg.DataHome, "vaultik") + lock, err := pidlock.Acquire(lockDir) if err != nil { if errors.Is(err, pidlock.ErrAlreadyRunning) { return fmt.Errorf("cannot start: %w", err) } + return fmt.Errorf("failed to acquire lock: %w", err) } + defer func() { - if err := lock.Release(); err != nil { + err := lock.Release() + if err != nil { log.Warn("Failed to release PID lock", "error", err) } }() app := NewApp(opts) + return RunApp(ctx, app) } diff --git a/internal/cli/config.go b/internal/cli/config.go index 1dc2c4f..c1bc9c4 100644 --- a/internal/cli/config.go +++ b/internal/cli/config.go @@ -1,6 +1,7 @@ package cli import ( + "errors" "fmt" "os" "os/exec" @@ -240,16 +241,20 @@ on macOS, ~/.config/ on Linux, /etc/vaultik/ as root).`, } dir := filepath.Dir(path) - if err := os.MkdirAll(dir, 0o755); err != nil { + + err := os.MkdirAll(dir, 0o755) + if err != nil { return fmt.Errorf("creating config directory %s: %w", dir, err) } - if err := os.WriteFile(path, []byte(defaultConfigTemplate), 0o600); err != nil { + err = os.WriteFile(path, []byte(defaultConfigTemplate), 0o600) + if err != nil { return fmt.Errorf("writing config file: %w", err) } fmt.Printf("Config written to %s\n", path) fmt.Println("Edit it to set your age_recipients, snapshots, and storage_url.") + return nil }, } @@ -276,6 +281,7 @@ func newConfigEditCommand() *cobra.Command { ed.Stdin = os.Stdin ed.Stdout = os.Stdout ed.Stderr = os.Stderr + return ed.Run() }, } @@ -305,6 +311,7 @@ func newConfigGetCommand() *cobra.Command { if node.Kind == yaml.ScalarNode { fmt.Println(node.Value) + return nil } @@ -312,7 +319,9 @@ func newConfigGetCommand() *cobra.Command { if err != nil { return fmt.Errorf("marshaling value: %w", err) } + fmt.Print(string(out)) + return nil }, } @@ -363,6 +372,7 @@ Examples: } fmt.Printf("%s = %s\n", args[0], args[1]) + return nil }, } @@ -399,8 +409,9 @@ func yamlPathGet(root *yaml.Node, keys []string) (*yaml.Node, error) { node := root if node.Kind == yaml.DocumentNode { if len(node.Content) == 0 { - return nil, fmt.Errorf("empty config file") + return nil, errors.New("empty config file") } + node = node.Content[0] } @@ -408,13 +419,16 @@ func yamlPathGet(root *yaml.Node, keys []string) (*yaml.Node, error) { switch node.Kind { case yaml.MappingNode: found := false + for j := 0; j+1 < len(node.Content); j += 2 { if node.Content[j].Value == key { node = node.Content[j+1] found = true + break } } + if !found { return nil, fmt.Errorf("key not found: %s", strings.Join(keys[:i+1], ".")) } @@ -423,9 +437,11 @@ func yamlPathGet(root *yaml.Node, keys []string) (*yaml.Node, error) { if err != nil { return nil, fmt.Errorf("key %q is a list; use a numeric index", strings.Join(keys[:i], ".")) } + if idx < 0 || idx >= len(node.Content) { return nil, fmt.Errorf("index %d out of range for %s (len %d)", idx, strings.Join(keys[:i], "."), len(node.Content)) } + node = node.Content[idx] default: return nil, fmt.Errorf("key %q is not a map or list", strings.Join(keys[:i], ".")) @@ -445,6 +461,7 @@ func yamlPathSet(root *yaml.Node, keys []string, value string) error { if len(node.Content) == 0 { node.Content = []*yaml.Node{{Kind: yaml.MappingNode}} } + node = node.Content[0] } @@ -454,19 +471,23 @@ func yamlPathSet(root *yaml.Node, keys []string, value string) error { switch node.Kind { case yaml.MappingNode: var valueNode *yaml.Node + for j := 0; j+1 < len(node.Content); j += 2 { if node.Content[j].Value == key { valueNode = node.Content[j+1] + break } } if valueNode == nil { keyNode := &yaml.Node{Kind: yaml.ScalarNode, Value: key} + valueNode = &yaml.Node{Kind: yaml.MappingNode} if last { valueNode = &yaml.Node{Kind: yaml.ScalarNode, Value: value} } + node.Content = append(node.Content, keyNode, valueNode) } else if last { setScalar(valueNode, value) @@ -479,18 +500,22 @@ func yamlPathSet(root *yaml.Node, keys []string, value string) error { if err != nil { return fmt.Errorf("key %q is a list; use a numeric index", strings.Join(keys[:i], ".")) } + if idx < 0 || idx > len(node.Content) { return fmt.Errorf("index %d out of range for %s (len %d)", idx, strings.Join(keys[:i], "."), len(node.Content)) } + if idx == len(node.Content) { newNode := &yaml.Node{Kind: yaml.MappingNode} if last { newNode = &yaml.Node{Kind: yaml.ScalarNode, Value: value} } + node.Content = append(node.Content, newNode) } else if last { setScalar(node.Content[idx], value) } + node = node.Content[idx] default: @@ -516,8 +541,10 @@ func configPathForInit() string { if rootFlags.ConfigPath != "" { return rootFlags.ConfigPath } + if envPath := os.Getenv("VAULTIK_CONFIG"); envPath != "" { return envPath } + return DefaultConfigPath() } diff --git a/internal/cli/config_test.go b/internal/cli/config_test.go index cf45f04..24ad8ab 100644 --- a/internal/cli/config_test.go +++ b/internal/cli/config_test.go @@ -12,7 +12,9 @@ import ( // that unmarshals into the Config struct with the expected snapshots. func TestDefaultConfigTemplateParses(t *testing.T) { var cfg config.Config - if err := yaml.Unmarshal([]byte(defaultConfigTemplate), &cfg); err != nil { + + err := yaml.Unmarshal([]byte(defaultConfigTemplate), &cfg) + if err != nil { t.Fatalf("default config template is not valid YAML: %v", err) } @@ -24,9 +26,11 @@ func TestDefaultConfigTemplateParses(t *testing.T) { if !ok { t.Fatal("expected 'home' snapshot in default config") } + if len(home.Paths) == 0 { t.Error("home snapshot should have at least one path") } + if len(home.Exclude) == 0 { t.Error("home snapshot should have exclude patterns") } @@ -35,9 +39,11 @@ func TestDefaultConfigTemplateParses(t *testing.T) { if !ok { t.Fatal("expected 'apps' snapshot in default config") } + if len(apps.Paths) != 1 || apps.Paths[0] != "/Applications" { t.Errorf("apps snapshot should back up /Applications, got %v", apps.Paths) } + if len(apps.Exclude) == 0 { t.Error("apps snapshot should have exclude patterns") } @@ -58,10 +64,14 @@ snapshots: func parseTestYAML(t *testing.T) *yaml.Node { t.Helper() + var root yaml.Node - if err := yaml.Unmarshal([]byte(testYAML), &root); err != nil { + + err := yaml.Unmarshal([]byte(testYAML), &root) + if err != nil { t.Fatalf("parsing test yaml: %v", err) } + return &root } @@ -91,11 +101,14 @@ func TestYAMLPathGet(t *testing.T) { if err == nil { t.Fatalf("expected error for %q", tt.path) } + return } + if err != nil { t.Fatalf("unexpected error: %v", err) } + if node.Value != tt.want { t.Errorf("get %q = %q, want %q", tt.path, node.Value, tt.want) } @@ -115,6 +128,7 @@ func TestYAMLPathSet(t *testing.T) { if err := yamlPathSet(root, splitPath("s3.endpoint"), "s3.example.com"); err != nil { t.Fatalf("set s3.endpoint: %v", err) } + if err := yamlPathSet(root, splitPath("newmap.newkey"), "val"); err != nil { t.Fatalf("set newmap.newkey: %v", err) } @@ -123,9 +137,11 @@ func TestYAMLPathSet(t *testing.T) { if err := yamlPathSet(root, splitPath("age_recipients.0"), "age1bbb"); err != nil { t.Fatalf("set age_recipients.0: %v", err) } + if err := yamlPathSet(root, splitPath("age_recipients.1"), "age1ccc"); err != nil { t.Fatalf("append age_recipients.1: %v", err) } + if err := yamlPathSet(root, splitPath("age_recipients.5"), "age1ddd"); err == nil { t.Error("expected out-of-range append to fail") } @@ -135,6 +151,7 @@ func TestYAMLPathSet(t *testing.T) { if err != nil { t.Fatalf("marshal: %v", err) } + text := string(out) for _, want := range []string{"newbucket", "s3.example.com", "newkey: val", "# top comment", "# inline comment", "age1bbb", "age1ccc"} { @@ -147,6 +164,7 @@ func TestYAMLPathSet(t *testing.T) { if err != nil { t.Fatalf("get after set: %v", err) } + if got.Value != "newbucket" { t.Errorf("s3.bucket = %q after set, want newbucket", got.Value) } diff --git a/internal/cli/database.go b/internal/cli/database.go index cf83da0..a6532fb 100644 --- a/internal/cli/database.go +++ b/internal/cli/database.go @@ -66,6 +66,7 @@ Use --force to skip the confirmation prompt.`, // Check if database exists if _, err := os.Stat(dbPath); os.IsNotExist(err) { fmt.Printf("Database does not exist: %s\n", dbPath) + return nil } @@ -73,9 +74,11 @@ Use --force to skip the confirmation prompt.`, if !force { fmt.Printf("This will delete the local state database at:\n %s\n\n", dbPath) fmt.Print("Are you sure? Type 'yes' to confirm: ") + var confirm string if _, err := fmt.Scanln(&confirm); err != nil || confirm != "yes" { fmt.Println("Aborted.") + return nil } } @@ -97,6 +100,7 @@ Use --force to skip the confirmation prompt.`, } log.Info("Local state database deleted", "path", dbPath) + return nil }, } diff --git a/internal/cli/duration.go b/internal/cli/duration.go index d9a92ab..8fd2350 100644 --- a/internal/cli/duration.go +++ b/internal/cli/duration.go @@ -1,6 +1,7 @@ package cli import ( + "errors" "fmt" "regexp" "strconv" @@ -25,7 +26,7 @@ func parseDuration(s string) (time.Duration, error) { // Extended duration parsing // Check for negative values if strings.HasPrefix(strings.TrimSpace(s), "-") { - return 0, fmt.Errorf("negative durations are not supported") + return 0, errors.New("negative durations are not supported") } // Pattern matches: number + unit, repeated @@ -48,6 +49,7 @@ func parseDuration(s string) (time.Duration, error) { } var d time.Duration + switch unit { // Standard time units case "ns", "nanosecond", "nanoseconds": @@ -75,7 +77,7 @@ func parseDuration(s string) (time.Duration, error) { d = time.Duration(value * float64(365*24*time.Hour)) default: // Try parsing as standard Go duration unit - testStr := fmt.Sprintf("1%s", unit) + testStr := "1" + unit if _, err := time.ParseDuration(testStr); err == nil { // It's a valid Go duration unit, parse the full value fullStr := fmt.Sprintf("%g%s", value, unit) diff --git a/internal/cli/duration_test.go b/internal/cli/duration_test.go index 88cd67e..3edb414 100644 --- a/internal/cli/duration_test.go +++ b/internal/cli/duration_test.go @@ -185,6 +185,7 @@ func TestParseDuration(t *testing.T) { if tt.wantErr { assert.Error(t, err, "expected error for input %q", tt.input) + return } diff --git a/internal/cli/entry.go b/internal/cli/entry.go index 4b9fed8..9709f08 100644 --- a/internal/cli/entry.go +++ b/internal/cli/entry.go @@ -19,24 +19,26 @@ func CLIEntry() { if len(short) > 12 { short = short[:12] } + writeStartupBanner(ui.New(os.Stdout), time.Now().UTC(), short) } rootCmd := NewRootCommand() rootCmd.SilenceErrors = true - if err := rootCmd.Execute(); err != nil { - ReportError("%s", err.Error()) + err := rootCmd.Execute() + if err != nil { + ReportErrorf("%s", err.Error()) os.Exit(1) } } -// ReportError emits a user-facing error to stderr in the standard +// ReportErrorf emits a user-facing error to stderr in the standard // 🛑 ERROR: format. Use it from goroutine error paths (where returning // an error to cobra isn't an option) and anywhere else a CLI command // must surface a failure outside the normal RunE return path. -func ReportError(format string, args ...any) { - ui.New(os.Stderr).Error(format, args...) +func ReportErrorf(format string, args ...any) { + ui.New(os.Stderr).Errorf(format, args...) } // bannerSuppressedInArgs reports whether any of args is a flag that @@ -48,10 +50,12 @@ func bannerSuppressedInArgs(args []string) bool { if a == "--" { return false } + switch a { case "--quiet", "-q", "--cron": return true } + if strings.HasPrefix(a, "--quiet=") || strings.HasPrefix(a, "--cron=") { return true } @@ -64,5 +68,6 @@ func bannerSuppressedInArgs(args []string) bool { } } } + return false } diff --git a/internal/cli/entry_test.go b/internal/cli/entry_test.go index cc7e623..f652399 100644 --- a/internal/cli/entry_test.go +++ b/internal/cli/entry_test.go @@ -21,12 +21,15 @@ func TestCLIEntry(t *testing.T) { expectedCommands := []string{"config", "snapshot", "prune", "info", "version", "remote", "database"} for _, expected := range expectedCommands { found := false + for _, cmd := range cmd.Commands() { if cmd.Use == expected || cmd.Name() == expected { found = true + break } } + if !found { t.Errorf("Expected command '%s' not found", expected) } @@ -41,12 +44,15 @@ func TestCLIEntry(t *testing.T) { expectedSubCommands := []string{"create", "list", "purge", "verify", "remove", "restore"} for _, expected := range expectedSubCommands { found := false + for _, subcmd := range snapshotCmd.Commands() { if subcmd.Use == expected || subcmd.Name() == expected { found = true + break } } + if !found { t.Errorf("Expected snapshot subcommand '%s' not found", expected) } diff --git a/internal/cli/info.go b/internal/cli/info.go index e4aca7e..8cc715d 100644 --- a/internal/cli/info.go +++ b/internal/cli/info.go @@ -2,6 +2,7 @@ package cli import ( "context" + "errors" "os" "github.com/spf13/cobra" @@ -31,6 +32,7 @@ func NewInfoCommand() *cobra.Command { // Use the app framework rootFlags := GetRootFlags() + return RunWithApp(cmd.Context(), AppOptions{ ConfigPath: configPath, LogOptions: log.LogOptions{ @@ -44,21 +46,26 @@ func NewInfoCommand() *cobra.Command { lc.Append(fx.Hook{ OnStart: func(ctx context.Context) error { go func() { - if err := v.ShowInfo(); err != nil { - if err != context.Canceled { + err := v.ShowInfo() + if err != nil { + if !errors.Is(err, context.Canceled) { log.Error("Failed to show info", "error", err) - ReportError("Failed to show info: %v", err) + ReportErrorf("Failed to show info: %v", err) os.Exit(1) } } - if err := v.Shutdowner.Shutdown(); err != nil { + + err = v.Shutdowner.Shutdown() + if err != nil { log.Error("Failed to shutdown", "error", err) } }() + return nil }, OnStop: func(ctx context.Context) error { v.Cancel() + return nil }, }) diff --git a/internal/cli/prune.go b/internal/cli/prune.go index b367e9c..b5cc67e 100644 --- a/internal/cli/prune.go +++ b/internal/cli/prune.go @@ -2,6 +2,7 @@ package cli import ( "context" + "errors" "os" "github.com/spf13/cobra" @@ -39,6 +40,7 @@ work (e.g. after a crashed backup or to reclaim storage).`, // Use the app framework like other commands rootFlags := GetRootFlags() + return RunWithApp(cmd.Context(), AppOptions{ ConfigPath: configPath, LogOptions: log.LogOptions{ @@ -54,26 +56,31 @@ work (e.g. after a crashed backup or to reclaim storage).`, // Start the prune operation in a goroutine go func() { // Run the prune operation - if err := v.Prune(opts); err != nil { - if err != context.Canceled { + err := v.Prune(opts) + if err != nil { + if !errors.Is(err, context.Canceled) { if !opts.JSON { log.Error("Prune operation failed", "error", err) - ReportError("Prune failed: %v", err) + ReportErrorf("Prune failed: %v", err) } + os.Exit(1) } } // Shutdown the app when prune completes - if err := v.Shutdowner.Shutdown(); err != nil { + err = v.Shutdowner.Shutdown() + if err != nil { log.Error("Failed to shutdown", "error", err) } }() + return nil }, OnStop: func(ctx context.Context) error { log.Debug("Stopping prune operation") v.Cancel() + return nil }, }) diff --git a/internal/cli/remote.go b/internal/cli/remote.go index cde873e..7acb001 100644 --- a/internal/cli/remote.go +++ b/internal/cli/remote.go @@ -2,7 +2,7 @@ package cli import ( "context" - "fmt" + "errors" "os" "github.com/spf13/cobra" @@ -41,7 +41,7 @@ This is destructive and irreversible. Requires --force.`, Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { if !force { - return fmt.Errorf("remote nuke requires --force (this deletes ALL remote snapshots and blobs)") + return errors.New("remote nuke requires --force (this deletes ALL remote snapshots and blobs)") } configPath, err := ResolveConfigPath() @@ -50,6 +50,7 @@ This is destructive and irreversible. Requires --force.`, } rootFlags := GetRootFlags() + return RunWithApp(cmd.Context(), AppOptions{ ConfigPath: configPath, LogOptions: log.LogOptions{ @@ -63,21 +64,26 @@ This is destructive and irreversible. Requires --force.`, lc.Append(fx.Hook{ OnStart: func(ctx context.Context) error { go func() { - if err := v.NukeRemote(true); err != nil { - if err != context.Canceled { + err := v.NukeRemote(true) + if err != nil { + if !errors.Is(err, context.Canceled) { log.Error("Remote nuke failed", "error", err) - ReportError("Remote nuke failed: %v", err) + ReportErrorf("Remote nuke failed: %v", err) os.Exit(1) } } - if err := v.Shutdowner.Shutdown(); err != nil { + + err = v.Shutdowner.Shutdown() + if err != nil { log.Error("Failed to shutdown", "error", err) } }() + return nil }, OnStop: func(ctx context.Context) error { v.Cancel() + return nil }, }) @@ -113,6 +119,7 @@ func newRemoteInfoCommand() *cobra.Command { } rootFlags := GetRootFlags() + return RunWithApp(cmd.Context(), AppOptions{ ConfigPath: configPath, LogOptions: log.LogOptions{ @@ -126,23 +133,29 @@ func newRemoteInfoCommand() *cobra.Command { lc.Append(fx.Hook{ OnStart: func(ctx context.Context) error { go func() { - if err := v.RemoteInfo(jsonOutput); err != nil { - if err != context.Canceled { + err := v.RemoteInfo(jsonOutput) + if err != nil { + if !errors.Is(err, context.Canceled) { if !jsonOutput { log.Error("Failed to get remote info", "error", err) - ReportError("Failed to get remote info: %v", err) + ReportErrorf("Failed to get remote info: %v", err) } + os.Exit(1) } } - if err := v.Shutdowner.Shutdown(); err != nil { + + err = v.Shutdowner.Shutdown() + if err != nil { log.Error("Failed to shutdown", "error", err) } }() + return nil }, OnStop: func(ctx context.Context) error { v.Cancel() + return nil }, }) diff --git a/internal/cli/root.go b/internal/cli/root.go index 5c29b14..2e53dad 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -78,6 +78,7 @@ func ResolveConfigPath() (string, error) { if _, err := os.Stat(path); err != nil { return "", fmt.Errorf("config file from --config not found: %s (run 'vaultik config init --config %s' to create it)", path, path) } + return path, nil } @@ -85,6 +86,7 @@ func ResolveConfigPath() (string, error) { if _, err := os.Stat(path); err != nil { return "", fmt.Errorf("config file from $VAULTIK_CONFIG not found: %s (unset VAULTIK_CONFIG, point it at an existing file, or run 'vaultik config init')", path) } + return path, nil } @@ -114,5 +116,6 @@ func DefaultConfigPath() string { if os.Getuid() == 0 { return "/etc/vaultik/config.yml" } + return filepath.Join(xdg.ConfigHome, "vaultik", "config.yml") } diff --git a/internal/cli/snapshot.go b/internal/cli/snapshot.go index ef01eaa..5f5e088 100644 --- a/internal/cli/snapshot.go +++ b/internal/cli/snapshot.go @@ -2,6 +2,7 @@ package cli import ( "context" + "errors" "fmt" "os" @@ -58,6 +59,7 @@ specifying a path using --config or by setting VAULTIK_CONFIG to a path.`, // Use the backup functionality from cli package rootFlags := GetRootFlags() + return RunWithApp(cmd.Context(), AppOptions{ ConfigPath: configPath, LogOptions: log.LogOptions{ @@ -74,25 +76,29 @@ specifying a path using --config or by setting VAULTIK_CONFIG to a path.`, // Start the snapshot creation in a goroutine go func() { // --cron suppression is wired through v.UI by setupGlobals. - if err := v.CreateSnapshot(opts); err != nil { - if err != context.Canceled { + err := v.CreateSnapshot(opts) + if err != nil { + if !errors.Is(err, context.Canceled) { log.Error("Snapshot creation failed", "error", err) - ReportError("Snapshot creation failed: %v", err) + ReportErrorf("Snapshot creation failed: %v", err) os.Exit(1) } } // Shutdown the app when snapshot completes - if err := v.Shutdowner.Shutdown(); err != nil { + err = v.Shutdowner.Shutdown() + if err != nil { log.Error("Failed to shutdown", "error", err) } }() + return nil }, OnStop: func(ctx context.Context) error { log.Debug("Stopping snapshot creation") // Cancel the Vaultik context v.Cancel() + return nil }, }) @@ -127,6 +133,7 @@ func newSnapshotListCommand() *cobra.Command { } rootFlags := GetRootFlags() + return RunWithApp(cmd.Context(), AppOptions{ ConfigPath: configPath, LogOptions: log.LogOptions{ @@ -140,21 +147,26 @@ func newSnapshotListCommand() *cobra.Command { lc.Append(fx.Hook{ OnStart: func(ctx context.Context) error { go func() { - if err := v.ListSnapshots(jsonOutput); err != nil { - if err != context.Canceled { + err := v.ListSnapshots(jsonOutput) + if err != nil { + if !errors.Is(err, context.Canceled) { log.Error("Failed to list snapshots", "error", err) - ReportError("Failed to list snapshots: %v", err) + ReportErrorf("Failed to list snapshots: %v", err) os.Exit(1) } } - if err := v.Shutdowner.Shutdown(); err != nil { + + err = v.Shutdowner.Shutdown() + if err != nil { log.Error("Failed to shutdown", "error", err) } }() + return nil }, OnStop: func(ctx context.Context) error { v.Cancel() + return nil }, }) @@ -185,10 +197,11 @@ restrict the operation to specific snapshot names.`, RunE: func(cmd *cobra.Command, args []string) error { // Validate flags if !opts.KeepLatest && opts.OlderThan == "" { - return fmt.Errorf("must specify either --keep-latest or --older-than") + return errors.New("must specify either --keep-latest or --older-than") } + if opts.KeepLatest && opts.OlderThan != "" { - return fmt.Errorf("cannot specify both --keep-latest and --older-than") + return errors.New("cannot specify both --keep-latest and --older-than") } // Use unified config resolution @@ -198,6 +211,7 @@ restrict the operation to specific snapshot names.`, } rootFlags := GetRootFlags() + return RunWithApp(cmd.Context(), AppOptions{ ConfigPath: configPath, LogOptions: log.LogOptions{ @@ -211,21 +225,26 @@ restrict the operation to specific snapshot names.`, lc.Append(fx.Hook{ OnStart: func(ctx context.Context) error { go func() { - if err := v.PurgeSnapshotsWithOptions(opts); err != nil { - if err != context.Canceled { + err := v.PurgeSnapshotsWithOptions(opts) + if err != nil { + if !errors.Is(err, context.Canceled) { log.Error("Failed to purge snapshots", "error", err) - ReportError("Failed to purge snapshots: %v", err) + ReportErrorf("Failed to purge snapshots: %v", err) os.Exit(1) } } - if err := v.Shutdowner.Shutdown(); err != nil { + + err = v.Shutdowner.Shutdown() + if err != nil { log.Error("Failed to shutdown", "error", err) } }() + return nil }, OnStop: func(ctx context.Context) error { v.Cancel() + return nil }, }) @@ -254,11 +273,14 @@ func newSnapshotVerifyCommand() *cobra.Command { Args: func(cmd *cobra.Command, args []string) error { if len(args) != 1 { _ = cmd.Help() + if len(args) == 0 { - return fmt.Errorf("snapshot ID required") + return errors.New("snapshot ID required") } + return fmt.Errorf("expected 1 argument, got %d", len(args)) } + return nil }, RunE: func(cmd *cobra.Command, args []string) error { @@ -271,6 +293,7 @@ func newSnapshotVerifyCommand() *cobra.Command { } rootFlags := GetRootFlags() + return RunWithApp(cmd.Context(), AppOptions{ ConfigPath: configPath, LogOptions: log.LogOptions{ @@ -284,23 +307,29 @@ func newSnapshotVerifyCommand() *cobra.Command { lc.Append(fx.Hook{ OnStart: func(ctx context.Context) error { go func() { - if err := v.VerifySnapshotWithOptions(snapshotID, opts); err != nil { - if err != context.Canceled { + err := v.VerifySnapshotWithOptions(snapshotID, opts) + if err != nil { + if !errors.Is(err, context.Canceled) { if !opts.JSON { log.Error("Verification failed", "error", err) - ReportError("Verification failed: %v", err) + ReportErrorf("Verification failed: %v", err) } + os.Exit(1) } } - if err := v.Shutdowner.Shutdown(); err != nil { + + err = v.Shutdowner.Shutdown() + if err != nil { log.Error("Failed to shutdown", "error", err) } }() + return nil }, OnStop: func(ctx context.Context) error { v.Cancel() + return nil }, }) @@ -345,11 +374,14 @@ nuke --force' — it is the single supported entry point for that.`, Args: func(cmd *cobra.Command, args []string) error { if len(args) != 1 { _ = cmd.Help() + if len(args) == 0 { - return fmt.Errorf("snapshot ID required") + return errors.New("snapshot ID required") } + return fmt.Errorf("expected 1 argument, got %d", len(args)) } + return nil }, RunE: func(cmd *cobra.Command, args []string) error { @@ -360,6 +392,7 @@ nuke --force' — it is the single supported entry point for that.`, } rootFlags := GetRootFlags() + return RunWithApp(cmd.Context(), AppOptions{ ConfigPath: configPath, LogOptions: log.LogOptions{ @@ -375,22 +408,26 @@ nuke --force' — it is the single supported entry point for that.`, go func() { _, err := v.RemoveSnapshot(args[0], opts) if err != nil { - if err != context.Canceled { + if !errors.Is(err, context.Canceled) { if !opts.JSON { log.Error("Failed to remove snapshot", "error", err) - ReportError("Failed to remove snapshot: %v", err) + ReportErrorf("Failed to remove snapshot: %v", err) } + os.Exit(1) } } + if err := v.Shutdowner.Shutdown(); err != nil { log.Error("Failed to shutdown", "error", err) } }() + return nil }, OnStop: func(ctx context.Context) error { v.Cancel() + return nil }, }) diff --git a/internal/cli/snapshot_restore.go b/internal/cli/snapshot_restore.go index 9f60b4a..4f5d878 100644 --- a/internal/cli/snapshot_restore.go +++ b/internal/cli/snapshot_restore.go @@ -2,6 +2,7 @@ package cli import ( "context" + "errors" "os" "github.com/spf13/cobra" @@ -70,6 +71,7 @@ Examples: // runRestore parses arguments and runs the restore operation through the app framework func runRestore(cmd *cobra.Command, args []string, opts *RestoreOptions) error { snapshotID := args[0] + opts.TargetDir = args[1] if len(args) > 2 { opts.Paths = args[2:] @@ -83,6 +85,7 @@ func runRestore(cmd *cobra.Command, args []string, opts *RestoreOptions) error { // Use the app framework like other commands rootFlags := GetRootFlags() + return RunWithApp(cmd.Context(), AppOptions{ ConfigPath: configPath, LogOptions: log.LogOptions{ @@ -129,24 +132,29 @@ func buildRestoreInvokes(snapshotID string, opts *RestoreOptions) []fx.Option { Verify: opts.Verify, SkipErrors: GetRootFlags().SkipErrors, } - if err := app.Vaultik.Restore(restoreOpts); err != nil { - if err != context.Canceled { + + err := app.Vaultik.Restore(restoreOpts) + if err != nil { + if !errors.Is(err, context.Canceled) { log.Error("Restore operation failed", "error", err) - ReportError("Restore failed: %v", err) + ReportErrorf("Restore failed: %v", err) os.Exit(1) } } // Shutdown the app when restore completes - if err := app.Shutdowner.Shutdown(); err != nil { + err = app.Shutdowner.Shutdown() + if err != nil { log.Error("Failed to shutdown", "error", err) } }() + return nil }, OnStop: func(ctx context.Context) error { log.Debug("Stopping restore operation") app.Vaultik.Cancel() + return nil }, }) diff --git a/internal/cli/version.go b/internal/cli/version.go index cbe3344..7c6d14c 100644 --- a/internal/cli/version.go +++ b/internal/cli/version.go @@ -24,6 +24,7 @@ func NewVersionCommand() *cobra.Command { fmt.Printf(" author: %s\n", globals.Author) fmt.Printf(" homepage: %s\n", globals.Homepage) fmt.Printf(" license: %s\n", globals.License) + if globals.Version == "dev" { fmt.Println() fmt.Println("This is a development build (no version information embedded).") diff --git a/internal/config/config.go b/internal/config/config.go index a47f347..cabf735 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1,6 +1,7 @@ package config import ( + "errors" "fmt" "os" "path/filepath" @@ -21,12 +22,16 @@ const appName = "vaultik" func expandTilde(path string) string { if path == "~" { home, _ := os.UserHomeDir() + return home } + if strings.HasPrefix(path, "~/") { home, _ := os.UserHomeDir() + return filepath.Join(home, path[2:]) } + return path } @@ -34,8 +39,10 @@ func expandTilde(path string) string { func expandTildeInURL(url string) string { if strings.HasPrefix(url, "file://~/") { home, _ := os.UserHomeDir() + return "file://" + filepath.Join(home, url[9:]) } + return url } @@ -63,6 +70,7 @@ func (c *Config) GetExcludes(snapshotName string) []string { combined := make([]string, 0, len(c.Exclude)+len(snap.Exclude)) combined = append(combined, c.Exclude...) combined = append(combined, snap.Exclude...) + return combined } @@ -74,6 +82,7 @@ func (c *Config) SnapshotNames() []string { } // Sort for deterministic order sort.Strings(names) + return names } @@ -126,7 +135,7 @@ type ConfigPath string // Returns an error if the path is empty or if loading fails. func New(path ConfigPath) (*Config, error) { if path == "" { - return nil, fmt.Errorf("config path not provided") + return nil, errors.New("config path not provided") } cfg, err := Load(string(path)) @@ -159,6 +168,7 @@ func Load(path string) (*Config, error) { // Convert smartconfig data to YAML then unmarshal configData := sc.Data() + yamlBytes, err := yaml.Marshal(configData) if err != nil { return nil, fmt.Errorf("failed to marshal config data: %w", err) @@ -177,6 +187,7 @@ func Load(path string) (*Config, error) { for i, path := range snap.Paths { snap.Paths[i] = expandTilde(path) } + cfg.Snapshots[name] = snap } @@ -196,6 +207,7 @@ func Load(path string) (*Config, error) { if err != nil { return nil, fmt.Errorf("failed to get hostname: %w", err) } + cfg.Hostname = hostname } @@ -203,6 +215,7 @@ func Load(path string) (*Config, error) { if cfg.S3.Region == "" { cfg.S3.Region = "us-east-1" } + if cfg.S3.PartSize == 0 { cfg.S3.PartSize = Size(5 * 1024 * 1024) // 5MB } @@ -236,11 +249,11 @@ func Load(path string) (*Config, error) { // Returns an error describing the first validation failure encountered. func (c *Config) Validate() error { if len(c.AgeRecipients) == 0 { - return fmt.Errorf("at least one age_recipient is required (generate with: age-keygen)") + return errors.New("at least one age_recipient is required (generate with: age-keygen)") } if len(c.Snapshots) == 0 { - return fmt.Errorf("at least one snapshot must be configured (see config.example.yml)") + return errors.New("at least one snapshot must be configured (see config.example.yml)") } for name, snap := range c.Snapshots { @@ -250,20 +263,21 @@ func (c *Config) Validate() error { } // Validate storage configuration - if err := c.validateStorage(); err != nil { + err := c.validateStorage() + if err != nil { return err } if c.ChunkSize.Int64() < 1024*1024 { // 1MB minimum - return fmt.Errorf("chunk_size must be at least 1MB") + return errors.New("chunk_size must be at least 1MB") } if c.BlobSizeLimit.Int64() < c.ChunkSize.Int64() { - return fmt.Errorf("blob_size_limit must be at least chunk_size") + return errors.New("blob_size_limit must be at least chunk_size") } if c.CompressionLevel < 1 || c.CompressionLevel > 19 { - return fmt.Errorf("compression_level must be between 1 and 19") + return errors.New("compression_level must be between 1 and 19") } return nil @@ -280,38 +294,43 @@ func (c *Config) validateStorage() error { // File storage doesn't need S3 credentials return nil } + if strings.HasPrefix(c.StorageURL, "s3://") { // S3 storage needs credentials if c.S3.AccessKeyID == "" { - return fmt.Errorf("s3.access_key_id is required for s3:// URLs") + return errors.New("s3.access_key_id is required for s3:// URLs") } + if c.S3.SecretAccessKey == "" { - return fmt.Errorf("s3.secret_access_key is required for s3:// URLs") + return errors.New("s3.secret_access_key is required for s3:// URLs") } + return nil } + if strings.HasPrefix(c.StorageURL, "rclone://") { // Rclone storage uses rclone's own config return nil } - return fmt.Errorf("storage_url must start with s3://, file://, or rclone://") + + return errors.New("storage_url must start with s3://, file://, or rclone://") } // Legacy S3 configuration if c.S3.Endpoint == "" { - return fmt.Errorf("storage not configured; set storage_url or provide s3.endpoint + s3.bucket + credentials") + return errors.New("storage not configured; set storage_url or provide s3.endpoint + s3.bucket + credentials") } if c.S3.Bucket == "" { - return fmt.Errorf("s3.bucket is required (or set storage_url)") + return errors.New("s3.bucket is required (or set storage_url)") } if c.S3.AccessKeyID == "" { - return fmt.Errorf("s3.access_key_id is required") + return errors.New("s3.access_key_id is required") } if c.S3.SecretAccessKey == "" { - return fmt.Errorf("s3.secret_access_key is required") + return errors.New("s3.secret_access_key is required") } return nil @@ -329,6 +348,7 @@ func extractAgeSecretKey(input string) string { if id, ok := identities[0].(*age.X25519Identity); ok { return id.String() } + return strings.TrimSpace(input) } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 4304af6..08d3bce 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -41,6 +41,7 @@ func TestConfigLoad(t *testing.T) { if len(cfg.AgeRecipients) != 2 { t.Errorf("Expected 2 age recipients, got %d", len(cfg.AgeRecipients)) } + if cfg.AgeRecipients[0] != TEST_SNEAK_AGE_PUBLIC_KEY { t.Errorf("Expected first age recipient to be %s, got '%s'", TEST_SNEAK_AGE_PUBLIC_KEY, cfg.AgeRecipients[0]) } diff --git a/internal/config/size.go b/internal/config/size.go index d66049d..58dbf73 100644 --- a/internal/config/size.go +++ b/internal/config/size.go @@ -1,6 +1,7 @@ package config import ( + "errors" "fmt" "github.com/dustin/go-humanize" @@ -14,18 +15,19 @@ type Size int64 // UnmarshalYAML implements yaml.Unmarshaler for Size, allowing it to be // parsed from YAML configuration files. It accepts both numeric values // (interpreted as bytes) and string values with units (e.g., "10MB"). -func (s *Size) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (s *Size) UnmarshalYAML(unmarshal func(any) error) error { // Try to unmarshal as int64 first var intVal int64 if err := unmarshal(&intVal); err == nil { *s = Size(intVal) + return nil } // Try to unmarshal as string var strVal string if err := unmarshal(&strVal); err != nil { - return fmt.Errorf("size must be a number or string") + return errors.New("size must be a number or string") } // Parse the string using go-humanize @@ -35,6 +37,7 @@ func (s *Size) UnmarshalYAML(unmarshal func(interface{}) error) error { } *s = Size(bytes) + return nil } @@ -58,5 +61,6 @@ func ParseSize(s string) (Size, error) { if err != nil { return 0, fmt.Errorf("invalid size format: %w", err) } + return Size(bytes), nil } diff --git a/internal/crypto/encryption.go b/internal/crypto/encryption.go index 1d7b3cb..d3380a7 100644 --- a/internal/crypto/encryption.go +++ b/internal/crypto/encryption.go @@ -2,6 +2,7 @@ package crypto import ( "bytes" + "errors" "fmt" "io" "sync" @@ -25,7 +26,7 @@ type Encryptor struct { // public keys are invalid or if no recipients are specified. func NewEncryptor(publicKeys []string) (*Encryptor, error) { if len(publicKeys) == 0 { - return nil, fmt.Errorf("at least one recipient is required") + return nil, errors.New("at least one recipient is required") } recipients := make([]age.Recipient, 0, len(publicKeys)) @@ -34,6 +35,7 @@ func NewEncryptor(publicKeys []string) (*Encryptor, error) { if err != nil { return nil, fmt.Errorf("parsing age recipient %s: %w", key, err) } + recipients = append(recipients, recipient) } @@ -126,7 +128,7 @@ func (e *Encryptor) EncryptWriter(dst io.Writer) (io.WriteCloser, error) { // of the public keys are invalid or if no recipients are specified. func (e *Encryptor) UpdateRecipients(publicKeys []string) error { if len(publicKeys) == 0 { - return fmt.Errorf("at least one recipient is required") + return errors.New("at least one recipient is required") } recipients := make([]age.Recipient, 0, len(publicKeys)) @@ -135,6 +137,7 @@ func (e *Encryptor) UpdateRecipients(publicKeys []string) error { if err != nil { return fmt.Errorf("parsing age recipient %s: %w", key, err) } + recipients = append(recipients, recipient) } diff --git a/internal/crypto/encryption_test.go b/internal/crypto/encryption_test.go index ddd92ca..584b216 100644 --- a/internal/crypto/encryption_test.go +++ b/internal/crypto/encryption_test.go @@ -58,10 +58,12 @@ func TestEncryptorMultipleRecipients(t *testing.T) { if err != nil { t.Fatalf("failed to generate identity1: %v", err) } + identity2, err := age.GenerateX25519Identity() if err != nil { t.Fatalf("failed to generate identity2: %v", err) } + identity3, err := age.GenerateX25519Identity() if err != nil { t.Fatalf("failed to generate identity3: %v", err) @@ -123,6 +125,7 @@ func TestEncryptorUpdateRecipients(t *testing.T) { // Encrypt with first key plaintext := []byte("test data") + ciphertext1, err := enc.Encrypt(plaintext) if err != nil { t.Fatalf("failed to encrypt: %v", err) @@ -143,6 +146,7 @@ func TestEncryptorUpdateRecipients(t *testing.T) { if _, err := age.Decrypt(bytes.NewReader(ciphertext1), identity1); err != nil { t.Error("failed to decrypt with identity1") } + if _, err := age.Decrypt(bytes.NewReader(ciphertext1), identity2); err == nil { t.Error("should not decrypt with identity2") } @@ -151,6 +155,7 @@ func TestEncryptorUpdateRecipients(t *testing.T) { if _, err := age.Decrypt(bytes.NewReader(ciphertext2), identity2); err != nil { t.Error("failed to decrypt with identity2") } + if _, err := age.Decrypt(bytes.NewReader(ciphertext2), identity1); err == nil { t.Error("should not decrypt with identity1") } diff --git a/internal/database/blob_chunks.go b/internal/database/blob_chunks.go index 13e6b27..6ff9522 100644 --- a/internal/database/blob_chunks.go +++ b/internal/database/blob_chunks.go @@ -49,12 +49,15 @@ func (r *BlobChunkRepository) GetByBlobID(ctx context.Context, blobID string) ([ defer CloseRows(rows) var blobChunks []*BlobChunk + for rows.Next() { var bc BlobChunk + err := rows.Scan(&bc.BlobID, &bc.ChunkHash, &bc.Offset, &bc.Length) if err != nil { return nil, fmt.Errorf("scanning blob chunk: %w", err) } + blobChunks = append(blobChunks, &bc) } @@ -70,7 +73,9 @@ func (r *BlobChunkRepository) GetByChunkHash(ctx context.Context, chunkHash stri ` LogSQL("GetByChunkHash", query, chunkHash) + var bc BlobChunk + err := r.db.conn.QueryRowContext(ctx, query, chunkHash).Scan( &bc.BlobID, &bc.ChunkHash, @@ -80,14 +85,18 @@ func (r *BlobChunkRepository) GetByChunkHash(ctx context.Context, chunkHash stri if err == sql.ErrNoRows { LogSQL("GetByChunkHash", "No rows found", chunkHash) + return nil, nil } + if err != nil { LogSQL("GetByChunkHash", "Error", chunkHash, err) + return nil, fmt.Errorf("querying blob chunk: %w", err) } LogSQL("GetByChunkHash", "Found blob", chunkHash, "blob", bc.BlobID) + return &bc, nil } @@ -101,7 +110,9 @@ func (r *BlobChunkRepository) GetByChunkHashTx(ctx context.Context, tx *sql.Tx, ` LogSQL("GetByChunkHashTx", query, chunkHash) + var bc BlobChunk + err := tx.QueryRowContext(ctx, query, chunkHash).Scan( &bc.BlobID, &bc.ChunkHash, @@ -111,14 +122,18 @@ func (r *BlobChunkRepository) GetByChunkHashTx(ctx context.Context, tx *sql.Tx, if err == sql.ErrNoRows { LogSQL("GetByChunkHashTx", "No rows found", chunkHash) + return nil, nil } + if err != nil { LogSQL("GetByChunkHashTx", "Error", chunkHash, err) + return nil, fmt.Errorf("querying blob chunk: %w", err) } LogSQL("GetByChunkHashTx", "Found blob", chunkHash, "blob", bc.BlobID) + return &bc, nil } diff --git a/internal/database/blob_chunks_test.go b/internal/database/blob_chunks_test.go index 9dae601..97d3e68 100644 --- a/internal/database/blob_chunks_test.go +++ b/internal/database/blob_chunks_test.go @@ -22,6 +22,7 @@ func TestBlobChunkRepository(t *testing.T) { Hash: types.BlobHash("blob1-hash"), CreatedTS: time.Now(), } + err := repos.Blobs.Create(ctx, nil, blob) if err != nil { t.Fatalf("failed to create blob: %v", err) @@ -34,6 +35,7 @@ func TestBlobChunkRepository(t *testing.T) { ChunkHash: chunkHash, Size: 1024, } + err = repos.Chunks.Create(ctx, nil, chunk) if err != nil { t.Fatalf("failed to create chunk %s: %v", chunkHash, err) @@ -60,6 +62,7 @@ func TestBlobChunkRepository(t *testing.T) { Offset: 1024, Length: 2048, } + err = repos.BlobChunks.Create(ctx, nil, bc2) if err != nil { t.Fatalf("failed to create second blob chunk: %v", err) @@ -71,6 +74,7 @@ func TestBlobChunkRepository(t *testing.T) { Offset: 3072, Length: 512, } + err = repos.BlobChunks.Create(ctx, nil, bc3) if err != nil { t.Fatalf("failed to create third blob chunk: %v", err) @@ -81,6 +85,7 @@ func TestBlobChunkRepository(t *testing.T) { if err != nil { t.Fatalf("failed to get blob chunks: %v", err) } + if len(blobChunks) != 3 { t.Errorf("expected 3 chunks, got %d", len(blobChunks)) } @@ -98,12 +103,15 @@ func TestBlobChunkRepository(t *testing.T) { if err != nil { t.Fatalf("failed to get blob chunk by chunk hash: %v", err) } + if bc == nil { t.Fatal("expected blob chunk, got nil") } + if bc.BlobID != blob.ID { t.Errorf("wrong blob ID: expected %s, got %s", blob.ID, bc.BlobID) } + if bc.Offset != 1024 { t.Errorf("wrong offset: expected 1024, got %d", bc.Offset) } @@ -113,6 +121,7 @@ func TestBlobChunkRepository(t *testing.T) { if err == nil { t.Fatal("duplicate blob_chunk insert should fail due to primary key constraint") } + if !strings.Contains(err.Error(), "UNIQUE") && !strings.Contains(err.Error(), "constraint") { t.Fatalf("expected constraint error, got: %v", err) } @@ -122,6 +131,7 @@ func TestBlobChunkRepository(t *testing.T) { if err != nil { t.Fatalf("unexpected error: %v", err) } + if bc != nil { t.Error("expected nil for non-existent chunk") } @@ -150,6 +160,7 @@ func TestBlobChunkRepositoryMultipleBlobs(t *testing.T) { if err != nil { t.Fatalf("failed to create blob1: %v", err) } + err = repos.Blobs.Create(ctx, nil, blob2) if err != nil { t.Fatalf("failed to create blob2: %v", err) @@ -162,6 +173,7 @@ func TestBlobChunkRepositoryMultipleBlobs(t *testing.T) { ChunkHash: chunkHash, Size: 1024, } + err = repos.Chunks.Create(ctx, nil, chunk) if err != nil { t.Fatalf("failed to create chunk %s: %v", chunkHash, err) @@ -189,6 +201,7 @@ func TestBlobChunkRepositoryMultipleBlobs(t *testing.T) { if err != nil { t.Fatalf("failed to get blob1 chunks: %v", err) } + if len(chunks) != 2 { t.Errorf("expected 2 chunks for blob1, got %d", len(chunks)) } @@ -198,6 +211,7 @@ func TestBlobChunkRepositoryMultipleBlobs(t *testing.T) { if err != nil { t.Fatalf("failed to get blob2 chunks: %v", err) } + if len(chunks) != 2 { t.Errorf("expected 2 chunks for blob2, got %d", len(chunks)) } @@ -207,6 +221,7 @@ func TestBlobChunkRepositoryMultipleBlobs(t *testing.T) { if err != nil { t.Fatalf("failed to get shared chunk: %v", err) } + if bc == nil { t.Fatal("expected shared chunk, got nil") } diff --git a/internal/database/blobs.go b/internal/database/blobs.go index 58aed0f..43a48e7 100644 --- a/internal/database/blobs.go +++ b/internal/database/blobs.go @@ -24,10 +24,12 @@ func (r *BlobRepository) Create(ctx context.Context, tx *sql.Tx, blob *Blob) err ` var finishedTS, uploadedTS *int64 + if blob.FinishedTS != nil { ts := blob.FinishedTS.Unix() finishedTS = &ts } + if blob.UploadedTS != nil { ts := blob.UploadedTS.Unix() uploadedTS = &ts @@ -56,9 +58,11 @@ func (r *BlobRepository) GetByHash(ctx context.Context, hash string) (*Blob, err WHERE blob_hash = ? ` - var blob Blob - var createdTSUnix int64 - var finishedTSUnix, uploadedTSUnix sql.NullInt64 + var ( + blob Blob + createdTSUnix int64 + finishedTSUnix, uploadedTSUnix sql.NullInt64 + ) err := r.db.conn.QueryRowContext(ctx, query, hash).Scan( &blob.ID, @@ -73,6 +77,7 @@ func (r *BlobRepository) GetByHash(ctx context.Context, hash string) (*Blob, err if err == sql.ErrNoRows { return nil, nil } + if err != nil { return nil, fmt.Errorf("querying blob: %w", err) } @@ -82,10 +87,12 @@ func (r *BlobRepository) GetByHash(ctx context.Context, hash string) (*Blob, err ts := time.Unix(finishedTSUnix.Int64, 0).UTC() blob.FinishedTS = &ts } + if uploadedTSUnix.Valid { ts := time.Unix(uploadedTSUnix.Int64, 0).UTC() blob.UploadedTS = &ts } + return &blob, nil } @@ -97,9 +104,11 @@ func (r *BlobRepository) GetByID(ctx context.Context, id string) (*Blob, error) WHERE id = ? ` - var blob Blob - var createdTSUnix int64 - var finishedTSUnix, uploadedTSUnix sql.NullInt64 + var ( + blob Blob + createdTSUnix int64 + finishedTSUnix, uploadedTSUnix sql.NullInt64 + ) err := r.db.conn.QueryRowContext(ctx, query, id).Scan( &blob.ID, @@ -114,6 +123,7 @@ func (r *BlobRepository) GetByID(ctx context.Context, id string) (*Blob, error) if err == sql.ErrNoRows { return nil, nil } + if err != nil { return nil, fmt.Errorf("querying blob: %w", err) } @@ -123,10 +133,12 @@ func (r *BlobRepository) GetByID(ctx context.Context, id string) (*Blob, error) ts := time.Unix(finishedTSUnix.Int64, 0).UTC() blob.FinishedTS = &ts } + if uploadedTSUnix.Valid { ts := time.Unix(uploadedTSUnix.Int64, 0).UTC() blob.UploadedTS = &ts } + return &blob, nil } @@ -146,11 +158,15 @@ func (r *BlobRepository) GetAll(ctx context.Context) (map[string]*Blob, error) { defer CloseRows(rows) out := make(map[string]*Blob) + for rows.Next() { - var blob Blob - var createdTSUnix int64 - var finishedTSUnix, uploadedTSUnix sql.NullInt64 - if err := rows.Scan( + var ( + blob Blob + createdTSUnix int64 + finishedTSUnix, uploadedTSUnix sql.NullInt64 + ) + + err := rows.Scan( &blob.ID, &blob.Hash, &createdTSUnix, @@ -158,20 +174,25 @@ func (r *BlobRepository) GetAll(ctx context.Context) (map[string]*Blob, error) { &blob.UncompressedSize, &blob.CompressedSize, &uploadedTSUnix, - ); err != nil { + ) + if err != nil { return nil, fmt.Errorf("scanning blob: %w", err) } + blob.CreatedTS = time.Unix(createdTSUnix, 0).UTC() if finishedTSUnix.Valid { ts := time.Unix(finishedTSUnix.Int64, 0).UTC() blob.FinishedTS = &ts } + if uploadedTSUnix.Valid { ts := time.Unix(uploadedTSUnix.Int64, 0).UTC() blob.UploadedTS = &ts } + out[blob.ID.String()] = &blob } + return out, rows.Err() } @@ -184,6 +205,7 @@ func (r *BlobRepository) UpdateFinished(ctx context.Context, tx *sql.Tx, id stri ` now := time.Now().UTC().Unix() + var err error if tx != nil { _, err = tx.ExecContext(ctx, query, hash, now, uncompressedSize, compressedSize, id) @@ -207,6 +229,7 @@ func (r *BlobRepository) UpdateUploaded(ctx context.Context, tx *sql.Tx, id stri ` now := time.Now().UTC().Unix() + var err error if tx != nil { _, err = tx.ExecContext(ctx, query, now, id) diff --git a/internal/database/blobs_test.go b/internal/database/blobs_test.go index 3d06c5d..2b8e88b 100644 --- a/internal/database/blobs_test.go +++ b/internal/database/blobs_test.go @@ -32,12 +32,15 @@ func TestBlobRepository(t *testing.T) { if err != nil { t.Fatalf("failed to get blob: %v", err) } + if retrieved == nil { t.Fatal("expected blob, got nil") } + if retrieved.Hash != blob.Hash { t.Errorf("blob hash mismatch: got %s, want %s", retrieved.Hash, blob.Hash) } + if !retrieved.CreatedTS.Equal(blob.CreatedTS) { t.Errorf("created timestamp mismatch: got %v, want %v", retrieved.CreatedTS, blob.CreatedTS) } @@ -47,9 +50,11 @@ func TestBlobRepository(t *testing.T) { if err != nil { t.Fatalf("failed to get blob by ID: %v", err) } + if retrievedByID == nil { t.Fatal("expected blob, got nil") } + if retrievedByID.ID != blob.ID { t.Errorf("blob ID mismatch: got %s, want %s", retrievedByID.ID, blob.ID) } @@ -60,6 +65,7 @@ func TestBlobRepository(t *testing.T) { Hash: types.BlobHash("blobhash456"), CreatedTS: time.Now().Truncate(time.Second), } + err = repo.Create(ctx, nil, blob2) if err != nil { t.Fatalf("failed to create second blob: %v", err) @@ -67,6 +73,7 @@ func TestBlobRepository(t *testing.T) { // Test UpdateFinished now := time.Now() + err = repo.UpdateFinished(ctx, nil, blob.ID.String(), blob.Hash.String(), 1000, 500) if err != nil { t.Fatalf("failed to update blob as finished: %v", err) @@ -77,12 +84,15 @@ func TestBlobRepository(t *testing.T) { if err != nil { t.Fatalf("failed to get updated blob: %v", err) } + if updated.FinishedTS == nil { t.Fatal("expected finished timestamp to be set") } + if updated.UncompressedSize != 1000 { t.Errorf("expected uncompressed size 1000, got %d", updated.UncompressedSize) } + if updated.CompressedSize != 500 { t.Errorf("expected compressed size 500, got %d", updated.CompressedSize) } @@ -98,6 +108,7 @@ func TestBlobRepository(t *testing.T) { if err != nil { t.Fatalf("failed to get uploaded blob: %v", err) } + if uploaded.UploadedTS == nil { t.Fatal("expected uploaded timestamp to be set") } diff --git a/internal/database/cascade_debug_test.go b/internal/database/cascade_debug_test.go index 6ef933f..fc71f9f 100644 --- a/internal/database/cascade_debug_test.go +++ b/internal/database/cascade_debug_test.go @@ -19,10 +19,12 @@ func TestCascadeDeleteDebug(t *testing.T) { // Check if foreign keys are enabled var fkEnabled int + err := db.conn.QueryRow("PRAGMA foreign_keys").Scan(&fkEnabled) if err != nil { t.Fatal(err) } + t.Logf("Foreign keys enabled: %d", fkEnabled) // Create a file @@ -34,18 +36,21 @@ func TestCascadeDeleteDebug(t *testing.T) { UID: 1000, GID: 1000, } + err = repos.Files.Create(ctx, nil, file) if err != nil { t.Fatalf("failed to create file: %v", err) } + t.Logf("Created file with ID: %s", file.ID) // Create chunks and file-chunk mappings - for i := 0; i < 3; i++ { + for i := range 3 { chunk := &Chunk{ ChunkHash: types.ChunkHash(fmt.Sprintf("cascade-chunk-%d", i)), Size: 1024, } + err = repos.Chunks.Create(ctx, nil, chunk) if err != nil { t.Fatalf("failed to create chunk: %v", err) @@ -56,10 +61,12 @@ func TestCascadeDeleteDebug(t *testing.T) { Idx: i, ChunkHash: chunk.ChunkHash, } + err = repos.FileChunks.Create(ctx, nil, fc) if err != nil { t.Fatalf("failed to create file chunk: %v", err) } + t.Logf("Created file chunk mapping: file_id=%s, idx=%d, chunk=%s", fc.FileID, fc.Idx, fc.ChunkHash) } @@ -68,10 +75,12 @@ func TestCascadeDeleteDebug(t *testing.T) { if err != nil { t.Fatal(err) } + t.Logf("File chunks before delete: %d", len(fileChunks)) // Check the foreign key constraint var fkInfo string + err = db.conn.QueryRow(` SELECT sql FROM sqlite_master WHERE type='table' AND name='file_chunks' @@ -79,10 +88,12 @@ func TestCascadeDeleteDebug(t *testing.T) { if err != nil { t.Fatal(err) } + t.Logf("file_chunks table definition:\n%s", fkInfo) // Delete the file t.Log("Deleting file...") + err = repos.Files.DeleteByID(ctx, nil, file.ID) if err != nil { t.Fatalf("failed to delete file: %v", err) @@ -93,6 +104,7 @@ func TestCascadeDeleteDebug(t *testing.T) { if err != nil { t.Fatal(err) } + if deletedFile != nil { t.Error("file should have been deleted") } else { @@ -104,14 +116,17 @@ func TestCascadeDeleteDebug(t *testing.T) { if err != nil { t.Fatal(err) } + t.Logf("File chunks after delete: %d", len(fileChunks)) // Manually check the database var count int + err = db.conn.QueryRow("SELECT COUNT(*) FROM file_chunks WHERE file_id = ?", file.ID).Scan(&count) if err != nil { t.Fatal(err) } + t.Logf("Manual count of file_chunks for deleted file: %d", count) if len(fileChunks) != 0 { diff --git a/internal/database/chunk_files.go b/internal/database/chunk_files.go index 0e5f0df..01e9175 100644 --- a/internal/database/chunk_files.go +++ b/internal/database/chunk_files.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "fmt" + "strings" "sneak.berlin/go/vaultik/internal/types" ) @@ -90,18 +91,25 @@ func (r *ChunkFileRepository) GetByFileID(ctx context.Context, fileID types.File // scanChunkFiles is a helper that scans chunk file rows func (r *ChunkFileRepository) scanChunkFiles(rows *sql.Rows) ([]*ChunkFile, error) { var chunkFiles []*ChunkFile + for rows.Next() { - var cf ChunkFile - var chunkHashStr, fileIDStr string + var ( + cf ChunkFile + chunkHashStr, fileIDStr string + ) + err := rows.Scan(&chunkHashStr, &fileIDStr, &cf.FileOffset, &cf.Length) if err != nil { return nil, fmt.Errorf("scanning chunk file: %w", err) } + cf.ChunkHash = types.ChunkHash(chunkHashStr) + cf.FileID, err = types.ParseFileID(fileIDStr) if err != nil { return nil, fmt.Errorf("parsing file ID: %w", err) } + chunkFiles = append(chunkFiles, &cf) } @@ -136,14 +144,13 @@ func (r *ChunkFileRepository) DeleteByFileIDs(ctx context.Context, tx *sql.Tx, f const batchSize = 500 for i := 0; i < len(fileIDs); i += batchSize { - end := i + batchSize - if end > len(fileIDs) { - end = len(fileIDs) - } + end := min(i+batchSize, len(fileIDs)) + batch := fileIDs[i:end] query := "DELETE FROM chunk_files WHERE file_id IN (?" + repeatPlaceholder(len(batch)-1) + ")" - args := make([]interface{}, len(batch)) + + args := make([]any, len(batch)) for j, id := range batch { args[j] = id.String() } @@ -154,6 +161,7 @@ func (r *ChunkFileRepository) DeleteByFileIDs(ctx context.Context, tx *sql.Tx, f } else { _, err = r.db.ExecWithLog(ctx, query, args...) } + if err != nil { return fmt.Errorf("batch deleting chunk_files: %w", err) } @@ -172,21 +180,28 @@ func (r *ChunkFileRepository) CreateBatch(ctx context.Context, tx *sql.Tx, cfs [ const batchSize = 200 for i := 0; i < len(cfs); i += batchSize { - end := i + batchSize - if end > len(cfs) { - end = len(cfs) - } + end := min(i+batchSize, len(cfs)) + batch := cfs[i:end] query := "INSERT INTO chunk_files (chunk_hash, file_id, file_offset, length) VALUES " - args := make([]interface{}, 0, len(batch)*4) + + args := make([]any, 0, len(batch)*4) + + var querySb183 strings.Builder + for j, cf := range batch { if j > 0 { - query += ", " + querySb183.WriteString(", ") } - query += "(?, ?, ?, ?)" + + querySb183.WriteString("(?, ?, ?, ?)") + args = append(args, cf.ChunkHash.String(), cf.FileID.String(), cf.FileOffset, cf.Length) } + + query += querySb183.String() + query += " ON CONFLICT(chunk_hash, file_id) DO NOTHING" var err error @@ -195,6 +210,7 @@ func (r *ChunkFileRepository) CreateBatch(ctx context.Context, tx *sql.Tx, cfs [ } else { _, err = r.db.ExecWithLog(ctx, query, args...) } + if err != nil { return fmt.Errorf("batch inserting chunk_files: %w", err) } diff --git a/internal/database/chunk_files_test.go b/internal/database/chunk_files_test.go index a4e8bbc..ca3fd98 100644 --- a/internal/database/chunk_files_test.go +++ b/internal/database/chunk_files_test.go @@ -28,6 +28,7 @@ func TestChunkFileRepository(t *testing.T) { GID: 1000, LinkTarget: "", } + err := fileRepo.Create(ctx, nil, file1) if err != nil { t.Fatalf("failed to create file1: %v", err) @@ -42,6 +43,7 @@ func TestChunkFileRepository(t *testing.T) { GID: 1000, LinkTarget: "", } + err = fileRepo.Create(ctx, nil, file2) if err != nil { t.Fatalf("failed to create file2: %v", err) @@ -52,6 +54,7 @@ func TestChunkFileRepository(t *testing.T) { ChunkHash: types.ChunkHash("chunk1"), Size: 1024, } + err = chunksRepo.Create(ctx, nil, chunk) if err != nil { t.Fatalf("failed to create chunk: %v", err) @@ -77,6 +80,7 @@ func TestChunkFileRepository(t *testing.T) { FileOffset: 2048, Length: 1024, } + err = repo.Create(ctx, nil, cf2) if err != nil { t.Fatalf("failed to create second chunk file: %v", err) @@ -87,6 +91,7 @@ func TestChunkFileRepository(t *testing.T) { if err != nil { t.Fatalf("failed to get chunk files: %v", err) } + if len(chunkFiles) != 2 { t.Errorf("expected 2 files for chunk, got %d", len(chunkFiles)) } @@ -94,14 +99,17 @@ func TestChunkFileRepository(t *testing.T) { // Verify both files are returned foundFile1 := false foundFile2 := false + for _, cf := range chunkFiles { if cf.FileID == file1.ID && cf.FileOffset == 0 { foundFile1 = true } + if cf.FileID == file2.ID && cf.FileOffset == 2048 { foundFile2 = true } } + if !foundFile1 || !foundFile2 { t.Error("not all expected files found") } @@ -111,9 +119,11 @@ func TestChunkFileRepository(t *testing.T) { if err != nil { t.Fatalf("failed to get chunks by file ID: %v", err) } + if len(chunkFiles) != 1 { t.Errorf("expected 1 chunk for file, got %d", len(chunkFiles)) } + if chunkFiles[0].ChunkHash != types.ChunkHash("chunk1") { t.Errorf("wrong chunk hash: expected chunk1, got %s", chunkFiles[0].ChunkHash) } @@ -143,9 +153,11 @@ func TestChunkFileRepositoryComplexDeduplication(t *testing.T) { if err := fileRepo.Create(ctx, nil, file1); err != nil { t.Fatalf("failed to create file1: %v", err) } + if err := fileRepo.Create(ctx, nil, file2); err != nil { t.Fatalf("failed to create file2: %v", err) } + if err := fileRepo.Create(ctx, nil, file3); err != nil { t.Fatalf("failed to create file3: %v", err) } @@ -157,6 +169,7 @@ func TestChunkFileRepositoryComplexDeduplication(t *testing.T) { ChunkHash: chunkHash, Size: 1024, } + err := chunksRepo.Create(ctx, nil, chunk) if err != nil { t.Fatalf("failed to create chunk %s: %v", chunkHash, err) @@ -194,6 +207,7 @@ func TestChunkFileRepositoryComplexDeduplication(t *testing.T) { if err != nil { t.Fatalf("failed to get files for chunk1: %v", err) } + if len(files) != 2 { t.Errorf("expected 2 files for chunk1, got %d", len(files)) } @@ -203,6 +217,7 @@ func TestChunkFileRepositoryComplexDeduplication(t *testing.T) { if err != nil { t.Fatalf("failed to get files for chunk2: %v", err) } + if len(files) != 2 { t.Errorf("expected 2 files for chunk2, got %d", len(files)) } @@ -212,6 +227,7 @@ func TestChunkFileRepositoryComplexDeduplication(t *testing.T) { if err != nil { t.Fatalf("failed to get chunks for file2: %v", err) } + if len(file2Chunks) != 3 { t.Errorf("expected 3 chunks for file2, got %d", len(file2Chunks)) } diff --git a/internal/database/chunks.go b/internal/database/chunks.go index 04fc4ef..5c54a14 100644 --- a/internal/database/chunks.go +++ b/internal/database/chunks.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "fmt" + "strings" "sneak.berlin/go/vaultik/internal/log" ) @@ -54,6 +55,7 @@ func (r *ChunkRepository) GetByHash(ctx context.Context, hash string) (*Chunk, e if err == sql.ErrNoRows { return nil, nil } + if err != nil { return nil, fmt.Errorf("querying chunk: %w", err) } @@ -71,14 +73,22 @@ func (r *ChunkRepository) GetByHashes(ctx context.Context, hashes []string) ([]* FROM chunks WHERE chunk_hash IN (` - args := make([]interface{}, len(hashes)) + args := make([]any, len(hashes)) + + var querySb75 strings.Builder + for i, hash := range hashes { if i > 0 { - query += ", " + querySb75.WriteString(", ") } - query += "?" + + querySb75.WriteString("?") + args[i] = hash } + + query += querySb75.String() + query += ") ORDER BY chunk_hash" rows, err := r.db.conn.QueryContext(ctx, query, args...) @@ -88,6 +98,7 @@ func (r *ChunkRepository) GetByHashes(ctx context.Context, hashes []string) ([]* defer CloseRows(rows) var chunks []*Chunk + for rows.Next() { var chunk Chunk @@ -122,6 +133,7 @@ func (r *ChunkRepository) ListUnpacked(ctx context.Context, limit int) ([]*Chunk defer CloseRows(rows) var chunks []*Chunk + for rows.Next() { var chunk Chunk diff --git a/internal/database/chunks_ext.go b/internal/database/chunks_ext.go index b38c170..a322a9a 100644 --- a/internal/database/chunks_ext.go +++ b/internal/database/chunks_ext.go @@ -19,6 +19,7 @@ func (r *ChunkRepository) List(ctx context.Context) ([]*Chunk, error) { defer CloseRows(rows) var chunks []*Chunk + for rows.Next() { var chunk Chunk diff --git a/internal/database/chunks_test.go b/internal/database/chunks_test.go index 5122bb1..8f96bd1 100644 --- a/internal/database/chunks_test.go +++ b/internal/database/chunks_test.go @@ -30,12 +30,15 @@ func TestChunkRepository(t *testing.T) { if err != nil { t.Fatalf("failed to get chunk: %v", err) } + if retrieved == nil { t.Fatal("expected chunk, got nil") } + if retrieved.ChunkHash != chunk.ChunkHash { t.Errorf("chunk hash mismatch: got %s, want %s", retrieved.ChunkHash, chunk.ChunkHash) } + if retrieved.Size != chunk.Size { t.Errorf("size mismatch: got %d, want %d", retrieved.Size, chunk.Size) } @@ -51,6 +54,7 @@ func TestChunkRepository(t *testing.T) { ChunkHash: types.ChunkHash("chunkhash456"), Size: 8192, } + err = repo.Create(ctx, nil, chunk2) if err != nil { t.Fatalf("failed to create second chunk: %v", err) @@ -60,6 +64,7 @@ func TestChunkRepository(t *testing.T) { if err != nil { t.Fatalf("failed to get chunks by hashes: %v", err) } + if len(chunks) != 2 { t.Errorf("expected 2 chunks, got %d", len(chunks)) } @@ -69,6 +74,7 @@ func TestChunkRepository(t *testing.T) { if err != nil { t.Fatalf("failed to list unpacked chunks: %v", err) } + if len(unpacked) != 2 { t.Errorf("expected 2 unpacked chunks, got %d", len(unpacked)) } @@ -86,6 +92,7 @@ func TestChunkRepositoryNotFound(t *testing.T) { if err != nil { t.Fatalf("unexpected error: %v", err) } + if chunk != nil { t.Error("expected nil for non-existent chunk") } @@ -95,6 +102,7 @@ func TestChunkRepositoryNotFound(t *testing.T) { if err != nil { t.Fatalf("unexpected error: %v", err) } + if chunks != nil { t.Error("expected nil for empty hash list") } diff --git a/internal/database/database.go b/internal/database/database.go index 5f0f14c..7fc717f 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -57,8 +57,8 @@ func ParseMigrationVersion(filename string) (int, error) { // Split on underscore to separate version from description. // If there's no underscore, the entire stem is the version. versionStr := name - if idx := strings.IndexByte(name, '_'); idx >= 0 { - versionStr = name[:idx] + if before, _, ok := strings.Cut(name, "_"); ok { + versionStr = before } if versionStr == "" { @@ -98,6 +98,7 @@ func New(ctx context.Context, path string) (*DB, error) { // First attempt with standard WAL mode log.Debug("Attempting to open database with WAL mode", "path", path) + conn, err := sql.Open( "sqlite", path+"?_journal_mode=WAL&_synchronous=NORMAL&_busy_timeout=10000&_locking_mode=NORMAL&_foreign_keys=ON", @@ -110,7 +111,8 @@ func New(ctx context.Context, path string) (*DB, error) { conn.SetMaxOpenConns(1) conn.SetMaxIdleConns(1) - if err := conn.PingContext(ctx); err == nil { + err := conn.PingContext(ctx) + if err == nil { // Success on first try log.Debug("Database opened successfully with WAL mode", "path", path) @@ -120,13 +122,19 @@ func New(ctx context.Context, path string) (*DB, error) { } db := &DB{conn: conn, path: path} - if err := applyMigrations(ctx, conn); err != nil { + + err := applyMigrations(ctx, conn) + if err != nil { _ = conn.Close() + return nil, fmt.Errorf("applying migrations: %w", err) } + return db, nil } + log.Debug("Failed to ping database, closing connection", "path", path, "error", err) + _ = conn.Close() } @@ -135,6 +143,7 @@ func New(ctx context.Context, path string) (*DB, error) { "Database appears locked, attempting recovery with TRUNCATE mode", "path", path, ) + conn, err = sql.Open( "sqlite", path+"?_journal_mode=TRUNCATE&_synchronous=NORMAL&_busy_timeout=10000&_foreign_keys=ON", @@ -152,7 +161,9 @@ func New(ctx context.Context, path string) (*DB, error) { if err := conn.PingContext(ctx); err != nil { log.Debug("Failed to ping database in recovery mode, closing", "path", path, "error", err) + _ = conn.Close() + return nil, fmt.Errorf( "database still locked after recovery attempt: %w", err, @@ -163,6 +174,7 @@ func New(ctx context.Context, path string) (*DB, error) { // Switch back to WAL mode log.Debug("Switching database back to WAL mode", "path", path) + if _, err := conn.ExecContext(ctx, "PRAGMA journal_mode=WAL"); err != nil { log.Warn("Failed to switch back to WAL mode", "path", path, "error", err) } @@ -175,10 +187,12 @@ func New(ctx context.Context, path string) (*DB, error) { db := &DB{conn: conn, path: path} if err := applyMigrations(ctx, conn); err != nil { _ = conn.Close() + return nil, fmt.Errorf("applying migrations: %w", err) } log.Debug("Database connection established successfully", "path", path) + return db, nil } @@ -187,11 +201,16 @@ func New(ctx context.Context, path string) (*DB, error) { // Returns an error if the database connection cannot be closed properly. func (db *DB) Close() error { log.Debug("Closing database connection", "path", db.path) - if err := db.conn.Close(); err != nil { + + err := db.conn.Close() + if err != nil { log.Error("Failed to close database", "path", db.path, "error", err) + return fmt.Errorf("failed to close database: %w", err) } + log.Debug("Database connection closed successfully", "path", db.path) + return nil } @@ -227,9 +246,10 @@ func (db *DB) BeginTx( func (db *DB) ExecWithLog( ctx context.Context, query string, - args ...interface{}, + args ...any, ) (sql.Result, error) { LogSQL("Execute", query, args...) + return db.conn.ExecContext(ctx, query, args...) } @@ -240,9 +260,10 @@ func (db *DB) ExecWithLog( func (db *DB) QueryRowWithLog( ctx context.Context, query string, - args ...interface{}, + args ...any, ) *sql.Row { LogSQL("QueryRow", query, args...) + return db.conn.QueryRowContext(ctx, query, args...) } @@ -375,6 +396,7 @@ func repeatPlaceholder(n int) string { if n <= 0 { return "" } + return strings.Repeat(", ?", n) } @@ -385,7 +407,7 @@ func repeatPlaceholder(n int) string { // The operation parameter describes the type of SQL operation (e.g., "Execute", "Query"). // The query parameter is the SQL statement being executed. // The args parameter contains the query arguments that will be interpolated. -func LogSQL(operation, query string, args ...interface{}) { +func LogSQL(operation, query string, args ...any) { if strings.Contains(os.Getenv("GODEBUG"), "vaultik") { log.Debug( "SQL "+operation, diff --git a/internal/database/database_test.go b/internal/database/database_test.go index 6d763a3..7700ac1 100644 --- a/internal/database/database_test.go +++ b/internal/database/database_test.go @@ -17,7 +17,8 @@ func TestDatabase(t *testing.T) { t.Fatalf("failed to create database: %v", err) } defer func() { - if err := db.Close(); err != nil { + err := db.Close() + if err != nil { t.Errorf("failed to close database: %v", err) } }() @@ -37,6 +38,7 @@ func TestDatabase(t *testing.T) { for _, table := range tables { var name string + err := db.conn.QueryRow("SELECT name FROM sqlite_master WHERE type='table' AND name=?", table).Scan(&name) if err != nil { t.Errorf("table %s does not exist: %v", table, err) @@ -63,7 +65,8 @@ func TestDatabaseConcurrentAccess(t *testing.T) { t.Fatalf("failed to create database: %v", err) } defer func() { - if err := db.Close(); err != nil { + err := db.Close() + if err != nil { t.Errorf("failed to close database: %v", err) } }() @@ -73,9 +76,10 @@ func TestDatabaseConcurrentAccess(t *testing.T) { index int err error } + results := make(chan result, 10) - for i := 0; i < 10; i++ { + for i := range 10 { go func(i int) { _, err := db.ExecWithLog(ctx, "INSERT INTO chunks (chunk_hash, size) VALUES (?, ?)", fmt.Sprintf("hash%d", i), i*1024) @@ -84,7 +88,7 @@ func TestDatabaseConcurrentAccess(t *testing.T) { } // Wait for all goroutines and check results - for i := 0; i < 10; i++ { + for range 10 { r := <-results if r.err != nil { t.Fatalf("concurrent insert %d failed: %v", r.index, r.err) @@ -93,10 +97,12 @@ func TestDatabaseConcurrentAccess(t *testing.T) { // Verify all inserts succeeded var count int + err = db.conn.QueryRowContext(ctx, "SELECT COUNT(*) FROM chunks").Scan(&count) if err != nil { t.Fatalf("failed to count chunks: %v", err) } + if count != 10 { t.Errorf("expected 10 chunks, got %d", count) } @@ -127,12 +133,16 @@ func TestParseMigrationVersion(t *testing.T) { if err == nil { t.Errorf("ParseMigrationVersion(%q) = %d, nil; want error", tc.filename, got) } + return } + if err != nil { t.Errorf("ParseMigrationVersion(%q) unexpected error: %v", tc.filename, err) + return } + if got != tc.wantVer { t.Errorf("ParseMigrationVersion(%q) = %d; want %d", tc.filename, got, tc.wantVer) } @@ -148,7 +158,8 @@ func TestApplyMigrations_Idempotent(t *testing.T) { t.Fatalf("failed to open database: %v", err) } defer func() { - if err := conn.Close(); err != nil { + err := conn.Close() + if err != nil { t.Errorf("failed to close database: %v", err) } }() @@ -191,7 +202,8 @@ func TestBootstrapMigrationsTable_FreshDatabase(t *testing.T) { t.Fatalf("failed to open database: %v", err) } defer func() { - if err := conn.Close(); err != nil { + err := conn.Close() + if err != nil { t.Errorf("failed to close database: %v", err) } }() @@ -206,6 +218,7 @@ func TestBootstrapMigrationsTable_FreshDatabase(t *testing.T) { ).Scan(&tableBefore); err != nil { t.Fatalf("failed to check for table before bootstrap: %v", err) } + if tableBefore != 0 { t.Fatal("schema_migrations table should not exist before bootstrap") } @@ -222,6 +235,7 @@ func TestBootstrapMigrationsTable_FreshDatabase(t *testing.T) { ).Scan(&tableAfter); err != nil { t.Fatalf("failed to check for table after bootstrap: %v", err) } + if tableAfter != 1 { t.Fatalf("schema_migrations table should exist after bootstrap, got count=%d", tableAfter) } @@ -233,6 +247,7 @@ func TestBootstrapMigrationsTable_FreshDatabase(t *testing.T) { ).Scan(&version); err != nil { t.Fatalf("version 0 row not found in schema_migrations: %v", err) } + if version != 0 { t.Errorf("expected version 0, got %d", version) } diff --git a/internal/database/errors.go b/internal/database/errors.go index b2b4497..860c23f 100644 --- a/internal/database/errors.go +++ b/internal/database/errors.go @@ -6,15 +6,16 @@ import ( "os" ) -// Fatal prints an error message to stderr and exits with status 1 -func Fatal(format string, args ...interface{}) { +// Fatalf prints an error message to stderr and exits with status 1 +func Fatalf(format string, args ...any) { fmt.Fprintf(os.Stderr, "FATAL: "+format+"\n", args...) os.Exit(1) } // CloseRows closes rows and exits on error func CloseRows(rows *sql.Rows) { - if err := rows.Close(); err != nil { - Fatal("failed to close rows: %v", err) + err := rows.Close() + if err != nil { + Fatalf("failed to close rows: %v", err) } } diff --git a/internal/database/file_chunks.go b/internal/database/file_chunks.go index 31fa142..c6b32f4 100644 --- a/internal/database/file_chunks.go +++ b/internal/database/file_chunks.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "fmt" + "strings" "sneak.berlin/go/vaultik/internal/types" ) @@ -84,6 +85,7 @@ func (r *FileChunkRepository) GetByPathTx(ctx context.Context, tx *sql.Tx, path ` LogSQL("GetByPathTx", query, path) + rows, err := tx.QueryContext(ctx, query, path) if err != nil { return nil, fmt.Errorf("querying file chunks: %w", err) @@ -92,23 +94,30 @@ func (r *FileChunkRepository) GetByPathTx(ctx context.Context, tx *sql.Tx, path fileChunks, err := r.scanFileChunks(rows) LogSQL("GetByPathTx", "Complete", path, "count", len(fileChunks)) + return fileChunks, err } // scanFileChunks is a helper that scans file chunk rows func (r *FileChunkRepository) scanFileChunks(rows *sql.Rows) ([]*FileChunk, error) { var fileChunks []*FileChunk + for rows.Next() { - var fc FileChunk - var fileIDStr, chunkHashStr string + var ( + fc FileChunk + fileIDStr, chunkHashStr string + ) + err := rows.Scan(&fileIDStr, &fc.Idx, &chunkHashStr) if err != nil { return nil, fmt.Errorf("scanning file chunk: %w", err) } + fc.FileID, err = types.ParseFileID(fileIDStr) if err != nil { return nil, fmt.Errorf("parsing file ID: %w", err) } + fc.ChunkHash = types.ChunkHash(chunkHashStr) fileChunks = append(fileChunks, &fc) } @@ -161,14 +170,13 @@ func (r *FileChunkRepository) DeleteByFileIDs(ctx context.Context, tx *sql.Tx, f const batchSize = 500 for i := 0; i < len(fileIDs); i += batchSize { - end := i + batchSize - if end > len(fileIDs) { - end = len(fileIDs) - } + end := min(i+batchSize, len(fileIDs)) + batch := fileIDs[i:end] query := "DELETE FROM file_chunks WHERE file_id IN (?" + repeatPlaceholder(len(batch)-1) + ")" - args := make([]interface{}, len(batch)) + + args := make([]any, len(batch)) for j, id := range batch { args[j] = id.String() } @@ -179,6 +187,7 @@ func (r *FileChunkRepository) DeleteByFileIDs(ctx context.Context, tx *sql.Tx, f } else { _, err = r.db.ExecWithLog(ctx, query, args...) } + if err != nil { return fmt.Errorf("batch deleting file_chunks: %w", err) } @@ -199,22 +208,29 @@ func (r *FileChunkRepository) CreateBatch(ctx context.Context, tx *sql.Tx, fcs [ const batchSize = 300 for i := 0; i < len(fcs); i += batchSize { - end := i + batchSize - if end > len(fcs) { - end = len(fcs) - } + end := min(i+batchSize, len(fcs)) + batch := fcs[i:end] // Build the query with multiple value sets query := "INSERT INTO file_chunks (file_id, idx, chunk_hash) VALUES " - args := make([]interface{}, 0, len(batch)*3) + + args := make([]any, 0, len(batch)*3) + + var querySb211 strings.Builder + for j, fc := range batch { if j > 0 { - query += ", " + querySb211.WriteString(", ") } - query += "(?, ?, ?)" + + querySb211.WriteString("(?, ?, ?)") + args = append(args, fc.FileID.String(), fc.Idx, fc.ChunkHash.String()) } + + query += querySb211.String() + query += " ON CONFLICT(file_id, idx) DO NOTHING" var err error @@ -223,6 +239,7 @@ func (r *FileChunkRepository) CreateBatch(ctx context.Context, tx *sql.Tx, fcs [ } else { _, err = r.db.ExecWithLog(ctx, query, args...) } + if err != nil { return fmt.Errorf("batch inserting file_chunks: %w", err) } @@ -236,6 +253,7 @@ func (r *FileChunkRepository) GetByFile(ctx context.Context, path string) ([]*Fi LogSQL("GetByFile", "Starting", path) result, err := r.GetByPath(ctx, path) LogSQL("GetByFile", "Complete", path, "count", len(result)) + return result, err } @@ -244,5 +262,6 @@ func (r *FileChunkRepository) GetByFileTx(ctx context.Context, tx *sql.Tx, path LogSQL("GetByFileTx", "Starting", path) result, err := r.GetByPathTx(ctx, tx, path) LogSQL("GetByFileTx", "Complete", path, "count", len(result)) + return result, err } diff --git a/internal/database/file_chunks_test.go b/internal/database/file_chunks_test.go index ebd2e01..e1eb2b9 100644 --- a/internal/database/file_chunks_test.go +++ b/internal/database/file_chunks_test.go @@ -28,6 +28,7 @@ func TestFileChunkRepository(t *testing.T) { GID: 1000, LinkTarget: "", } + err := fileRepo.Create(ctx, nil, file) if err != nil { t.Fatalf("failed to create file: %v", err) @@ -36,11 +37,13 @@ func TestFileChunkRepository(t *testing.T) { // Create chunks first chunks := []types.ChunkHash{"chunk1", "chunk2", "chunk3"} chunkRepo := NewChunkRepository(db) + for _, chunkHash := range chunks { chunk := &Chunk{ ChunkHash: chunkHash, Size: 1024, } + err = chunkRepo.Create(ctx, nil, chunk) if err != nil { t.Fatalf("failed to create chunk %s: %v", chunkHash, err) @@ -65,6 +68,7 @@ func TestFileChunkRepository(t *testing.T) { Idx: 1, ChunkHash: types.ChunkHash("chunk2"), } + err = repo.Create(ctx, nil, fc2) if err != nil { t.Fatalf("failed to create second file chunk: %v", err) @@ -75,6 +79,7 @@ func TestFileChunkRepository(t *testing.T) { Idx: 2, ChunkHash: types.ChunkHash("chunk3"), } + err = repo.Create(ctx, nil, fc3) if err != nil { t.Fatalf("failed to create third file chunk: %v", err) @@ -85,6 +90,7 @@ func TestFileChunkRepository(t *testing.T) { if err != nil { t.Fatalf("failed to get file chunks: %v", err) } + if len(fileChunks) != 3 { t.Errorf("expected 3 chunks, got %d", len(fileChunks)) } @@ -112,6 +118,7 @@ func TestFileChunkRepository(t *testing.T) { if err != nil { t.Fatalf("failed to get deleted file chunks: %v", err) } + if len(fileChunks) != 0 { t.Errorf("expected 0 chunks after delete, got %d", len(fileChunks)) } @@ -140,22 +147,26 @@ func TestFileChunkRepositoryMultipleFiles(t *testing.T) { GID: 1000, LinkTarget: "", } + err := fileRepo.Create(ctx, nil, file) if err != nil { t.Fatalf("failed to create file %s: %v", path, err) } + files[i] = file } // Create all chunks first chunkRepo := NewChunkRepository(db) + for i := range files { - for j := 0; j < 2; j++ { + for j := range 2 { chunkHash := types.ChunkHash(fmt.Sprintf("file%d_chunk%d", i, j)) chunk := &Chunk{ ChunkHash: chunkHash, Size: 1024, } + err := chunkRepo.Create(ctx, nil, chunk) if err != nil { t.Fatalf("failed to create chunk %s: %v", chunkHash, err) @@ -165,12 +176,13 @@ func TestFileChunkRepositoryMultipleFiles(t *testing.T) { // Create chunks for multiple files for i, file := range files { - for j := 0; j < 2; j++ { + for j := range 2 { fc := &FileChunk{ FileID: file.ID, Idx: j, ChunkHash: types.ChunkHash(fmt.Sprintf("file%d_chunk%d", i, j)), } + err := repo.Create(ctx, nil, fc) if err != nil { t.Fatalf("failed to create file chunk: %v", err) @@ -184,6 +196,7 @@ func TestFileChunkRepositoryMultipleFiles(t *testing.T) { if err != nil { t.Fatalf("failed to get chunks for file %d: %v", i, err) } + if len(chunks) != 2 { t.Errorf("expected 2 chunks for file %d, got %d", i, len(chunks)) } diff --git a/internal/database/files.go b/internal/database/files.go index bf81e18..77ec1ec 100644 --- a/internal/database/files.go +++ b/internal/database/files.go @@ -3,7 +3,9 @@ package database import ( "context" "database/sql" + "errors" "fmt" + "strings" "time" "sneak.berlin/go/vaultik/internal/log" @@ -38,8 +40,11 @@ func (r *FileRepository) Create(ctx context.Context, tx *sql.Tx, file *File) err RETURNING id ` - var idStr string - var err error + var ( + idStr string + err error + ) + if tx != nil { LogSQL("Execute", query, file.ID.String(), file.Path.String(), file.SourcePath.String(), file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID, file.LinkTarget.String()) err = tx.QueryRowContext(ctx, query, file.ID.String(), file.Path.String(), file.SourcePath.String(), file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID, file.LinkTarget.String()).Scan(&idStr) @@ -68,9 +73,10 @@ func (r *FileRepository) GetByPath(ctx context.Context, path string) (*File, err ` file, err := r.scanFile(r.db.conn.QueryRowContext(ctx, query, path)) - if err == sql.ErrNoRows { + if errors.Is(err, sql.ErrNoRows) { return nil, nil } + if err != nil { return nil, fmt.Errorf("querying file: %w", err) } @@ -87,9 +93,10 @@ func (r *FileRepository) GetByID(ctx context.Context, id types.FileID) (*File, e ` file, err := r.scanFile(r.db.conn.QueryRowContext(ctx, query, id.String())) - if err == sql.ErrNoRows { + if errors.Is(err, sql.ErrNoRows) { return nil, nil } + if err != nil { return nil, fmt.Errorf("querying file: %w", err) } @@ -108,9 +115,10 @@ func (r *FileRepository) GetByPathTx(ctx context.Context, tx *sql.Tx, path strin file, err := r.scanFile(tx.QueryRowContext(ctx, query, path)) LogSQL("GetByPathTx Scan complete", query, path) - if err == sql.ErrNoRows { + if errors.Is(err, sql.ErrNoRows) { return nil, nil } + if err != nil { return nil, fmt.Errorf("querying file: %w", err) } @@ -120,10 +128,12 @@ func (r *FileRepository) GetByPathTx(ctx context.Context, tx *sql.Tx, path strin // scanFile is a helper that scans a single file row func (r *FileRepository) scanFile(row *sql.Row) (*File, error) { - var file File - var idStr, pathStr, sourcePathStr string - var mtimeUnix int64 - var linkTarget sql.NullString + var ( + file File + idStr, pathStr, sourcePathStr string + mtimeUnix int64 + linkTarget sql.NullString + ) err := row.Scan( &idStr, @@ -144,8 +154,10 @@ func (r *FileRepository) scanFile(row *sql.Row) (*File, error) { if err != nil { return nil, fmt.Errorf("parsing file ID: %w", err) } + file.Path = types.FilePath(pathStr) file.SourcePath = types.SourcePath(sourcePathStr) + file.MTime = time.Unix(mtimeUnix, 0).UTC() if linkTarget.Valid { file.LinkTarget = types.FilePath(linkTarget.String) @@ -156,10 +168,12 @@ func (r *FileRepository) scanFile(row *sql.Row) (*File, error) { // scanFileRows is a helper that scans a file row from rows iterator func (r *FileRepository) scanFileRows(rows *sql.Rows) (*File, error) { - var file File - var idStr, pathStr, sourcePathStr string - var mtimeUnix int64 - var linkTarget sql.NullString + var ( + file File + idStr, pathStr, sourcePathStr string + mtimeUnix int64 + linkTarget sql.NullString + ) err := rows.Scan( &idStr, @@ -180,8 +194,10 @@ func (r *FileRepository) scanFileRows(rows *sql.Rows) (*File, error) { if err != nil { return nil, fmt.Errorf("parsing file ID: %w", err) } + file.Path = types.FilePath(pathStr) file.SourcePath = types.SourcePath(sourcePathStr) + file.MTime = time.Unix(mtimeUnix, 0).UTC() if linkTarget.Valid { file.LinkTarget = types.FilePath(linkTarget.String) @@ -205,11 +221,13 @@ func (r *FileRepository) ListModifiedSince(ctx context.Context, since time.Time) defer CloseRows(rows) var files []*File + for rows.Next() { file, err := r.scanFileRows(rows) if err != nil { return nil, fmt.Errorf("scanning file: %w", err) } + files = append(files, file) } @@ -266,11 +284,13 @@ func (r *FileRepository) ListByPrefix(ctx context.Context, prefix string) ([]*Fi defer CloseRows(rows) var files []*File + for rows.Next() { file, err := r.scanFileRows(rows) if err != nil { return nil, fmt.Errorf("scanning file: %w", err) } + files = append(files, file) } @@ -292,11 +312,13 @@ func (r *FileRepository) ListAll(ctx context.Context) ([]*File, error) { defer CloseRows(rows) var files []*File + for rows.Next() { file, err := r.scanFileRows(rows) if err != nil { return nil, fmt.Errorf("scanning file: %w", err) } + files = append(files, file) } @@ -314,21 +336,28 @@ func (r *FileRepository) CreateBatch(ctx context.Context, tx *sql.Tx, files []*F const batchSize = 100 for i := 0; i < len(files); i += batchSize { - end := i + batchSize - if end > len(files) { - end = len(files) - } + end := min(i+batchSize, len(files)) + batch := files[i:end] query := `INSERT INTO files (id, path, source_path, mtime, size, mode, uid, gid, link_target) VALUES ` - args := make([]interface{}, 0, len(batch)*9) + + args := make([]any, 0, len(batch)*9) + + var querySb325 strings.Builder + for j, f := range batch { if j > 0 { - query += ", " + querySb325.WriteString(", ") } - query += "(?, ?, ?, ?, ?, ?, ?, ?, ?)" + + querySb325.WriteString("(?, ?, ?, ?, ?, ?, ?, ?, ?)") + args = append(args, f.ID.String(), f.Path.String(), f.SourcePath.String(), f.MTime.Unix(), f.Size, f.Mode, f.UID, f.GID, f.LinkTarget.String()) } + + query += querySb325.String() + query += ` ON CONFLICT(path) DO UPDATE SET source_path = excluded.source_path, mtime = excluded.mtime, @@ -344,6 +373,7 @@ func (r *FileRepository) CreateBatch(ctx context.Context, tx *sql.Tx, files []*F } else { _, err = r.db.ExecWithLog(ctx, query, args...) } + if err != nil { return fmt.Errorf("batch inserting files: %w", err) } diff --git a/internal/database/files_test.go b/internal/database/files_test.go index 8f16421..7f4b472 100644 --- a/internal/database/files_test.go +++ b/internal/database/files_test.go @@ -3,7 +3,7 @@ package database import ( "context" "database/sql" - "fmt" + "errors" "os" "path/filepath" "testing" @@ -20,7 +20,8 @@ func setupTestDB(t *testing.T) (*DB, func()) { } cleanup := func() { - if err := db.Close(); err != nil { + err := db.Close() + if err != nil { t.Errorf("failed to close database: %v", err) } } @@ -56,18 +57,23 @@ func TestFileRepository(t *testing.T) { if err != nil { t.Fatalf("failed to get file: %v", err) } + if retrieved == nil { t.Fatal("expected file, got nil") } + if retrieved.Path != file.Path { t.Errorf("path mismatch: got %s, want %s", retrieved.Path, file.Path) } + if !retrieved.MTime.Equal(file.MTime) { t.Errorf("mtime mismatch: got %v, want %v", retrieved.MTime, file.MTime) } + if retrieved.Size != file.Size { t.Errorf("size mismatch: got %d, want %d", retrieved.Size, file.Size) } + if retrieved.Mode != file.Mode { t.Errorf("mode mismatch: got %o, want %o", retrieved.Mode, file.Mode) } @@ -75,6 +81,7 @@ func TestFileRepository(t *testing.T) { // Test Update (upsert) file.Size = 2048 file.MTime = time.Now().Truncate(time.Second) + err = repo.Create(ctx, nil, file) if err != nil { t.Fatalf("failed to update file: %v", err) @@ -84,6 +91,7 @@ func TestFileRepository(t *testing.T) { if err != nil { t.Fatalf("failed to get updated file: %v", err) } + if retrieved.Size != 2048 { t.Errorf("size not updated: got %d, want %d", retrieved.Size, 2048) } @@ -93,6 +101,7 @@ func TestFileRepository(t *testing.T) { if err != nil { t.Fatalf("failed to list files: %v", err) } + if len(files) != 1 { t.Errorf("expected 1 file, got %d", len(files)) } @@ -107,6 +116,7 @@ func TestFileRepository(t *testing.T) { if err != nil { t.Fatalf("error getting deleted file: %v", err) } + if retrieved != nil { t.Error("expected nil for deleted file") } @@ -139,9 +149,11 @@ func TestFileRepositorySymlink(t *testing.T) { if err != nil { t.Fatalf("failed to get symlink: %v", err) } + if !retrieved.IsSymlink() { t.Error("expected IsSymlink() to be true") } + if retrieved.LinkTarget != symlink.LinkTarget { t.Errorf("link target mismatch: got %s, want %s", retrieved.LinkTarget, symlink.LinkTarget) } @@ -165,12 +177,13 @@ func TestFileRepositoryTransaction(t *testing.T) { GID: 1000, } - if err := repos.Files.Create(ctx, tx, file); err != nil { + err := repos.Files.Create(ctx, tx, file) + if err != nil { return err } // Return error to trigger rollback - return fmt.Errorf("test rollback") + return errors.New("test rollback") }) if err == nil || err.Error() != "test rollback" { @@ -182,6 +195,7 @@ func TestFileRepositoryTransaction(t *testing.T) { if err != nil { t.Fatalf("error checking for file: %v", err) } + if retrieved != nil { t.Error("file should not exist after rollback") } diff --git a/internal/database/local_meta.go b/internal/database/local_meta.go index cf92ff5..83c2048 100644 --- a/internal/database/local_meta.go +++ b/internal/database/local_meta.go @@ -27,15 +27,18 @@ func NewLocalMetaRepository(db *DB) *LocalMetaRepository { // "unset" (bind on first use) from "set to something" (compare). func (r *LocalMetaRepository) Get(ctx context.Context, key string) (string, error) { var value string + err := r.db.conn.QueryRowContext(ctx, "SELECT value FROM local_meta WHERE key = ?", key, ).Scan(&value) if errors.Is(err, sql.ErrNoRows) { return "", nil } + if err != nil { return "", fmt.Errorf("reading local_meta %q: %w", key, err) } + return value, nil } @@ -49,5 +52,6 @@ func (r *LocalMetaRepository) Set(ctx context.Context, key, value string) error if err != nil { return fmt.Errorf("writing local_meta %q: %w", key, err) } + return nil } diff --git a/internal/database/local_meta_test.go b/internal/database/local_meta_test.go index b2be08d..d9b1d64 100644 --- a/internal/database/local_meta_test.go +++ b/internal/database/local_meta_test.go @@ -11,18 +11,20 @@ import ( func TestLocalMetaEmptyOnFresh(t *testing.T) { db, err := database.NewTestDB() require.NoError(t, err) + defer func() { _ = db.Close() }() repos := database.NewRepositories(db) got, err := repos.LocalMeta.Get(context.Background(), database.LocalMetaKeyStorageURL) require.NoError(t, err) - require.Equal(t, "", got, "fresh DB must return empty for unset keys, not error") + require.Empty(t, got, "fresh DB must return empty for unset keys, not error") } func TestLocalMetaSetGetRoundTrip(t *testing.T) { db, err := database.NewTestDB() require.NoError(t, err) + defer func() { _ = db.Close() }() repos := database.NewRepositories(db) @@ -38,6 +40,7 @@ func TestLocalMetaSetGetRoundTrip(t *testing.T) { func TestLocalMetaSetOverwrites(t *testing.T) { db, err := database.NewTestDB() require.NoError(t, err) + defer func() { _ = db.Close() }() repos := database.NewRepositories(db) diff --git a/internal/database/module.go b/internal/database/module.go index 98a5b74..e4af208 100644 --- a/internal/database/module.go +++ b/internal/database/module.go @@ -34,11 +34,16 @@ func provideDatabase(lc fx.Lifecycle, cfg *config.Config) (*DB, error) { lc.Append(fx.Hook{ OnStop: func(ctx context.Context) error { log.Debug("Database module OnStop hook called") - if err := db.Close(); err != nil { + + err := db.Close() + if err != nil { log.Error("Failed to close database in OnStop hook", "error", err) + return err } + log.Debug("Database closed successfully in OnStop hook") + return nil }, }) diff --git a/internal/database/repositories.go b/internal/database/repositories.go index 8bcf2e7..64a3f66 100644 --- a/internal/database/repositories.go +++ b/internal/database/repositories.go @@ -50,21 +50,26 @@ type TxFunc func(ctx context.Context, tx *sql.Tx) error // This method should be used for all write operations to ensure atomicity. func (r *Repositories) WithTx(ctx context.Context, fn TxFunc) error { LogSQL("WithTx", "Beginning transaction", "") + tx, err := r.db.BeginTx(ctx, nil) if err != nil { return fmt.Errorf("beginning transaction: %w", err) } + LogSQL("WithTx", "Transaction started", "") defer func() { if p := recover(); p != nil { - if rollbackErr := tx.Rollback(); rollbackErr != nil { - Fatal("failed to rollback transaction: %v", rollbackErr) + rollbackErr := tx.Rollback() + if rollbackErr != nil { + Fatalf("failed to rollback transaction: %v", rollbackErr) } + panic(p) } else if err != nil { - if rollbackErr := tx.Rollback(); rollbackErr != nil { - Fatal("failed to rollback transaction: %v", rollbackErr) + rollbackErr := tx.Rollback() + if rollbackErr != nil { + Fatalf("failed to rollback transaction: %v", rollbackErr) } } }() @@ -90,6 +95,7 @@ func (r *Repositories) WithReadTx(ctx context.Context, fn TxFunc) error { opts := &sql.TxOptions{ ReadOnly: true, } + tx, err := r.db.BeginTx(ctx, opts) if err != nil { return fmt.Errorf("beginning read transaction: %w", err) @@ -97,13 +103,16 @@ func (r *Repositories) WithReadTx(ctx context.Context, fn TxFunc) error { defer func() { if p := recover(); p != nil { - if rollbackErr := tx.Rollback(); rollbackErr != nil { - Fatal("failed to rollback transaction: %v", rollbackErr) + rollbackErr := tx.Rollback() + if rollbackErr != nil { + Fatalf("failed to rollback transaction: %v", rollbackErr) } + panic(p) } else if err != nil { - if rollbackErr := tx.Rollback(); rollbackErr != nil { - Fatal("failed to rollback transaction: %v", rollbackErr) + rollbackErr := tx.Rollback() + if rollbackErr != nil { + Fatalf("failed to rollback transaction: %v", rollbackErr) } } }() diff --git a/internal/database/repositories_test.go b/internal/database/repositories_test.go index 6324824..485ba61 100644 --- a/internal/database/repositories_test.go +++ b/internal/database/repositories_test.go @@ -3,7 +3,7 @@ package database import ( "context" "database/sql" - "fmt" + "errors" "testing" "time" @@ -28,7 +28,9 @@ func TestRepositoriesTransaction(t *testing.T) { UID: 1000, GID: 1000, } - if err := repos.Files.Create(ctx, tx, file); err != nil { + + err := repos.Files.Create(ctx, tx, file) + if err != nil { return err } @@ -37,7 +39,9 @@ func TestRepositoriesTransaction(t *testing.T) { ChunkHash: types.ChunkHash("tx_chunk1"), Size: 512, } - if err := repos.Chunks.Create(ctx, tx, chunk1); err != nil { + + err = repos.Chunks.Create(ctx, tx, chunk1) + if err != nil { return err } @@ -45,7 +49,9 @@ func TestRepositoriesTransaction(t *testing.T) { ChunkHash: types.ChunkHash("tx_chunk2"), Size: 512, } - if err := repos.Chunks.Create(ctx, tx, chunk2); err != nil { + + err = repos.Chunks.Create(ctx, tx, chunk2) + if err != nil { return err } @@ -55,7 +61,9 @@ func TestRepositoriesTransaction(t *testing.T) { Idx: 0, ChunkHash: chunk1.ChunkHash, } - if err := repos.FileChunks.Create(ctx, tx, fc1); err != nil { + + err = repos.FileChunks.Create(ctx, tx, fc1) + if err != nil { return err } @@ -64,7 +72,9 @@ func TestRepositoriesTransaction(t *testing.T) { Idx: 1, ChunkHash: chunk2.ChunkHash, } - if err := repos.FileChunks.Create(ctx, tx, fc2); err != nil { + + err = repos.FileChunks.Create(ctx, tx, fc2) + if err != nil { return err } @@ -74,7 +84,9 @@ func TestRepositoriesTransaction(t *testing.T) { Hash: types.BlobHash("tx_blob1"), CreatedTS: time.Now().Truncate(time.Second), } - if err := repos.Blobs.Create(ctx, tx, blob); err != nil { + + err = repos.Blobs.Create(ctx, tx, blob) + if err != nil { return err } @@ -85,7 +97,9 @@ func TestRepositoriesTransaction(t *testing.T) { Offset: 0, Length: 512, } - if err := repos.BlobChunks.Create(ctx, tx, bc1); err != nil { + + err = repos.BlobChunks.Create(ctx, tx, bc1) + if err != nil { return err } @@ -95,13 +109,14 @@ func TestRepositoriesTransaction(t *testing.T) { Offset: 512, Length: 512, } - if err := repos.BlobChunks.Create(ctx, tx, bc2); err != nil { + + err = repos.BlobChunks.Create(ctx, tx, bc2) + if err != nil { return err } return nil }) - if err != nil { t.Fatalf("transaction failed: %v", err) } @@ -111,6 +126,7 @@ func TestRepositoriesTransaction(t *testing.T) { if err != nil { t.Fatalf("failed to get file: %v", err) } + if file == nil { t.Error("expected file after transaction") } @@ -119,6 +135,7 @@ func TestRepositoriesTransaction(t *testing.T) { if err != nil { t.Fatalf("failed to get file chunks: %v", err) } + if len(chunks) != 2 { t.Errorf("expected 2 file chunks, got %d", len(chunks)) } @@ -127,6 +144,7 @@ func TestRepositoriesTransaction(t *testing.T) { if err != nil { t.Fatalf("failed to get blob: %v", err) } + if blob == nil { t.Error("expected blob after transaction") } @@ -150,7 +168,9 @@ func TestRepositoriesTransactionRollback(t *testing.T) { UID: 1000, GID: 1000, } - if err := repos.Files.Create(ctx, tx, file); err != nil { + + err := repos.Files.Create(ctx, tx, file) + if err != nil { return err } @@ -159,12 +179,14 @@ func TestRepositoriesTransactionRollback(t *testing.T) { ChunkHash: types.ChunkHash("rollback_chunk"), Size: 1024, } - if err := repos.Chunks.Create(ctx, tx, chunk); err != nil { + + err = repos.Chunks.Create(ctx, tx, chunk) + if err != nil { return err } // Return error to trigger rollback - return fmt.Errorf("intentional rollback") + return errors.New("intentional rollback") }) if err == nil || err.Error() != "intentional rollback" { @@ -176,6 +198,7 @@ func TestRepositoriesTransactionRollback(t *testing.T) { if err != nil { t.Fatalf("error checking for file: %v", err) } + if file != nil { t.Error("file should not exist after rollback") } @@ -184,6 +207,7 @@ func TestRepositoriesTransactionRollback(t *testing.T) { if err != nil { t.Fatalf("error checking for chunk: %v", err) } + if chunk != nil { t.Error("chunk should not exist after rollback") } @@ -205,6 +229,7 @@ func TestRepositoriesReadTransaction(t *testing.T) { UID: 1000, GID: 1000, } + err := repos.Files.Create(ctx, nil, file) if err != nil { t.Fatalf("failed to create file: %v", err) @@ -212,8 +237,10 @@ func TestRepositoriesReadTransaction(t *testing.T) { // Test read-only transaction var retrievedFile *File + err = repos.WithReadTx(ctx, func(ctx context.Context, tx *sql.Tx) error { var err error + retrievedFile, err = repos.Files.GetByPathTx(ctx, tx, "/test/read_file.txt") if err != nil { return err @@ -232,7 +259,6 @@ func TestRepositoriesReadTransaction(t *testing.T) { return nil }) - if err != nil { t.Fatalf("read transaction failed: %v", err) } diff --git a/internal/database/repository_comprehensive_test.go b/internal/database/repository_comprehensive_test.go index 384f80a..f3fce80 100644 --- a/internal/database/repository_comprehensive_test.go +++ b/internal/database/repository_comprehensive_test.go @@ -3,6 +3,7 @@ package database import ( "context" "database/sql" + "errors" "fmt" "testing" "time" @@ -39,6 +40,7 @@ func TestFileRepositoryUUIDGeneration(t *testing.T) { } uuids := make(map[string]bool) + for _, file := range files { err := repo.Create(ctx, nil, file) if err != nil { @@ -54,6 +56,7 @@ func TestFileRepositoryUUIDGeneration(t *testing.T) { if uuids[file.ID.String()] { t.Errorf("duplicate UUID generated: %s", file.ID) } + uuids[file.ID.String()] = true } } @@ -90,16 +93,19 @@ func TestFileRepositoryGetByID(t *testing.T) { if retrieved.ID != file.ID { t.Errorf("ID mismatch: expected %s, got %s", file.ID, retrieved.ID) } + if retrieved.Path != file.Path { t.Errorf("Path mismatch: expected %s, got %s", file.Path, retrieved.Path) } // Test non-existent ID nonExistentID := types.NewFileID() // Generate a new UUID that won't exist in the database + nonExistent, err := repo.GetByID(ctx, nonExistentID) if err != nil { t.Fatalf("GetByID should not return error for non-existent ID: %v", err) } + if nonExistent != nil { t.Error("expected nil for non-existent ID") } @@ -135,6 +141,7 @@ func TestOrphanedFileCleanup(t *testing.T) { if err != nil { t.Fatalf("failed to create file1: %v", err) } + err = repos.Files.Create(ctx, nil, file2) if err != nil { t.Fatalf("failed to create file2: %v", err) @@ -146,6 +153,7 @@ func TestOrphanedFileCleanup(t *testing.T) { Hostname: "test-host", StartedAt: time.Now(), } + err = repos.Snapshots.Create(ctx, nil, snapshot) if err != nil { t.Fatalf("failed to create snapshot: %v", err) @@ -168,6 +176,7 @@ func TestOrphanedFileCleanup(t *testing.T) { if err != nil { t.Fatalf("error getting file: %v", err) } + if orphanedFile != nil { t.Error("orphaned file should have been deleted") } @@ -177,6 +186,7 @@ func TestOrphanedFileCleanup(t *testing.T) { if err != nil { t.Fatalf("error getting file: %v", err) } + if referencedFile == nil { t.Error("referenced file should not have been deleted") } @@ -204,6 +214,7 @@ func TestOrphanedChunkCleanup(t *testing.T) { if err != nil { t.Fatalf("failed to create chunk1: %v", err) } + err = repos.Chunks.Create(ctx, nil, chunk2) if err != nil { t.Fatalf("failed to create chunk2: %v", err) @@ -218,6 +229,7 @@ func TestOrphanedChunkCleanup(t *testing.T) { UID: 1000, GID: 1000, } + err = repos.Files.Create(ctx, nil, file) if err != nil { t.Fatalf("failed to create file: %v", err) @@ -229,6 +241,7 @@ func TestOrphanedChunkCleanup(t *testing.T) { Idx: 0, ChunkHash: chunk2.ChunkHash, } + err = repos.FileChunks.Create(ctx, nil, fc) if err != nil { t.Fatalf("failed to create file chunk: %v", err) @@ -245,6 +258,7 @@ func TestOrphanedChunkCleanup(t *testing.T) { if err != nil { t.Fatalf("error getting chunk: %v", err) } + if orphanedChunk != nil { t.Error("orphaned chunk should have been deleted") } @@ -254,6 +268,7 @@ func TestOrphanedChunkCleanup(t *testing.T) { if err != nil { t.Fatalf("error getting chunk: %v", err) } + if referencedChunk == nil { t.Error("referenced chunk should not have been deleted") } @@ -283,6 +298,7 @@ func TestOrphanedBlobCleanup(t *testing.T) { if err != nil { t.Fatalf("failed to create blob1: %v", err) } + err = repos.Blobs.Create(ctx, nil, blob2) if err != nil { t.Fatalf("failed to create blob2: %v", err) @@ -294,6 +310,7 @@ func TestOrphanedBlobCleanup(t *testing.T) { Hostname: "test-host", StartedAt: time.Now(), } + err = repos.Snapshots.Create(ctx, nil, snapshot) if err != nil { t.Fatalf("failed to create snapshot: %v", err) @@ -316,6 +333,7 @@ func TestOrphanedBlobCleanup(t *testing.T) { if err != nil { t.Fatalf("error getting blob: %v", err) } + if orphanedBlob != nil { t.Error("orphaned blob should have been deleted") } @@ -325,6 +343,7 @@ func TestOrphanedBlobCleanup(t *testing.T) { if err != nil { t.Fatalf("error getting blob: %v", err) } + if referencedBlob == nil { t.Error("referenced blob should not have been deleted") } @@ -347,6 +366,7 @@ func TestFileChunkRepositoryWithUUIDs(t *testing.T) { UID: 1000, GID: 1000, } + err := repos.Files.Create(ctx, nil, file) if err != nil { t.Fatalf("failed to create file: %v", err) @@ -359,6 +379,7 @@ func TestFileChunkRepositoryWithUUIDs(t *testing.T) { ChunkHash: chunkHash, Size: 1024, } + err = repos.Chunks.Create(ctx, nil, chunk) if err != nil { t.Fatalf("failed to create chunk: %v", err) @@ -370,6 +391,7 @@ func TestFileChunkRepositoryWithUUIDs(t *testing.T) { Idx: i, ChunkHash: chunkHash, } + err = repos.FileChunks.Create(ctx, nil, fc) if err != nil { t.Fatalf("failed to create file chunk: %v", err) @@ -381,6 +403,7 @@ func TestFileChunkRepositoryWithUUIDs(t *testing.T) { if err != nil { t.Fatalf("failed to get file chunks: %v", err) } + if len(fileChunks) != 3 { t.Errorf("expected 3 chunks, got %d", len(fileChunks)) } @@ -395,6 +418,7 @@ func TestFileChunkRepositoryWithUUIDs(t *testing.T) { if err != nil { t.Fatalf("failed to get file chunks after delete: %v", err) } + if len(fileChunks) != 0 { t.Errorf("expected 0 chunks after delete, got %d", len(fileChunks)) } @@ -430,6 +454,7 @@ func TestChunkFileRepositoryWithUUIDs(t *testing.T) { if err != nil { t.Fatalf("failed to create file1: %v", err) } + err = repos.Files.Create(ctx, nil, file2) if err != nil { t.Fatalf("failed to create file2: %v", err) @@ -440,6 +465,7 @@ func TestChunkFileRepositoryWithUUIDs(t *testing.T) { ChunkHash: types.ChunkHash("shared-chunk"), Size: 1024, } + err = repos.Chunks.Create(ctx, nil, chunk) if err != nil { t.Fatalf("failed to create chunk: %v", err) @@ -463,6 +489,7 @@ func TestChunkFileRepositoryWithUUIDs(t *testing.T) { if err != nil { t.Fatalf("failed to create chunk file 1: %v", err) } + err = repos.ChunkFiles.Create(ctx, nil, cf2) if err != nil { t.Fatalf("failed to create chunk file 2: %v", err) @@ -473,6 +500,7 @@ func TestChunkFileRepositoryWithUUIDs(t *testing.T) { if err != nil { t.Fatalf("failed to get chunk files: %v", err) } + if len(chunkFiles) != 2 { t.Errorf("expected 2 files for chunk, got %d", len(chunkFiles)) } @@ -482,6 +510,7 @@ func TestChunkFileRepositoryWithUUIDs(t *testing.T) { if err != nil { t.Fatalf("failed to get chunks by file ID: %v", err) } + if len(chunkFiles) != 1 { t.Errorf("expected 1 chunk for file, got %d", len(chunkFiles)) } @@ -528,15 +557,19 @@ func TestSnapshotRepositoryExtendedFields(t *testing.T) { if retrieved.VaultikVersion != snapshot.VaultikVersion { t.Errorf("version mismatch: expected %s, got %s", snapshot.VaultikVersion, retrieved.VaultikVersion) } + if retrieved.VaultikGitRevision != snapshot.VaultikGitRevision { t.Errorf("git revision mismatch: expected %s, got %s", snapshot.VaultikGitRevision, retrieved.VaultikGitRevision) } + if retrieved.CompressionLevel != snapshot.CompressionLevel { t.Errorf("compression level mismatch: expected %d, got %d", snapshot.CompressionLevel, retrieved.CompressionLevel) } + if retrieved.BlobUncompressedSize != snapshot.BlobUncompressedSize { t.Errorf("uncompressed size mismatch: expected %d, got %d", snapshot.BlobUncompressedSize, retrieved.BlobUncompressedSize) } + if retrieved.UploadDurationMs != snapshot.UploadDurationMs { t.Errorf("upload duration mismatch: expected %d, got %d", snapshot.UploadDurationMs, retrieved.UploadDurationMs) } @@ -566,6 +599,7 @@ func TestComplexOrphanedDataScenario(t *testing.T) { if err != nil { t.Fatalf("failed to create snapshot1: %v", err) } + err = repos.Snapshots.Create(ctx, nil, snapshot2) if err != nil { t.Fatalf("failed to create snapshot2: %v", err) @@ -582,6 +616,7 @@ func TestComplexOrphanedDataScenario(t *testing.T) { UID: 1000, GID: 1000, } + err = repos.Files.Create(ctx, nil, files[i]) if err != nil { t.Fatalf("failed to create file%d: %v", i, err) @@ -598,14 +633,17 @@ func TestComplexOrphanedDataScenario(t *testing.T) { if err != nil { t.Fatal(err) } + err = repos.Snapshots.AddFileByID(ctx, nil, snapshot1.ID.String(), files[1].ID) if err != nil { t.Fatal(err) } + err = repos.Snapshots.AddFileByID(ctx, nil, snapshot2.ID.String(), files[1].ID) if err != nil { t.Fatal(err) } + err = repos.Snapshots.AddFileByID(ctx, nil, snapshot2.ID.String(), files[2].ID) if err != nil { t.Fatal(err) @@ -616,6 +654,7 @@ func TestComplexOrphanedDataScenario(t *testing.T) { if err != nil { t.Fatal(err) } + err = repos.Snapshots.Delete(ctx, snapshot1.ID.String()) if err != nil { t.Fatal(err) @@ -633,6 +672,7 @@ func TestComplexOrphanedDataScenario(t *testing.T) { if err != nil { t.Fatalf("error getting file0: %v", err) } + if file0 != nil { t.Error("file0 should have been deleted") } @@ -642,6 +682,7 @@ func TestComplexOrphanedDataScenario(t *testing.T) { if err != nil { t.Fatalf("error getting file1: %v", err) } + if file1 == nil { t.Error("file1 should still exist") } @@ -651,6 +692,7 @@ func TestComplexOrphanedDataScenario(t *testing.T) { if err != nil { t.Fatalf("error getting file2: %v", err) } + if file2 == nil { t.Error("file2 should still exist") } @@ -673,17 +715,19 @@ func TestCascadeDelete(t *testing.T) { UID: 1000, GID: 1000, } + err := repos.Files.Create(ctx, nil, file) if err != nil { t.Fatalf("failed to create file: %v", err) } // Create chunks and file-chunk mappings - for i := 0; i < 3; i++ { + for i := range 3 { chunk := &Chunk{ ChunkHash: types.ChunkHash(fmt.Sprintf("cascade-chunk-%d", i)), Size: 1024, } + err = repos.Chunks.Create(ctx, nil, chunk) if err != nil { t.Fatalf("failed to create chunk: %v", err) @@ -694,6 +738,7 @@ func TestCascadeDelete(t *testing.T) { Idx: i, ChunkHash: chunk.ChunkHash, } + err = repos.FileChunks.Create(ctx, nil, fc) if err != nil { t.Fatalf("failed to create file chunk: %v", err) @@ -705,6 +750,7 @@ func TestCascadeDelete(t *testing.T) { if err != nil { t.Fatal(err) } + if len(fileChunks) != 3 { t.Errorf("expected 3 file chunks, got %d", len(fileChunks)) } @@ -720,6 +766,7 @@ func TestCascadeDelete(t *testing.T) { if err != nil { t.Fatal(err) } + if len(fileChunks) != 0 { t.Errorf("expected 0 file chunks after cascade delete, got %d", len(fileChunks)) } @@ -744,6 +791,7 @@ func TestTransactionIsolation(t *testing.T) { UID: 1000, GID: 1000, } + err := repos.Files.Create(ctx, tx, file) if err != nil { return err @@ -754,9 +802,8 @@ func TestTransactionIsolation(t *testing.T) { // For now, we'll just test that rollback works // Return an error to trigger rollback - return fmt.Errorf("intentional rollback") + return errors.New("intentional rollback") }) - if err == nil { t.Fatal("expected error from transaction") } @@ -766,6 +813,7 @@ func TestTransactionIsolation(t *testing.T) { if err != nil { t.Fatal(err) } + if len(files) != 0 { t.Error("file should not exist after rollback") } @@ -790,13 +838,14 @@ func TestConcurrentOrphanedCleanup(t *testing.T) { Hostname: "test-host", StartedAt: time.Now(), } + err := repos.Snapshots.Create(ctx, nil, snapshot) if err != nil { t.Fatal(err) } // Create many files, some orphaned - for i := 0; i < 20; i++ { + for i := range 20 { file := &File{ Path: types.FilePath(fmt.Sprintf("/concurrent-%d.txt", i)), MTime: time.Now().Truncate(time.Second), @@ -805,6 +854,7 @@ func TestConcurrentOrphanedCleanup(t *testing.T) { UID: 1000, GID: 1000, } + err = repos.Files.Create(ctx, nil, file) if err != nil { t.Fatal(err) @@ -822,14 +872,15 @@ func TestConcurrentOrphanedCleanup(t *testing.T) { // Run multiple cleanup operations concurrently // Note: SQLite has limited support for concurrent writes, so we expect some to fail done := make(chan error, 3) - for i := 0; i < 3; i++ { + + for range 3 { go func() { done <- repos.Files.DeleteOrphaned(ctx) }() } // Wait for all to complete - for i := 0; i < 3; i++ { + for i := range 3 { err := <-done if err != nil { t.Errorf("cleanup %d failed: %v", i, err) @@ -850,10 +901,12 @@ func TestConcurrentOrphanedCleanup(t *testing.T) { // Verify all remaining files are even-numbered for _, file := range files { var num int + _, err := fmt.Sscanf(file.Path.String(), "/concurrent-%d.txt", &num) if err != nil { t.Logf("failed to parse file number from %s: %v", file.Path, err) } + if num%2 != 0 { t.Errorf("odd-numbered file %s should have been deleted", file.Path) } diff --git a/internal/database/repository_debug_test.go b/internal/database/repository_debug_test.go index 2bd9493..9deedaa 100644 --- a/internal/database/repository_debug_test.go +++ b/internal/database/repository_debug_test.go @@ -36,12 +36,14 @@ func TestOrphanedFileCleanupDebug(t *testing.T) { if err != nil { t.Fatalf("failed to create file1: %v", err) } + t.Logf("Created file1 with ID: %s", file1.ID) err = repos.Files.Create(ctx, nil, file2) if err != nil { t.Fatalf("failed to create file2: %v", err) } + t.Logf("Created file2 with ID: %s", file2.ID) // Create a snapshot and reference only file2 @@ -50,18 +52,22 @@ func TestOrphanedFileCleanupDebug(t *testing.T) { Hostname: "test-host", StartedAt: time.Now(), } + err = repos.Snapshots.Create(ctx, nil, snapshot) if err != nil { t.Fatalf("failed to create snapshot: %v", err) } + t.Logf("Created snapshot: %s", snapshot.ID) // Check snapshot_files before adding var count int + err = db.conn.QueryRow("SELECT COUNT(*) FROM snapshot_files").Scan(&count) if err != nil { t.Fatal(err) } + t.Logf("snapshot_files count before add: %d", count) // Add file2 to snapshot @@ -69,6 +75,7 @@ func TestOrphanedFileCleanupDebug(t *testing.T) { if err != nil { t.Fatalf("failed to add file to snapshot: %v", err) } + t.Logf("Added file2 to snapshot") // Check snapshot_files after adding @@ -76,6 +83,7 @@ func TestOrphanedFileCleanupDebug(t *testing.T) { if err != nil { t.Fatal(err) } + t.Logf("snapshot_files count after add: %d", count) // Check which files are referenced @@ -84,16 +92,22 @@ func TestOrphanedFileCleanupDebug(t *testing.T) { t.Fatal(err) } defer func() { - if err := rows.Close(); err != nil { + err := rows.Close() + if err != nil { t.Logf("failed to close rows: %v", err) } }() + t.Log("Files in snapshot_files:") + for rows.Next() { var fileID string - if err := rows.Scan(&fileID); err != nil { + + err := rows.Scan(&fileID) + if err != nil { t.Fatal(err) } + t.Logf(" - %s", fileID) } @@ -102,6 +116,7 @@ func TestOrphanedFileCleanupDebug(t *testing.T) { if err != nil { t.Fatal(err) } + t.Logf("Files count before cleanup: %d", count) // Run orphaned cleanup @@ -109,6 +124,7 @@ func TestOrphanedFileCleanupDebug(t *testing.T) { if err != nil { t.Fatalf("failed to delete orphaned files: %v", err) } + t.Log("Ran orphaned cleanup") // Check files after cleanup @@ -116,6 +132,7 @@ func TestOrphanedFileCleanupDebug(t *testing.T) { if err != nil { t.Fatal(err) } + t.Logf("Files count after cleanup: %d", count) // List remaining files @@ -123,7 +140,9 @@ func TestOrphanedFileCleanupDebug(t *testing.T) { if err != nil { t.Fatal(err) } + t.Log("Remaining files:") + for _, f := range files { t.Logf(" - ID: %s, Path: %s", f.ID, f.Path) } @@ -133,10 +152,12 @@ func TestOrphanedFileCleanupDebug(t *testing.T) { if err != nil { t.Fatalf("error getting file: %v", err) } + if orphanedFile != nil { t.Error("orphaned file should have been deleted") // Let's check why it wasn't deleted var exists bool + err = db.conn.QueryRow(` SELECT EXISTS( SELECT 1 FROM snapshot_files @@ -145,6 +166,7 @@ func TestOrphanedFileCleanupDebug(t *testing.T) { if err != nil { t.Fatal(err) } + t.Logf("File1 exists in snapshot_files: %v", exists) } else { t.Log("Orphaned file was correctly deleted") @@ -155,6 +177,7 @@ func TestOrphanedFileCleanupDebug(t *testing.T) { if err != nil { t.Fatalf("error getting file: %v", err) } + if referencedFile == nil { t.Error("referenced file should not have been deleted") } else { diff --git a/internal/database/repository_edge_cases_test.go b/internal/database/repository_edge_cases_test.go index b4ba6ba..666f33f 100644 --- a/internal/database/repository_edge_cases_test.go +++ b/internal/database/repository_edge_cases_test.go @@ -98,6 +98,7 @@ func TestFileRepositoryEdgeCases(t *testing.T) { if (err != nil) != tt.wantErr { t.Errorf("Create() error = %v, wantErr %v", err, tt.wantErr) } + if err != nil && tt.errMsg != "" && !strings.Contains(err.Error(), tt.errMsg) { t.Errorf("Create() error = %v, want error containing %q", err, tt.errMsg) } @@ -136,6 +137,7 @@ func TestDuplicateHandling(t *testing.T) { if err != nil { t.Fatalf("failed to create file1: %v", err) } + originalID := file1.ID // Create with same path should update the existing record (UPSERT behavior) @@ -190,6 +192,7 @@ func TestDuplicateHandling(t *testing.T) { UID: 1000, GID: 1000, } + err := repos.Files.Create(ctx, nil, file) if err != nil { t.Fatal(err) @@ -199,6 +202,7 @@ func TestDuplicateHandling(t *testing.T) { ChunkHash: types.ChunkHash("test-chunk-dup"), Size: 1024, } + err = repos.Chunks.Create(ctx, nil, chunk) if err != nil { t.Fatal(err) @@ -325,6 +329,7 @@ func TestLargeDatasets(t *testing.T) { Hostname: "test-host", StartedAt: time.Now(), } + err := repos.Snapshots.Create(ctx, nil, snapshot) if err != nil { t.Fatal(err) @@ -332,11 +337,12 @@ func TestLargeDatasets(t *testing.T) { // Create many files const fileCount = 1000 + fileIDs := make([]types.FileID, fileCount) t.Run("create many files", func(t *testing.T) { start := time.Now() - for i := 0; i < fileCount; i++ { + for i := range fileCount { file := &File{ Path: types.FilePath(fmt.Sprintf("/large/file%05d.txt", i)), MTime: time.Now(), @@ -345,10 +351,12 @@ func TestLargeDatasets(t *testing.T) { UID: uint32(1000 + (i % 10)), GID: uint32(1000 + (i % 10)), } + err := repos.Files.Create(ctx, nil, file) if err != nil { t.Fatalf("failed to create file %d: %v", i, err) } + fileIDs[i] = file.ID // Add half to snapshot @@ -359,29 +367,35 @@ func TestLargeDatasets(t *testing.T) { } } } + t.Logf("Created %d files in %v", fileCount, time.Since(start)) }) // Test ListByPrefix performance t.Run("list by prefix performance", func(t *testing.T) { start := time.Now() + files, err := repos.Files.ListByPrefix(ctx, "/large/") if err != nil { t.Fatal(err) } + if len(files) != fileCount { t.Errorf("expected %d files, got %d", fileCount, len(files)) } + t.Logf("Listed %d files in %v", len(files), time.Since(start)) }) // Test orphaned cleanup performance t.Run("orphaned cleanup performance", func(t *testing.T) { start := time.Now() + err := repos.Files.DeleteOrphaned(ctx) if err != nil { t.Fatal(err) } + t.Logf("Cleaned up orphaned files in %v", time.Since(start)) // Verify correct number remain @@ -389,6 +403,7 @@ func TestLargeDatasets(t *testing.T) { if err != nil { t.Fatal(err) } + if len(files) != fileCount/2 { t.Errorf("expected %d files after cleanup, got %d", fileCount/2, len(files)) } @@ -409,6 +424,7 @@ func TestErrorPropagation(t *testing.T) { if err != nil { t.Errorf("GetByID should not return error for non-existent ID, got: %v", err) } + if file != nil { t.Error("expected nil file for non-existent ID") } @@ -420,6 +436,7 @@ func TestErrorPropagation(t *testing.T) { if err != nil { t.Errorf("GetByPath should not return error for non-existent path, got: %v", err) } + if file != nil { t.Error("expected nil file for non-existent path") } @@ -432,10 +449,12 @@ func TestErrorPropagation(t *testing.T) { Idx: 0, ChunkHash: types.ChunkHash("some-chunk"), } + err := repos.FileChunks.Create(ctx, nil, fc) if err == nil { t.Error("expected error for invalid foreign key") } + if !strings.Contains(err.Error(), "FOREIGN KEY") { t.Errorf("expected foreign key error, got: %v", err) } @@ -475,6 +494,7 @@ func TestQueryInjection(t *testing.T) { // Verify tables still exist var count int + err := db.conn.QueryRow("SELECT COUNT(*) FROM files").Scan(&count) if err != nil { t.Fatal("files table was damaged by injection") diff --git a/internal/database/snapshots.go b/internal/database/snapshots.go index 93cc517..7b620a2 100644 --- a/internal/database/snapshots.go +++ b/internal/database/snapshots.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "fmt" + "strings" "time" "sneak.berlin/go/vaultik/internal/types" @@ -26,6 +27,7 @@ func (r *SnapshotRepository) Create(ctx context.Context, tx *sql.Tx, snapshot *S ` var completedAt *int64 + if snapshot.CompletedAt != nil { ts := snapshot.CompletedAt.Unix() completedAt = &ts @@ -84,9 +86,11 @@ func (r *SnapshotRepository) UpdateCounts(ctx context.Context, tx *sql.Tx, snaps func (r *SnapshotRepository) UpdateExtendedStats(ctx context.Context, tx *sql.Tx, snapshotID string, blobUncompressedSize int64, compressionLevel int, uploadDurationMs int64) error { // Calculate compression ratio based on uncompressed vs compressed sizes var compressionRatio float64 + if blobUncompressedSize > 0 { // Get current blob_size from DB to calculate ratio var blobSize int64 + queryGet := `SELECT blob_size FROM snapshots WHERE id = ?` if tx != nil { err := tx.QueryRowContext(ctx, queryGet, snapshotID).Scan(&blobSize) @@ -99,6 +103,7 @@ func (r *SnapshotRepository) UpdateExtendedStats(ctx context.Context, tx *sql.Tx return fmt.Errorf("getting blob size: %w", err) } } + compressionRatio = float64(blobSize) / float64(blobUncompressedSize) } else { compressionRatio = 1.0 @@ -124,6 +129,7 @@ func (r *SnapshotRepository) UpdateExtendedStats(ctx context.Context, tx *sql.Tx if err != nil { return fmt.Errorf("updating extended stats: %w", err) } + return nil } @@ -136,9 +142,11 @@ func (r *SnapshotRepository) GetByID(ctx context.Context, snapshotID string) (*S WHERE id = ? ` - var snapshot Snapshot - var startedAtUnix int64 - var completedAtUnix *int64 + var ( + snapshot Snapshot + startedAtUnix int64 + completedAtUnix *int64 + ) err := r.db.conn.QueryRowContext(ctx, query, snapshotID).Scan( &snapshot.ID, @@ -162,6 +170,7 @@ func (r *SnapshotRepository) GetByID(ctx context.Context, snapshotID string) (*S if err == sql.ErrNoRows { return nil, nil } + if err != nil { return nil, fmt.Errorf("querying snapshot: %w", err) } @@ -190,10 +199,13 @@ func (r *SnapshotRepository) ListRecent(ctx context.Context, limit int) ([]*Snap defer CloseRows(rows) var snapshots []*Snapshot + for rows.Next() { - var snapshot Snapshot - var startedAtUnix int64 - var completedAtUnix *int64 + var ( + snapshot Snapshot + startedAtUnix int64 + completedAtUnix *int64 + ) err := rows.Scan( &snapshot.ID, @@ -301,28 +313,35 @@ func (r *SnapshotRepository) AddFilesByIDBatch(ctx context.Context, tx *sql.Tx, const batchSize = 400 for i := 0; i < len(fileIDs); i += batchSize { - end := i + batchSize - if end > len(fileIDs) { - end = len(fileIDs) - } + end := min(i+batchSize, len(fileIDs)) + batch := fileIDs[i:end] query := "INSERT OR IGNORE INTO snapshot_files (snapshot_id, file_id) VALUES " - args := make([]interface{}, 0, len(batch)*2) + + args := make([]any, 0, len(batch)*2) + + var querySb312 strings.Builder + for j, fileID := range batch { if j > 0 { - query += ", " + querySb312.WriteString(", ") } - query += "(?, ?)" + + querySb312.WriteString("(?, ?)") + args = append(args, snapshotID, fileID.String()) } + query += querySb312.String() + var err error if tx != nil { _, err = tx.ExecContext(ctx, query, args...) } else { _, err = r.db.ExecWithLog(ctx, query, args...) } + if err != nil { return fmt.Errorf("batch adding files to snapshot: %w", err) } @@ -353,18 +372,22 @@ func (r *SnapshotRepository) PopulateReferencedBlobs(ctx context.Context, tx *sq AND blobs.blob_hash IS NOT NULL ` - var result sql.Result - var err error + var ( + result sql.Result + err error + ) if tx != nil { result, err = tx.ExecContext(ctx, query, snapshotID, snapshotID) } else { result, err = r.db.ExecWithLog(ctx, query, snapshotID, snapshotID) } + if err != nil { return 0, fmt.Errorf("populating referenced blobs: %w", err) } n, _ := result.RowsAffected() + return n, nil } @@ -405,11 +428,15 @@ func (r *SnapshotRepository) GetBlobHashes(ctx context.Context, snapshotID strin defer CloseRows(rows) var blobs []string + for rows.Next() { var blobHash string - if err := rows.Scan(&blobHash); err != nil { + + err := rows.Scan(&blobHash) + if err != nil { return nil, fmt.Errorf("scanning blob hash: %w", err) } + blobs = append(blobs, blobHash) } @@ -426,6 +453,7 @@ func (r *SnapshotRepository) GetSnapshotTotalCompressedSize(ctx context.Context, ` var totalSize int64 + err := r.db.conn.QueryRowContext(ctx, query, snapshotID).Scan(&totalSize) if err != nil { return 0, fmt.Errorf("querying total compressed size: %w", err) @@ -449,6 +477,7 @@ func (r *SnapshotRepository) GetSnapshotUncompressedChunkSize(ctx context.Contex ` var totalSize int64 + err := r.db.conn.QueryRowContext(ctx, query, snapshotID).Scan(&totalSize) if err != nil { return 0, fmt.Errorf("querying uncompressed chunk size: %w", err) @@ -485,6 +514,7 @@ func (r *SnapshotRepository) GetSnapshotNewChunkSize(ctx context.Context, snapsh ` var totalSize int64 + err := r.db.conn.QueryRowContext(ctx, query, snapshotID, snapshotID, snapshotID).Scan(&totalSize) if err != nil { return 0, fmt.Errorf("querying new chunk size: %w", err) @@ -509,10 +539,13 @@ func (r *SnapshotRepository) GetIncompleteSnapshots(ctx context.Context) ([]*Sna defer CloseRows(rows) var snapshots []*Snapshot + for rows.Next() { - var snapshot Snapshot - var startedAtUnix int64 - var completedAtUnix *int64 + var ( + snapshot Snapshot + startedAtUnix int64 + completedAtUnix *int64 + ) err := rows.Scan( &snapshot.ID, @@ -560,10 +593,13 @@ func (r *SnapshotRepository) GetIncompleteByHostname(ctx context.Context, hostna defer CloseRows(rows) var snapshots []*Snapshot + for rows.Next() { - var snapshot Snapshot - var startedAtUnix int64 - var completedAtUnix *int64 + var ( + snapshot Snapshot + startedAtUnix int64 + completedAtUnix *int64 + ) err := rows.Scan( &snapshot.ID, diff --git a/internal/database/snapshots_test.go b/internal/database/snapshots_test.go index c856163..a1efaa1 100644 --- a/internal/database/snapshots_test.go +++ b/internal/database/snapshots_test.go @@ -52,15 +52,19 @@ func TestSnapshotRepository(t *testing.T) { if err != nil { t.Fatalf("failed to get snapshot: %v", err) } + if retrieved == nil { t.Fatal("expected snapshot, got nil") } + if retrieved.ID != snapshot.ID { t.Errorf("ID mismatch: got %s, want %s", retrieved.ID, snapshot.ID) } + if retrieved.Hostname != snapshot.Hostname { t.Errorf("hostname mismatch: got %s, want %s", retrieved.Hostname, snapshot.Hostname) } + if retrieved.FileCount != snapshot.FileCount { t.Errorf("file count mismatch: got %d, want %d", retrieved.FileCount, snapshot.FileCount) } @@ -75,21 +79,27 @@ func TestSnapshotRepository(t *testing.T) { if err != nil { t.Fatalf("failed to get updated snapshot: %v", err) } + if retrieved.FileCount != 200 { t.Errorf("file count not updated: got %d, want %d", retrieved.FileCount, 200) } + if retrieved.ChunkCount != 1000 { t.Errorf("chunk count not updated: got %d, want %d", retrieved.ChunkCount, 1000) } + if retrieved.BlobCount != 20 { t.Errorf("blob count not updated: got %d, want %d", retrieved.BlobCount, 20) } + if retrieved.TotalSize != twoHundredMebibytes { t.Errorf("total size not updated: got %d, want %d", retrieved.TotalSize, twoHundredMebibytes) } + if retrieved.BlobSize != sixtyMebibytes { t.Errorf("blob size not updated: got %d, want %d", retrieved.BlobSize, sixtyMebibytes) } + expectedRatio := compressionRatioPoint3 // 0.3 if math.Abs(retrieved.CompressionRatio-expectedRatio) > 0.001 { t.Errorf("compression ratio not updated: got %f, want %f", retrieved.CompressionRatio, expectedRatio) @@ -108,6 +118,7 @@ func TestSnapshotRepository(t *testing.T) { ChunkCount: int64(500 * i), BlobCount: int64(10 * i), } + err := repo.Create(ctx, nil, s) if err != nil { t.Fatalf("failed to create snapshot %d: %v", i, err) @@ -119,12 +130,13 @@ func TestSnapshotRepository(t *testing.T) { if err != nil { t.Fatalf("failed to list recent snapshots: %v", err) } + if len(recent) != 3 { t.Errorf("expected 3 recent snapshots, got %d", len(recent)) } // Verify order (most recent first) - for i := 0; i < len(recent)-1; i++ { + for i := range len(recent) - 1 { if recent[i].StartedAt.Before(recent[i+1].StartedAt) { t.Error("snapshots not in descending order") } @@ -143,6 +155,7 @@ func TestSnapshotRepositoryNotFound(t *testing.T) { if err != nil { t.Fatalf("unexpected error: %v", err) } + if snapshot != nil { t.Error("expected nil for non-existent snapshot") } diff --git a/internal/database/uploads.go b/internal/database/uploads.go index c1a3c09..154cd09 100644 --- a/internal/database/uploads.go +++ b/internal/database/uploads.go @@ -53,6 +53,7 @@ func (r *UploadRepository) GetByBlobHash(ctx context.Context, blobHash string) ( ` var upload Upload + err := r.conn.QueryRowContext(ctx, query, blobHash).Scan( &upload.BlobHash, &upload.UploadedAt, @@ -63,6 +64,7 @@ func (r *UploadRepository) GetByBlobHash(ctx context.Context, blobHash string) ( if err == sql.ErrNoRows { return nil, nil } + if err != nil { return nil, err } @@ -84,17 +86,22 @@ func (r *UploadRepository) GetRecentUploads(ctx context.Context, limit int) ([]* return nil, err } defer func() { - if err := rows.Close(); err != nil { + err := rows.Close() + if err != nil { log.Error("failed to close rows", "error", err) } }() var uploads []*Upload + for rows.Next() { var upload Upload - if err := rows.Scan(&upload.BlobHash, &upload.UploadedAt, &upload.Size, &upload.DurationMs); err != nil { + + err := rows.Scan(&upload.BlobHash, &upload.UploadedAt, &upload.Size, &upload.DurationMs) + if err != nil { return nil, err } + uploads = append(uploads, &upload) } @@ -115,6 +122,7 @@ func (r *UploadRepository) GetUploadStats(ctx context.Context, since time.Time) ` var stats UploadStats + err := r.conn.QueryRowContext(ctx, query, since).Scan( &stats.Count, &stats.TotalSize, @@ -138,10 +146,13 @@ type UploadStats struct { // GetCountBySnapshot returns the count of uploads for a specific snapshot func (r *UploadRepository) GetCountBySnapshot(ctx context.Context, snapshotID string) (int64, error) { query := `SELECT COUNT(*) FROM uploads WHERE snapshot_id = ?` + var count int64 + err := r.conn.QueryRowContext(ctx, query, snapshotID).Scan(&count) if err != nil { return 0, err } + return count, nil } diff --git a/internal/globals/globals.go b/internal/globals/globals.go index 80c538b..21edab6 100644 --- a/internal/globals/globals.go +++ b/internal/globals/globals.go @@ -50,5 +50,6 @@ func (g *Globals) ShortCommit() string { if len(g.Commit) > 12 { return g.Commit[:12] } + return g.Commit } diff --git a/internal/log/log.go b/internal/log/log.go index 2843d25..c9867ee 100644 --- a/internal/log/log.go +++ b/internal/log/log.go @@ -84,6 +84,7 @@ func getCaller(skip int) string { if !ok { return "unknown" } + return fmt.Sprintf("%s:%d", filepath.Base(file), line) } @@ -94,6 +95,7 @@ func Fatal(msg string, args ...any) { args = append(args, "caller", getCaller(2)) logger.Error(msg, args...) } + os.Exit(1) } @@ -172,6 +174,7 @@ func With(args ...any) *slog.Logger { if logger != nil { return logger.With(args...) } + return slog.Default() } diff --git a/internal/log/tty_handler.go b/internal/log/tty_handler.go index 91ab803..81c1fe6 100644 --- a/internal/log/tty_handler.go +++ b/internal/log/tty_handler.go @@ -33,6 +33,7 @@ func NewTTYHandler(out io.Writer, opts *slog.HandlerOptions) *TTYHandler { if opts == nil { opts = &slog.HandlerOptions{} } + return &TTYHandler{ out: out, opts: *opts, @@ -54,7 +55,9 @@ func (h *TTYHandler) Handle(_ context.Context, r slog.Record) error { // Level and color level := r.Level.String() + var levelColor string + switch r.Level { case slog.LevelDebug: levelColor = colorGray @@ -96,10 +99,12 @@ func (h *TTYHandler) Handle(_ context.Context, r slog.Record) error { _, _ = fmt.Fprintf(h.out, " %s%s%s=%s%s%s", colorCyan, a.Key, colorReset, colorBlue, value, colorReset) + return true }) _, _ = fmt.Fprintln(h.out) + return nil } @@ -122,6 +127,7 @@ func formatDuration(d time.Duration) string { } else if d < time.Minute { return fmt.Sprintf("%.1fs", d.Seconds()) } + return d.String() } @@ -131,10 +137,12 @@ func formatBytes(b int64) string { if b < unit { return fmt.Sprintf("%d B", b) } + div, exp := int64(unit), 0 for n := b / unit; n >= unit; n /= unit { div *= unit exp++ } + return fmt.Sprintf("%.1f %cB", float64(b)/float64(div), "KMGTPE"[exp]) } diff --git a/internal/pidlock/pidlock.go b/internal/pidlock/pidlock.go index dfe0306..e576970 100644 --- a/internal/pidlock/pidlock.go +++ b/internal/pidlock/pidlock.go @@ -77,6 +77,7 @@ func (l *Lock) Release() error { } l.path = "" // Prevent double-release + return nil } @@ -104,5 +105,6 @@ func isProcessRunning(pid int) bool { // On Unix, FindProcess always succeeds. We need to send signal 0 to check. err = process.Signal(syscall.Signal(0)) + return err == nil } diff --git a/internal/pidlock/pidlock_test.go b/internal/pidlock/pidlock_test.go index d256ee1..dce224d 100644 --- a/internal/pidlock/pidlock_test.go +++ b/internal/pidlock/pidlock_test.go @@ -40,6 +40,7 @@ func TestAcquireBlocksSecondInstance(t *testing.T) { // Acquire first lock lock1, err := Acquire(tmpDir) require.NoError(t, err) + require.NotNil(t, lock1) defer func() { _ = lock1.Release() }() @@ -61,6 +62,7 @@ func TestAcquireWithStaleLock(t *testing.T) { // Should be able to acquire lock (stale lock is cleaned up) lock, err := Acquire(tmpDir) require.NoError(t, err) + require.NotNil(t, lock) defer func() { _ = lock.Release() }() @@ -88,6 +90,7 @@ func TestReleaseIsIdempotent(t *testing.T) { func TestReleaseNilLock(t *testing.T) { var lock *Lock + err := lock.Release() assert.NoError(t, err) } @@ -98,6 +101,7 @@ func TestAcquireCreatesDirectory(t *testing.T) { lock, err := Acquire(nestedDir) require.NoError(t, err) + require.NotNil(t, lock) defer func() { _ = lock.Release() }() diff --git a/internal/s3/client.go b/internal/s3/client.go index 2861be7..1079b8f 100644 --- a/internal/s3/client.go +++ b/internal/s3/client.go @@ -42,7 +42,7 @@ type Config struct { // Used to suppress SDK warnings about checksums. type nopLogger struct{} -func (nopLogger) Logf(classification logging.Classification, format string, v ...interface{}) {} +func (nopLogger) Logf(classification logging.Classification, format string, v ...any) {} // NewClient creates a new S3 client with the provided configuration. // It establishes a connection to the S3-compatible storage service and @@ -92,6 +92,7 @@ func (c *Client) PutObject(ctx context.Context, key string, data io.Reader) erro Key: aws.String(fullKey), Body: data, }) + return err } @@ -137,6 +138,7 @@ func (c *Client) PutObjectWithProgress(ctx context.Context, key string, data io. // close the returned reader when done to avoid resource leaks. func (c *Client) GetObject(ctx context.Context, key string) (io.ReadCloser, error) { fullKey := c.prefix + key + result, err := c.s3Client.GetObject(ctx, &s3.GetObjectInput{ Bucket: aws.String(c.bucket), Key: aws.String(fullKey), @@ -144,6 +146,7 @@ func (c *Client) GetObject(ctx context.Context, key string) (io.ReadCloser, erro if err != nil { return nil, err } + return result.Body, nil } @@ -156,6 +159,7 @@ func (c *Client) DeleteObject(ctx context.Context, key string) error { Bucket: aws.String(c.bucket), Key: aws.String(fullKey), }) + return err } @@ -168,6 +172,7 @@ func (c *Client) ListObjects(ctx context.Context, prefix string) ([]string, erro fullPrefix := c.prefix + prefix var keys []string + paginator := s3.NewListObjectsV2Paginator(c.s3Client, &s3.ListObjectsV2Input{ Bucket: aws.String(c.bucket), Prefix: aws.String(fullPrefix), @@ -186,6 +191,7 @@ func (c *Client) ListObjects(ctx context.Context, prefix string) ([]string, erro if len(key) > len(c.prefix) { key = key[len(c.prefix):] } + keys = append(keys, key) } } @@ -200,18 +206,23 @@ func (c *Client) ListObjects(ctx context.Context, prefix string) ([]string, erro // Note: This method returns false for any error, not just "not found". func (c *Client) HeadObject(ctx context.Context, key string) (bool, error) { fullKey := c.prefix + key + _, err := c.s3Client.HeadObject(ctx, &s3.HeadObjectInput{ Bucket: aws.String(c.bucket), Key: aws.String(fullKey), }) if err != nil { - var notFound *s3types.NotFound - var noSuchKey *s3types.NoSuchKey + var ( + notFound *s3types.NotFound + noSuchKey *s3types.NoSuchKey + ) if errors.As(err, ¬Found) || errors.As(err, &noSuchKey) { return false, nil } + return false, err } + return true, nil } @@ -247,6 +258,7 @@ func (c *Client) ListObjectsStream(ctx context.Context, prefix string, recursive page, err := paginator.NextPage(ctx) if err != nil { ch <- ObjectInfo{Err: err} + return } @@ -257,6 +269,7 @@ func (c *Client) ListObjectsStream(ctx context.Context, prefix string, recursive if len(key) > len(c.prefix) { key = key[len(c.prefix):] } + ch <- ObjectInfo{ Key: key, Size: *obj.Size, @@ -275,6 +288,7 @@ func (c *Client) ListObjectsStream(ctx context.Context, prefix string, recursive // Returns an error if the object doesn't exist or if the operation fails. func (c *Client) StatObject(ctx context.Context, key string) (*ObjectInfo, error) { fullKey := c.prefix + key + result, err := c.s3Client.HeadObject(ctx, &s3.HeadObjectInput{ Bucket: aws.String(c.bucket), Key: aws.String(fullKey), @@ -313,6 +327,7 @@ func (c *Client) Endpoint() string { if c.endpoint == "" { return "s3.amazonaws.com" } + return c.endpoint } @@ -329,11 +344,14 @@ func (pr *progressReader) Read(p []byte) (int, error) { n, err := pr.reader.Read(p) if n > 0 { atomic.AddInt64(&pr.read, int64(n)) + if pr.callback != nil { - if callbackErr := pr.callback(atomic.LoadInt64(&pr.read)); callbackErr != nil { + callbackErr := pr.callback(atomic.LoadInt64(&pr.read)) + if callbackErr != nil { return n, callbackErr } } } + return n, err } diff --git a/internal/s3/client_test.go b/internal/s3/client_test.go index 1bb267e..7ebab27 100644 --- a/internal/s3/client_test.go +++ b/internal/s3/client_test.go @@ -12,7 +12,8 @@ import ( func TestClient(t *testing.T) { ts := NewTestServer(t) defer func() { - if err := ts.Cleanup(); err != nil { + err := ts.Cleanup() + if err != nil { t.Errorf("cleanup failed: %v", err) } }() @@ -35,6 +36,7 @@ func TestClient(t *testing.T) { // Test PutObject testKey := "foo/bar.txt" testData := []byte("test data") + err = client.PutObject(ctx, testKey, bytes.NewReader(testData)) if err != nil { t.Fatalf("failed to put object: %v", err) @@ -46,7 +48,8 @@ func TestClient(t *testing.T) { t.Fatalf("failed to get object: %v", err) } defer func() { - if err := reader.Close(); err != nil { + err := reader.Close() + if err != nil { t.Errorf("failed to close reader: %v", err) } }() @@ -65,6 +68,7 @@ func TestClient(t *testing.T) { if err != nil { t.Fatalf("failed to head object: %v", err) } + if !exists { t.Error("expected object to exist") } @@ -74,9 +78,11 @@ func TestClient(t *testing.T) { if err != nil { t.Fatalf("failed to list objects: %v", err) } + if len(keys) != 1 { t.Errorf("expected 1 key, got %d", len(keys)) } + if keys[0] != testKey { t.Errorf("unexpected key: got %s, want %s", keys[0], testKey) } @@ -92,6 +98,7 @@ func TestClient(t *testing.T) { if err != nil { t.Fatalf("failed to head object after deletion: %v", err) } + if exists { t.Error("expected object to not exist after deletion") } diff --git a/internal/s3/s3_test.go b/internal/s3/s3_test.go index bc359f0..57fca9a 100644 --- a/internal/s3/s3_test.go +++ b/internal/s3/s3_test.go @@ -57,7 +57,8 @@ func NewTestServer(t *testing.T) *TestServer { // Start server in background go func() { - if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed { + err := server.ListenAndServe() + if err != nil && err != http.ErrServerClosed { t.Logf("test server error: %v", err) } }() @@ -77,7 +78,7 @@ func NewTestServer(t *testing.T) *TestServer { "", )), config.WithClientLogMode(aws.LogRetries|aws.LogRequestWithBody|aws.LogResponseWithBody), - config.WithLogger(logging.LoggerFunc(func(classification logging.Classification, format string, v ...interface{}) { + config.WithLogger(logging.LoggerFunc(func(classification logging.Classification, format string, v ...any) { // Capture logs to buffer instead of stdout fmt.Fprintf(logBuf, "SDK %s %s %s\n", time.Now().Format("2006/01/02 15:04:05"), @@ -125,7 +126,8 @@ func (ts *TestServer) Cleanup() error { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - if err := ts.server.Shutdown(ctx); err != nil { + err := ts.server.Shutdown(ctx) + if err != nil { return err } @@ -141,7 +143,8 @@ func (ts *TestServer) Client() *s3.Client { func TestBasicS3Operations(t *testing.T) { ts := NewTestServer(t) defer func() { - if err := ts.Cleanup(); err != nil { + err := ts.Cleanup() + if err != nil { t.Errorf("cleanup failed: %v", err) } }() @@ -172,7 +175,8 @@ func TestBasicS3Operations(t *testing.T) { t.Fatalf("failed to get object: %v", err) } defer func() { - if err := result.Body.Close(); err != nil { + err := result.Body.Close() + if err != nil { t.Errorf("failed to close body: %v", err) } }() @@ -192,7 +196,8 @@ func TestBasicS3Operations(t *testing.T) { func TestBlobOperations(t *testing.T) { ts := NewTestServer(t) defer func() { - if err := ts.Cleanup(); err != nil { + err := ts.Cleanup() + if err != nil { t.Errorf("cleanup failed: %v", err) } }() @@ -255,7 +260,8 @@ func TestBlobOperations(t *testing.T) { func TestMetadataOperations(t *testing.T) { ts := NewTestServer(t) defer func() { - if err := ts.Cleanup(); err != nil { + err := ts.Cleanup() + if err != nil { t.Errorf("cleanup failed: %v", err) } }() diff --git a/internal/snapshot/backup_test.go b/internal/snapshot/backup_test.go index fa6b9b4..b310fe7 100644 --- a/internal/snapshot/backup_test.go +++ b/internal/snapshot/backup_test.go @@ -4,6 +4,8 @@ import ( "context" "crypto/sha256" "database/sql" + "encoding/hex" + "errors" "fmt" "io" "io/fs" @@ -30,6 +32,7 @@ func NewMockS3Client() *MockS3Client { func (m *MockS3Client) PutBlob(ctx context.Context, hash string, data []byte) error { m.storage[hash] = data + return nil } @@ -38,11 +41,13 @@ func (m *MockS3Client) GetBlob(ctx context.Context, hash string) ([]byte, error) if !ok { return nil, fmt.Errorf("blob not found: %s", hash) } + return data, nil } func (m *MockS3Client) BlobExists(ctx context.Context, hash string) (bool, error) { _, ok := m.storage[hash] + return ok, nil } @@ -81,12 +86,15 @@ func TestBackupWithInMemoryFS(t *testing.T) { // Initialize the database ctx := context.Background() + db, err := database.New(ctx, dbPath) if err != nil { t.Fatalf("Failed to create database: %v", err) } + defer func() { - if err := db.Close(); err != nil { + err := db.Close() + if err != nil { t.Logf("Failed to close database: %v", err) } }() @@ -142,12 +150,14 @@ func TestBackupWithInMemoryFS(t *testing.T) { if !expectedFiles[file.Path.String()] { t.Errorf("Unexpected file in database: %s", file.Path) } + delete(expectedFiles, file.Path.String()) // Verify file metadata fsFile := testFS[file.Path.String()] if fsFile == nil { t.Errorf("File %s not found in test filesystem", file.Path) + continue } @@ -187,6 +197,7 @@ func TestBackupWithInMemoryFS(t *testing.T) { if err != nil { t.Fatalf("Failed to get blob hashes: %v", err) } + if len(blobHashes) == 0 { t.Error("Expected at least one blob to be created") } @@ -197,6 +208,7 @@ func TestBackupWithInMemoryFS(t *testing.T) { if err != nil { t.Errorf("Failed to check blob %s: %v", blobHash, err) } + if !exists { t.Errorf("Blob %s not found in S3", blobHash) } @@ -229,12 +241,15 @@ func TestBackupDeduplication(t *testing.T) { // Initialize the database ctx := context.Background() + db, err := database.New(ctx, dbPath) if err != nil { t.Fatalf("Failed to create database: %v", err) } + defer func() { - if err := db.Close(); err != nil { + err := db.Close() + if err != nil { t.Logf("Failed to close database: %v", err) } }() @@ -348,6 +363,7 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str UID: 1000, // Default UID for test GID: 1000, // Default GID for test } + err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { return b.repos.Files.Create(ctx, tx, file) }) @@ -364,7 +380,8 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str return err } defer func() { - if err := f.Close(); err != nil { + err := f.Close() + if err != nil { // Log but don't fail since we're already in an error path potentially fmt.Fprintf(os.Stderr, "Failed to close file: %v\n", err) } @@ -376,9 +393,10 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str for { n, err := f.Read(buffer) - if err != nil && err != io.EOF { + if err != nil && !errors.Is(err, io.EOF) { return err } + if n == 0 { break } @@ -395,11 +413,13 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str ChunkHash: types.ChunkHash(chunkHash), Size: int64(n), } + return b.repos.Chunks.Create(ctx, tx, chunk) }) if err != nil { return err } + processedChunks[chunkHash] = true } @@ -410,6 +430,7 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str Idx: chunkIndex, ChunkHash: types.ChunkHash(chunkHash), } + return b.repos.FileChunks.Create(ctx, tx, fileChunk) }) if err != nil { @@ -424,6 +445,7 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str FileOffset: int64(chunkIndex * defaultChunkSize), Length: int64(n), } + return b.repos.ChunkFiles.Create(ctx, tx, chunkFile) }) if err != nil { @@ -435,7 +457,6 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str return nil }) - if err != nil { return "", err } @@ -464,12 +485,14 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str // Create blob entry in a short transaction blobID := types.NewBlobID() + err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { blob := &database.Blob{ ID: blobID, Hash: types.BlobHash(blobHash), CreatedTS: time.Now(), } + return b.repos.Blobs.Create(ctx, tx, blob) }) if err != nil { @@ -487,6 +510,7 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str Offset: 0, Length: chunk.Size, } + return b.repos.BlobChunks.Create(ctx, tx, blobChunk) }) if err != nil { @@ -506,7 +530,6 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { return b.repos.Snapshots.UpdateCounts(ctx, tx, snapshotID, fileCount, chunkCount, blobCount, totalSize, blobSize) }) - if err != nil { return "", err } @@ -517,16 +540,18 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str func calculateHash(data []byte) string { h := sha256.New() h.Write(data) - return fmt.Sprintf("%x", h.Sum(nil)) + + return hex.EncodeToString(h.Sum(nil)) } func generateLargeFileContent(size int) []byte { data := make([]byte, size) // Fill with pattern that changes every chunk to avoid deduplication - for i := 0; i < size; i++ { + for i := range size { chunkNum := i / defaultChunkSize data[i] = byte((i + chunkNum) % 256) } + return data } diff --git a/internal/snapshot/exclude_test.go b/internal/snapshot/exclude_test.go index ba2efef..f908250 100644 --- a/internal/snapshot/exclude_test.go +++ b/internal/snapshot/exclude_test.go @@ -63,6 +63,7 @@ func setupExcludeTestFS(t *testing.T) afero.Fs { } testTime := time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC) + for path, content := range files { dir := filepath.Dir(path) err := fs.MkdirAll(dir, 0755) @@ -107,6 +108,7 @@ func createTestScanner(t *testing.T, fs afero.Fs, excludePatterns []string) (*sn func createSnapshotRecord(t *testing.T, ctx context.Context, repos *database.Repositories, snapshotID string) { t.Helper() + err := repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { snap := &database.Snapshot{ ID: types.SnapshotID(snapshotID), @@ -121,6 +123,7 @@ func createSnapshotRecord(t *testing.T, ctx context.Context, repos *database.Rep BlobSize: 0, CompressionRatio: 1.0, } + return repos.Snapshots.Create(ctx, tx, snap) }) require.NoError(t, err) @@ -128,8 +131,10 @@ func createSnapshotRecord(t *testing.T, ctx context.Context, repos *database.Rep func TestExcludePatterns_ExcludeGitDirectory(t *testing.T) { fs := setupExcludeTestFS(t) + scanner, repos, cleanup := createTestScanner(t, fs, []string{".git"}) defer cleanup() + require.NotNil(t, scanner) ctx := context.Background() @@ -148,8 +153,10 @@ func TestExcludePatterns_ExcludeGitDirectory(t *testing.T) { func TestExcludePatterns_ExcludeByExtension(t *testing.T) { fs := setupExcludeTestFS(t) + scanner, repos, cleanup := createTestScanner(t, fs, []string{"*.log"}) defer cleanup() + require.NotNil(t, scanner) ctx := context.Background() @@ -165,8 +172,10 @@ func TestExcludePatterns_ExcludeByExtension(t *testing.T) { func TestExcludePatterns_ExcludeNodeModules(t *testing.T) { fs := setupExcludeTestFS(t) + scanner, repos, cleanup := createTestScanner(t, fs, []string{"node_modules"}) defer cleanup() + require.NotNil(t, scanner) ctx := context.Background() @@ -182,8 +191,10 @@ func TestExcludePatterns_ExcludeNodeModules(t *testing.T) { func TestExcludePatterns_MultiplePatterns(t *testing.T) { fs := setupExcludeTestFS(t) + scanner, repos, cleanup := createTestScanner(t, fs, []string{".git", "node_modules", "*.log", ".DS_Store", "thumbs.db", "cache", "build"}) defer cleanup() + require.NotNil(t, scanner) ctx := context.Background() @@ -199,8 +210,10 @@ func TestExcludePatterns_MultiplePatterns(t *testing.T) { func TestExcludePatterns_NoExclusions(t *testing.T) { fs := setupExcludeTestFS(t) + scanner, repos, cleanup := createTestScanner(t, fs, []string{}) defer cleanup() + require.NotNil(t, scanner) ctx := context.Background() @@ -215,8 +228,10 @@ func TestExcludePatterns_NoExclusions(t *testing.T) { func TestExcludePatterns_ExcludeHiddenFiles(t *testing.T) { fs := setupExcludeTestFS(t) + scanner, repos, cleanup := createTestScanner(t, fs, []string{".*"}) defer cleanup() + require.NotNil(t, scanner) ctx := context.Background() @@ -232,8 +247,10 @@ func TestExcludePatterns_ExcludeHiddenFiles(t *testing.T) { func TestExcludePatterns_DoubleStarGlob(t *testing.T) { fs := setupExcludeTestFS(t) + scanner, repos, cleanup := createTestScanner(t, fs, []string{"**/*.pack"}) defer cleanup() + require.NotNil(t, scanner) ctx := context.Background() @@ -249,8 +266,10 @@ func TestExcludePatterns_DoubleStarGlob(t *testing.T) { func TestExcludePatterns_ExactFileName(t *testing.T) { fs := setupExcludeTestFS(t) + scanner, repos, cleanup := createTestScanner(t, fs, []string{"thumbs.db", ".DS_Store"}) defer cleanup() + require.NotNil(t, scanner) ctx := context.Background() @@ -267,8 +286,10 @@ func TestExcludePatterns_ExactFileName(t *testing.T) { func TestExcludePatterns_CaseSensitive(t *testing.T) { // Pattern matching should be case-sensitive fs := setupExcludeTestFS(t) + scanner, repos, cleanup := createTestScanner(t, fs, []string{"THUMBS.DB"}) defer cleanup() + require.NotNil(t, scanner) ctx := context.Background() @@ -287,6 +308,7 @@ func TestExcludePatterns_DirectoryWithTrailingSlash(t *testing.T) { // Some users might add trailing slashes to directory patterns scanner, repos, cleanup := createTestScanner(t, fs, []string{"cache/", "build/"}) defer cleanup() + require.NotNil(t, scanner) ctx := context.Background() @@ -305,6 +327,7 @@ func TestExcludePatterns_PatternInSubdirectory(t *testing.T) { // Exclude .hidden file specifically in src directory scanner, repos, cleanup := createTestScanner(t, fs, []string{"src/.hidden"}) defer cleanup() + require.NotNil(t, scanner) ctx := context.Background() @@ -343,6 +366,7 @@ func setupAnchoredTestFS(t *testing.T) afero.Fs { } testTime := time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC) + for path, content := range files { dir := filepath.Dir(path) err := fs.MkdirAll(dir, 0755) @@ -359,8 +383,10 @@ func setupAnchoredTestFS(t *testing.T) afero.Fs { func TestExcludePatterns_AnchoredPattern(t *testing.T) { // Pattern starting with / should only match from root of source dir fs := setupAnchoredTestFS(t) + scanner, repos, cleanup := createTestScanner(t, fs, []string{"/projectname"}) defer cleanup() + require.NotNil(t, scanner) ctx := context.Background() @@ -378,8 +404,10 @@ func TestExcludePatterns_AnchoredPattern(t *testing.T) { func TestExcludePatterns_UnanchoredPattern(t *testing.T) { // Pattern without leading / should match anywhere in path fs := setupAnchoredTestFS(t) + scanner, repos, cleanup := createTestScanner(t, fs, []string{"projectname"}) defer cleanup() + require.NotNil(t, scanner) ctx := context.Background() @@ -398,8 +426,10 @@ func TestExcludePatterns_UnanchoredPattern(t *testing.T) { func TestExcludePatterns_AnchoredPatternWithGlob(t *testing.T) { // Anchored pattern with glob fs := setupAnchoredTestFS(t) + scanner, repos, cleanup := createTestScanner(t, fs, []string{"/src/*.go"}) defer cleanup() + require.NotNil(t, scanner) ctx := context.Background() @@ -416,8 +446,10 @@ func TestExcludePatterns_AnchoredPatternWithGlob(t *testing.T) { func TestExcludePatterns_AnchoredPatternFile(t *testing.T) { // Anchored pattern for exact file at root fs := setupAnchoredTestFS(t) + scanner, repos, cleanup := createTestScanner(t, fs, []string{"/file.txt"}) defer cleanup() + require.NotNil(t, scanner) ctx := context.Background() @@ -435,8 +467,10 @@ func TestExcludePatterns_AnchoredPatternFile(t *testing.T) { func TestExcludePatterns_UnanchoredPatternFile(t *testing.T) { // Unanchored pattern for file should match anywhere fs := setupAnchoredTestFS(t) + scanner, repos, cleanup := createTestScanner(t, fs, []string{"file.txt"}) defer cleanup() + require.NotNil(t, scanner) ctx := context.Background() diff --git a/internal/snapshot/file_change_test.go b/internal/snapshot/file_change_test.go index 033934b..a918010 100644 --- a/internal/snapshot/file_change_test.go +++ b/internal/snapshot/file_change_test.go @@ -30,9 +30,11 @@ func TestFileContentChange(t *testing.T) { // Create test database db, err := database.NewTestDB() + require.NoError(t, err) defer func() { - if err := db.Close(); err != nil { + err := db.Close() + if err != nil { t.Errorf("failed to close database: %v", err) } }() @@ -59,6 +61,7 @@ func TestFileContentChange(t *testing.T) { VaultikVersion: "test", StartedAt: time.Now(), } + return repos.Snapshots.Create(ctx, tx, snapshot) }) require.NoError(t, err) @@ -81,6 +84,7 @@ func TestFileContentChange(t *testing.T) { // Modify the file time.Sleep(10 * time.Millisecond) // Ensure mtime changes + err = afero.WriteFile(fs, "/test.txt", []byte("Modified content with different data"), 0644) require.NoError(t, err) @@ -93,6 +97,7 @@ func TestFileContentChange(t *testing.T) { VaultikVersion: "test", StartedAt: time.Now(), } + return repos.Snapshots.Create(ctx, tx, snapshot) }) require.NoError(t, err) @@ -130,6 +135,7 @@ func TestFileContentChange(t *testing.T) { // Verify that chunk_files for old chunk no longer references this file oldChunkFiles, err := repos.ChunkFiles.GetByChunkHash(ctx, oldChunkHash) require.NoError(t, err) + for _, cf := range oldChunkFiles { file, err := repos.Files.GetByID(ctx, cf.FileID) require.NoError(t, err) @@ -159,9 +165,11 @@ func TestMultipleFileChanges(t *testing.T) { // Create test database db, err := database.NewTestDB() + require.NoError(t, err) defer func() { - if err := db.Close(); err != nil { + err := db.Close() + if err != nil { t.Errorf("failed to close database: %v", err) } }() @@ -188,6 +196,7 @@ func TestMultipleFileChanges(t *testing.T) { VaultikVersion: "test", StartedAt: time.Now(), } + return repos.Snapshots.Create(ctx, tx, snapshot) }) require.NoError(t, err) @@ -200,6 +209,7 @@ func TestMultipleFileChanges(t *testing.T) { // Modify two files time.Sleep(10 * time.Millisecond) // Ensure mtime changes + err = afero.WriteFile(fs, "/file1.txt", []byte("Modified content 1"), 0644) require.NoError(t, err) err = afero.WriteFile(fs, "/file3.txt", []byte("Modified content 3"), 0644) @@ -214,6 +224,7 @@ func TestMultipleFileChanges(t *testing.T) { VaultikVersion: "test", StartedAt: time.Now(), } + return repos.Snapshots.Create(ctx, tx, snapshot) }) require.NoError(t, err) diff --git a/internal/snapshot/manifest.go b/internal/snapshot/manifest.go index 3b152cc..f6dc3ae 100644 --- a/internal/snapshot/manifest.go +++ b/internal/snapshot/manifest.go @@ -52,6 +52,7 @@ func EncodeManifest(manifest *Manifest, compressionLevel int) ([]byte, error) { // Compress using zstd var compressedBuf bytes.Buffer + writer, err := zstd.NewWriter(&compressedBuf, zstd.WithEncoderLevel(zstd.EncoderLevelFromZstd(compressionLevel))) if err != nil { return nil, fmt.Errorf("creating zstd writer: %w", err) @@ -59,6 +60,7 @@ func EncodeManifest(manifest *Manifest, compressionLevel int) ([]byte, error) { if _, err := writer.Write(jsonData); err != nil { _ = writer.Close() + return nil, fmt.Errorf("writing compressed data: %w", err) } diff --git a/internal/snapshot/permission_error_test.go b/internal/snapshot/permission_error_test.go index bbde682..d339d3d 100644 --- a/internal/snapshot/permission_error_test.go +++ b/internal/snapshot/permission_error_test.go @@ -12,7 +12,9 @@ import ( func TestWrapPermissionError(t *testing.T) { // Non-permission errors pass through unchanged. plain := errors.New("disk on fire") - if got := wrapPermissionError("/some/path", plain); got != plain { + + got := wrapPermissionError("/some/path", plain) + if !errors.Is(got, plain) { t.Errorf("non-permission error should pass through, got %v", got) } @@ -23,6 +25,7 @@ func TestWrapPermissionError(t *testing.T) { if !errors.Is(wrapped, os.ErrPermission) { t.Error("wrapped error should still match os.ErrPermission") } + if !strings.Contains(wrapped.Error(), "/Users/u/Library/Calendars") { t.Error("wrapped error should name the offending path") } @@ -31,6 +34,7 @@ func TestWrapPermissionError(t *testing.T) { if !strings.Contains(wrapped.Error(), "Full Disk Access") { t.Errorf("macOS permission error should mention Full Disk Access:\n%s", wrapped.Error()) } + if !strings.Contains(wrapped.Error(), "System Settings") { t.Errorf("macOS permission error should point at System Settings:\n%s", wrapped.Error()) } diff --git a/internal/snapshot/progress.go b/internal/snapshot/progress.go index 568efd2..8795e92 100644 --- a/internal/snapshot/progress.go +++ b/internal/snapshot/progress.go @@ -153,6 +153,7 @@ func (pr *ProgressReporter) printSummaryStatus() { if uploadInfo, ok := pr.stats.CurrentUpload.Load().(*UploadInfo); ok && uploadInfo != nil { // Show upload progress instead pr.printUploadProgress(uploadInfo) + return } @@ -175,16 +176,18 @@ func (pr *ProgressReporter) printSummaryStatus() { // Calculate ETA if we have total size and are processing etaStr := "" + if totalSize > 0 && bytesProcessed > 0 { processStart, ok := pr.stats.ProcessStartTime.Load().(time.Time) if ok && !processStart.IsZero() { processElapsed := time.Since(processStart) + rate := float64(bytesProcessed) / processElapsed.Seconds() if rate > 0 { remainingBytes := totalSize - bytesProcessed remainingSeconds := float64(remainingBytes) / rate eta := time.Duration(remainingSeconds * float64(time.Second)) - etaStr = fmt.Sprintf(" | ETA: %s", formatDuration(eta)) + etaStr = " | ETA: " + formatDuration(eta) } } } @@ -206,7 +209,7 @@ func (pr *ProgressReporter) printSummaryStatus() { ) if currentFile != "" { - status += fmt.Sprintf(" | Current: %s", truncatePath(currentFile, 40)) + status += " | Current: " + truncatePath(currentFile, 40) } log.Info(status) @@ -242,6 +245,7 @@ func (pr *ProgressReporter) printDetailedStatus() { processStart, ok := pr.stats.ProcessStartTime.Load().(time.Time) if ok && !processStart.IsZero() { processElapsed := time.Since(processStart) + processRate := float64(bytesProcessed) / processElapsed.Seconds() if processRate > 0 { remainingBytes := totalSize - bytesProcessed @@ -276,9 +280,11 @@ func (pr *ProgressReporter) printDetailedStatus() { log.Info("Total uploaded to remote", "uploaded", humanize.Bytes(uint64(bytesUploaded)), "compression_ratio", formatRatio(bytesUploaded, bytesScanned)) + if currentFile != "" { log.Info("Current file", "path", currentFile) } + log.Notice("=============================") } @@ -288,12 +294,15 @@ func formatDuration(d time.Duration) string { if d < 0 { return "unknown" } + if d < time.Minute { return fmt.Sprintf("%ds", int(d.Seconds())) } + if d < time.Hour { return fmt.Sprintf("%dm%ds", int(d.Minutes()), int(d.Seconds())%60) } + return fmt.Sprintf("%dh%dm", int(d.Hours()), int(d.Minutes())%60) } @@ -301,6 +310,7 @@ func formatPercent(numerator, denominator int64) string { if denominator == 0 { return "0.0%" } + return fmt.Sprintf("%.1f%%", float64(numerator)/float64(denominator)*100) } @@ -308,7 +318,9 @@ func formatRatio(compressed, uncompressed int64) string { if uncompressed == 0 { return "1.00" } + ratio := float64(compressed) / float64(uncompressed) + return fmt.Sprintf("%.2f", ratio) } @@ -353,6 +365,7 @@ func (pr *ProgressReporter) ReportUploadComplete(blobHash string, size int64, du if duration < time.Millisecond { duration = time.Millisecond } + bytesPerSec := float64(size) / duration.Seconds() bitsPerSec := bytesPerSec * 8 @@ -398,6 +411,7 @@ func (pr *ProgressReporter) ReportUploadProgress(blobHash string, bytesUploaded, // Calculate ETA based on current speed etaStr := "unknown" + if instantSpeed > 0 && bytesUploaded < totalSize { remainingBytes := totalSize - bytesUploaded remainingSeconds := float64(remainingBytes) / instantSpeed diff --git a/internal/snapshot/remotekey.go b/internal/snapshot/remotekey.go index 031b9ce..7338f07 100644 --- a/internal/snapshot/remotekey.go +++ b/internal/snapshot/remotekey.go @@ -36,5 +36,6 @@ const remoteKeyPrefix = "vaultik|" func RemoteSnapshotKey(snapshotID string) string { first := sha256.Sum256([]byte(remoteKeyPrefix + snapshotID)) second := sha256.Sum256(first[:]) + return hex.EncodeToString(second[:]) } diff --git a/internal/snapshot/scanner.go b/internal/snapshot/scanner.go index c6b2442..06452e6 100644 --- a/internal/snapshot/scanner.go +++ b/internal/snapshot/scanner.go @@ -119,6 +119,7 @@ func NewScanner(cfg ScannerConfig) *Scanner { // Create encryptor (required for blob packing) if len(cfg.AgeRecipients) == 0 { log.Error("No age recipients configured - encryption is required") + return nil } @@ -130,9 +131,11 @@ func NewScanner(cfg ScannerConfig) *Scanner { Repositories: cfg.Repositories, Fs: cfg.FS, } + packer, err := blob.NewPacker(packerCfg) if err != nil { log.Error("Failed to create packer", "error", err) + return nil } @@ -199,11 +202,14 @@ func (s *Scanner) Scan(ctx context.Context, path string, snapshotID string) (*Sc // Phase 1: Scan directory, collect files to process, and track existing files // (builds existingFiles map during walk to avoid double traversal) log.Info("Phase 1/3: Scanning directory structure") + existingFiles := make(map[string]struct{}) + scanResult, err := s.scanPhase(ctx, path, result, existingFiles, knownFiles) if err != nil { return nil, fmt.Errorf("scan phase failed: %w", err) } + filesToProcess := scanResult.FilesToProcess // Phase 1b: Detect deleted files by comparing DB against scanned files @@ -213,8 +219,10 @@ func (s *Scanner) Scan(ctx context.Context, path string, snapshotID string) (*Sc // Phase 1c: Associate unchanged files with this snapshot (no new records needed) if len(scanResult.UnchangedFileIDs) > 0 { - s.ui.Begin("Associating %s unchanged files with the snapshot.", s.ui.Count(len(scanResult.UnchangedFileIDs))) - if err := s.batchAddFilesToSnapshot(ctx, scanResult.UnchangedFileIDs); err != nil { + s.ui.Beginf("Associating %s unchanged files with the snapshot.", s.ui.Count(len(scanResult.UnchangedFileIDs))) + + err := s.batchAddFilesToSnapshot(ctx, scanResult.UnchangedFileIDs) + if err != nil { return nil, fmt.Errorf("associating unchanged files: %w", err) } } @@ -224,13 +232,15 @@ func (s *Scanner) Scan(ctx context.Context, path string, snapshotID string) (*Sc // Phase 2: Process files and create chunks if len(filesToProcess) > 0 { - s.ui.Begin("Backing up %s snapshot source files (chunking, compressing, encrypting, uploading).", s.ui.Count(len(filesToProcess))) + s.ui.Beginf("Backing up %s snapshot source files (chunking, compressing, encrypting, uploading).", s.ui.Count(len(filesToProcess))) log.Info("Phase 2/3: Creating snapshot (chunking, compressing, encrypting, and uploading blobs)") - if err := s.processPhase(ctx, filesToProcess, result); err != nil { + + err := s.processPhase(ctx, filesToProcess, result) + if err != nil { return nil, fmt.Errorf("process phase failed: %w", err) } } else { - s.ui.Info("Snapshot file backup skipped: no changed files (creating metadata-only snapshot).") + s.ui.Infof("Snapshot file backup skipped: no changed files (creating metadata-only snapshot).") log.Info("Phase 2/3: Skipping (no files need processing, metadata-only snapshot)") } @@ -243,18 +253,22 @@ func (s *Scanner) Scan(ctx context.Context, path string, snapshotID string) (*Sc // loadDatabaseState loads known files and chunks from the database into memory for fast lookup // This avoids per-file and per-chunk database queries during the scan and process phases func (s *Scanner) loadDatabaseState(ctx context.Context, path string) (map[string]*database.File, error) { - s.ui.Begin("Loading known files from local index database.") + s.ui.Beginf("Loading known files from local index database.") + knownFiles, err := s.loadKnownFiles(ctx, path) if err != nil { return nil, fmt.Errorf("loading known files: %w", err) } - s.ui.Complete("Loaded %s known files from local index database.", s.ui.Count(len(knownFiles))) - s.ui.Begin("Loading known chunks from local index database.") + s.ui.Completef("Loaded %s known files from local index database.", s.ui.Count(len(knownFiles))) + + s.ui.Beginf("Loading known chunks from local index database.") + if err := s.loadKnownChunks(ctx); err != nil { return nil, fmt.Errorf("loading known chunks: %w", err) } - s.ui.Complete("Loaded %s known chunks from local index database.", s.ui.Count(len(s.knownChunks))) + + s.ui.Completef("Loaded %s known chunks from local index database.", s.ui.Count(len(s.knownChunks))) return knownFiles, nil } @@ -288,7 +302,8 @@ func (s *Scanner) summarizeScanPhase(result *ScanResult, filesToProcess []*FileT s.ui.Count(result.FilesDeleted), s.ui.Size(result.BytesDeleted)) } - s.ui.Complete("%s.", msg) + + s.ui.Completef("%s.", msg) } // finalizeScanResult populates final blob statistics in the scan result @@ -337,6 +352,7 @@ func (s *Scanner) loadKnownChunks(ctx context.Context) error { } s.knownChunksMu.Lock() + s.knownChunks = make(map[string]struct{}, len(chunks)) for _, c := range chunks { s.knownChunks[c.ChunkHash.String()] = struct{}{} @@ -351,6 +367,7 @@ func (s *Scanner) chunkExists(hash string) bool { s.knownChunksMu.RLock() _, exists := s.knownChunks[hash] s.knownChunksMu.RUnlock() + return exists } @@ -371,7 +388,9 @@ func (s *Scanner) addPendingChunkHash(hash string) { // removePendingChunkHashes removes committed chunk hashes from the pending set func (s *Scanner) removePendingChunkHashes(hashes []string) { log.Debug("removePendingChunkHashes: starting", "count", len(hashes)) + start := time.Now() + s.pendingChunkHashesMu.Lock() for _, hash := range hashes { delete(s.pendingChunkHashes, hash) @@ -385,6 +404,7 @@ func (s *Scanner) isChunkPending(hash string) bool { s.pendingChunkHashesMu.Lock() _, pending := s.pendingChunkHashes[hash] s.pendingChunkHashesMu.Unlock() + return pending } @@ -411,37 +431,45 @@ func (s *Scanner) flushPendingFiles(ctx context.Context) error { return s.repos.WithTx(ctx, func(txCtx context.Context, tx *sql.Tx) error { for _, data := range files { // Create or update the file record - if err := s.repos.Files.Create(txCtx, tx, data.file); err != nil { + err := s.repos.Files.Create(txCtx, tx, data.file) + if err != nil { return fmt.Errorf("creating file record: %w", err) } // Delete any existing file_chunks and chunk_files for this file - if err := s.repos.FileChunks.DeleteByFileID(txCtx, tx, data.file.ID); err != nil { + err = s.repos.FileChunks.DeleteByFileID(txCtx, tx, data.file.ID) + if err != nil { return fmt.Errorf("deleting old file chunks: %w", err) } - if err := s.repos.ChunkFiles.DeleteByFileID(txCtx, tx, data.file.ID); err != nil { + + err = s.repos.ChunkFiles.DeleteByFileID(txCtx, tx, data.file.ID) + if err != nil { return fmt.Errorf("deleting old chunk files: %w", err) } // Create file-chunk mappings for i := range data.fileChunks { - if err := s.repos.FileChunks.Create(txCtx, tx, &data.fileChunks[i]); err != nil { + err := s.repos.FileChunks.Create(txCtx, tx, &data.fileChunks[i]) + if err != nil { return fmt.Errorf("creating file chunk: %w", err) } } // Create chunk-file mappings for i := range data.chunkFiles { - if err := s.repos.ChunkFiles.Create(txCtx, tx, &data.chunkFiles[i]); err != nil { + err := s.repos.ChunkFiles.Create(txCtx, tx, &data.chunkFiles[i]) + if err != nil { return fmt.Errorf("creating chunk file: %w", err) } } // Add file to snapshot - if err := s.repos.Snapshots.AddFileByID(txCtx, tx, s.snapshotID, data.file.ID); err != nil { + err = s.repos.Snapshots.AddFileByID(txCtx, tx, s.snapshotID, data.file.ID) + if err != nil { return fmt.Errorf("adding file to snapshot: %w", err) } } + return nil }) } @@ -455,6 +483,7 @@ func (s *Scanner) flushAllPending(ctx context.Context) error { // Files with pending chunks are kept in the queue for later flushing func (s *Scanner) flushCompletedPendingFiles(ctx context.Context) error { flushStart := time.Now() + log.Debug("flushCompletedPendingFiles: starting") // Partition pending files into those ready to flush and those still waiting @@ -462,6 +491,7 @@ func (s *Scanner) flushCompletedPendingFiles(ctx context.Context) error { if len(canFlush) == 0 { log.Debug("flushCompletedPendingFiles: nothing to flush") + return nil } @@ -474,10 +504,13 @@ func (s *Scanner) flushCompletedPendingFiles(ctx context.Context) error { // Execute the batch flush in a single transaction log.Debug("flushCompletedPendingFiles: starting transaction") + txStart := time.Now() err := s.executeBatchFileFlush(ctx, allFiles, allFileIDs, allFileChunks, allChunkFiles) + log.Debug("flushCompletedPendingFiles: transaction done", "duration", time.Since(txStart)) log.Debug("flushCompletedPendingFiles: total duration", "duration", time.Since(flushStart)) + return err } @@ -492,21 +525,27 @@ func (s *Scanner) partitionPendingByChunkStatus() (canFlush []pendingFileData, s var stillPending []pendingFileData log.Debug("flushCompletedPendingFiles: checking which files can flush") + checkStart := time.Now() + for _, data := range s.pendingFiles { allChunksCommitted := true + for _, fc := range data.fileChunks { if s.isChunkPending(fc.ChunkHash.String()) { allChunksCommitted = false + break } } + if allChunksCommitted { canFlush = append(canFlush, data) } else { stillPending = append(stillPending, data) } } + log.Debug("flushCompletedPendingFiles: check done", "duration", time.Since(checkStart), "can_flush", len(canFlush), "still_pending", len(stillPending)) s.pendingFiles = stillPending @@ -520,12 +559,15 @@ func (s *Scanner) partitionPendingByChunkStatus() (canFlush []pendingFileData, s // mappings from the given pending file data for efficient batch database operations func (s *Scanner) collectBatchFlushData(canFlush []pendingFileData) ([]*database.File, []types.FileID, []database.FileChunk, []database.ChunkFile) { log.Debug("flushCompletedPendingFiles: collecting data for batch ops") + collectStart := time.Now() - var allFileChunks []database.FileChunk - var allChunkFiles []database.ChunkFile - var allFileIDs []types.FileID - var allFiles []*database.File + var ( + allFileChunks []database.FileChunk + allChunkFiles []database.ChunkFile + allFileIDs []types.FileID + allFiles []*database.File + ) for _, data := range canFlush { allFileChunks = append(allFileChunks, data.fileChunks...) @@ -551,52 +593,77 @@ func (s *Scanner) executeBatchFileFlush(ctx context.Context, allFiles []*databas // Batch delete old file_chunks and chunk_files log.Debug("flushCompletedPendingFiles: deleting old file_chunks") + opStart := time.Now() - if err := s.repos.FileChunks.DeleteByFileIDs(txCtx, tx, allFileIDs); err != nil { + + err := s.repos.FileChunks.DeleteByFileIDs(txCtx, tx, allFileIDs) + if err != nil { return fmt.Errorf("batch deleting old file chunks: %w", err) } + log.Debug("flushCompletedPendingFiles: deleted file_chunks", "duration", time.Since(opStart)) log.Debug("flushCompletedPendingFiles: deleting old chunk_files") + opStart = time.Now() - if err := s.repos.ChunkFiles.DeleteByFileIDs(txCtx, tx, allFileIDs); err != nil { + + err = s.repos.ChunkFiles.DeleteByFileIDs(txCtx, tx, allFileIDs) + if err != nil { return fmt.Errorf("batch deleting old chunk files: %w", err) } + log.Debug("flushCompletedPendingFiles: deleted chunk_files", "duration", time.Since(opStart)) // Batch create/update file records log.Debug("flushCompletedPendingFiles: creating files") + opStart = time.Now() - if err := s.repos.Files.CreateBatch(txCtx, tx, allFiles); err != nil { + + err = s.repos.Files.CreateBatch(txCtx, tx, allFiles) + if err != nil { return fmt.Errorf("batch creating file records: %w", err) } + log.Debug("flushCompletedPendingFiles: created files", "duration", time.Since(opStart)) // Batch insert file_chunks log.Debug("flushCompletedPendingFiles: inserting file_chunks") + opStart = time.Now() - if err := s.repos.FileChunks.CreateBatch(txCtx, tx, allFileChunks); err != nil { + + err = s.repos.FileChunks.CreateBatch(txCtx, tx, allFileChunks) + if err != nil { return fmt.Errorf("batch creating file chunks: %w", err) } + log.Debug("flushCompletedPendingFiles: inserted file_chunks", "duration", time.Since(opStart)) // Batch insert chunk_files log.Debug("flushCompletedPendingFiles: inserting chunk_files") + opStart = time.Now() - if err := s.repos.ChunkFiles.CreateBatch(txCtx, tx, allChunkFiles); err != nil { + + err = s.repos.ChunkFiles.CreateBatch(txCtx, tx, allChunkFiles) + if err != nil { return fmt.Errorf("batch creating chunk files: %w", err) } + log.Debug("flushCompletedPendingFiles: inserted chunk_files", "duration", time.Since(opStart)) // Batch add files to snapshot log.Debug("flushCompletedPendingFiles: adding files to snapshot") + opStart = time.Now() - if err := s.repos.Snapshots.AddFilesByIDBatch(txCtx, tx, s.snapshotID, allFileIDs); err != nil { + + err = s.repos.Snapshots.AddFilesByIDBatch(txCtx, tx, s.snapshotID, allFileIDs) + if err != nil { return fmt.Errorf("batch adding files to snapshot: %w", err) } + log.Debug("flushCompletedPendingFiles: added files to snapshot", "duration", time.Since(opStart)) log.Debug("flushCompletedPendingFiles: transaction complete") + return nil }) } @@ -616,24 +683,31 @@ func (s *Scanner) scanPhase(ctx context.Context, path string, result *ScanResult estimatedTotal := int64(len(knownFiles)) var filesToProcess []*FileToProcess + var unchangedFileIDs []types.FileID // Just IDs - no new records needed + var mu sync.Mutex // Set up periodic status output startTime := time.Now() lastStatusTime := time.Now() statusInterval := 15 * time.Second + var filesScanned int64 log.Debug("Starting directory walk", "path", path) + err := afero.Walk(s.fs, path, func(filePath string, info os.FileInfo, err error) error { if err != nil { if s.skipErrors { log.Error("Failed to access file (skipping due to --skip-errors)", "path", filePath, "error", err) - s.ui.Error("Failed to access %s: %v. Skipping (--skip-errors).", s.ui.Path(filePath), err) + s.ui.Errorf("Failed to access %s: %v. Skipping (--skip-errors).", s.ui.Path(filePath), err) + return nil // Continue scanning } + log.Debug("Error accessing filesystem entry", "path", filePath, "error", err) + return wrapPermissionError(filePath, err) } @@ -649,6 +723,7 @@ func (s *Scanner) scanPhase(ctx context.Context, path string, result *ScanResult if info.IsDir() { return filepath.SkipDir } + return nil } @@ -657,7 +732,9 @@ func (s *Scanner) scanPhase(ctx context.Context, path string, result *ScanResult file := s.buildSymlinkEntry(filePath, info) if file != nil { existingFiles[filePath] = struct{}{} + mu.Lock() + filesToProcess = append(filesToProcess, &FileToProcess{ Path: filePath, FileInfo: info, @@ -667,6 +744,7 @@ func (s *Scanner) scanPhase(ctx context.Context, path string, result *ScanResult mu.Unlock() s.updateScanEntryStats(result, true, info) } + return nil } @@ -674,7 +752,9 @@ func (s *Scanner) scanPhase(ctx context.Context, path string, result *ScanResult if info.IsDir() { file := s.buildDirectoryEntry(filePath, info) existingFiles[filePath] = struct{}{} + mu.Lock() + filesToProcess = append(filesToProcess, &FileToProcess{ Path: filePath, FileInfo: info, @@ -682,6 +762,7 @@ func (s *Scanner) scanPhase(ctx context.Context, path string, result *ScanResult }) filesScanned++ mu.Unlock() + return nil } @@ -708,6 +789,7 @@ func (s *Scanner) scanPhase(ctx context.Context, path string, result *ScanResult // Unchanged file with existing ID - just need snapshot association unchangedFileIDs = append(unchangedFileIDs, file.ID) } + filesScanned++ changedCount := len(filesToProcess) mu.Unlock() @@ -718,12 +800,12 @@ func (s *Scanner) scanPhase(ctx context.Context, path string, result *ScanResult // Output periodic status if time.Since(lastStatusTime) >= statusInterval { s.printScanProgressLine(filesScanned, changedCount, estimatedTotal, startTime) + lastStatusTime = time.Now() } return nil }) - if err != nil { return nil, err } @@ -745,11 +827,13 @@ func (s *Scanner) updateScanEntryStats(result *ScanResult, needsProcessing bool, } else { result.FilesSkipped++ result.BytesSkipped += info.Size() + if s.progress != nil { s.progress.GetStats().FilesSkipped.Add(1) s.progress.GetStats().BytesSkipped.Add(info.Size()) } } + result.FilesScanned++ if s.progress != nil { s.progress.GetStats().FilesScanned.Add(1) @@ -768,16 +852,16 @@ func (s *Scanner) printScanProgressLine(filesScanned int64, changedCount int, es if pct > 100 { pct = 100 // Cap at 100% for display } - remaining := estimatedTotal - filesScanned - if remaining < 0 { - remaining = 0 - } + + remaining := max(estimatedTotal-filesScanned, 0) + var eta time.Duration if rate > 0 && remaining > 0 { eta = time.Duration(float64(remaining)/rate) * time.Second } + if eta > 0 { - s.ui.Progress("Snapshot source files enumeration: %s files (~%s), %s changed or new, %.0f files/sec, enumeration elapsed: %s, enumeration ETA: %s (est remain %s).", + s.ui.Progressf("Snapshot source files enumeration: %s files (~%s), %s changed or new, %.0f files/sec, enumeration elapsed: %s, enumeration ETA: %s (est remain %s).", s.ui.Count(int(filesScanned)), s.ui.Percent(pct), s.ui.Count(changedCount), @@ -786,7 +870,7 @@ func (s *Scanner) printScanProgressLine(filesScanned int64, changedCount int, es s.ui.Time(time.Now().Add(eta)), s.ui.Duration(eta)) } else { - s.ui.Progress("Snapshot source files enumeration: %s files (~%s), %s changed or new, %.0f files/sec, enumeration elapsed: %s.", + s.ui.Progressf("Snapshot source files enumeration: %s files (~%s), %s changed or new, %.0f files/sec, enumeration elapsed: %s.", s.ui.Count(int(filesScanned)), s.ui.Percent(pct), s.ui.Count(changedCount), @@ -794,7 +878,7 @@ func (s *Scanner) printScanProgressLine(filesScanned int64, changedCount int, es s.ui.Duration(elapsed)) } } else { - s.ui.Progress("Snapshot source files enumeration: %s files seen, %s changed or new, %.0f files/sec, enumeration elapsed: %s.", + s.ui.Progressf("Snapshot source files enumeration: %s files seen, %s changed or new, %.0f files/sec, enumeration elapsed: %s.", s.ui.Count(int(filesScanned)), s.ui.Count(changedCount), rate, @@ -808,6 +892,7 @@ func (s *Scanner) buildSymlinkEntry(path string, info os.FileInfo) *database.Fil target, err := os.Readlink(path) if err != nil { log.Debug("Cannot read symlink target", "path", path, "error", err) + return nil } @@ -860,9 +945,11 @@ func (s *Scanner) buildDirectoryEntry(path string, info os.FileInfo) *database.F // and associates it with the current snapshot. No chunking is performed. func (s *Scanner) recordNonRegularFile(ctx context.Context, ftp *FileToProcess) error { return s.repos.WithTx(ctx, func(txCtx context.Context, tx *sql.Tx) error { - if err := s.repos.Files.Create(txCtx, tx, ftp.File); err != nil { + err := s.repos.Files.Create(txCtx, tx, ftp.File) + if err != nil { return fmt.Errorf("creating non-regular file record: %w", err) } + return s.repos.Snapshots.AddFileByID(txCtx, tx, s.snapshotID, ftp.File.ID) }) } @@ -941,18 +1028,18 @@ func (s *Scanner) batchAddFilesToSnapshot(ctx context.Context, fileIDs []types.F default: } - end := i + batchSize - if end > len(fileIDs) { - end = len(fileIDs) - } + end := min(i+batchSize, len(fileIDs)) + batch := fileIDs[i:end] err := s.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { for _, fileID := range batch { - if err := s.repos.Snapshots.AddFileByID(ctx, tx, s.snapshotID, fileID); err != nil { + err := s.repos.Snapshots.AddFileByID(ctx, tx, s.snapshotID, fileID) + if err != nil { return fmt.Errorf("adding file to snapshot: %w", err) } } + return nil }) if err != nil { @@ -964,15 +1051,16 @@ func (s *Scanner) batchAddFilesToSnapshot(ctx context.Context, fileIDs []types.F elapsed := time.Since(startTime) rate := float64(end) / elapsed.Seconds() pct := float64(end) / float64(len(fileIDs)) * 100 - s.ui.Progress("Snapshot unchanged-file association: %s/%s (%s), %.0f files/sec.", + s.ui.Progressf("Snapshot unchanged-file association: %s/%s (%s), %.0f files/sec.", s.ui.Count(end), s.ui.Count(len(fileIDs)), s.ui.Percent(pct), rate) + lastStatusTime = time.Now() } } elapsed := time.Since(startTime) rate := float64(len(fileIDs)) / elapsed.Seconds() - s.ui.Complete("Associated %s unchanged files with the snapshot in %s (%.0f files/sec).", + s.ui.Completef("Associated %s unchanged files with the snapshot in %s (%.0f files/sec).", s.ui.Count(len(fileIDs)), s.ui.Duration(elapsed), rate) return nil @@ -991,7 +1079,9 @@ func (s *Scanner) processPhase(ctx context.Context, filesToProcess []*FileToProc statusInterval := 15 * time.Second startTime := time.Now() filesProcessed := 0 + var bytesProcessed int64 + totalFiles := len(filesToProcess) // Process each file @@ -1006,6 +1096,7 @@ func (s *Scanner) processPhase(ctx context.Context, filesToProcess []*FileToProc if err != nil { return err } + if skipped { continue } @@ -1021,6 +1112,7 @@ func (s *Scanner) processPhase(ctx context.Context, filesToProcess []*FileToProc // Output periodic status if time.Since(lastStatusTime) >= statusInterval { s.printProcessingProgress(filesProcessed, totalFiles, bytesProcessed, totalBytes, startTime) + lastStatusTime = time.Now() } } @@ -1032,22 +1124,29 @@ func (s *Scanner) processPhase(ctx context.Context, filesToProcess []*FileToProc // processFileWithErrorHandling wraps processFileStreaming with error recovery for // deleted files and skip-errors mode. Returns (skipped, error). func (s *Scanner) processFileWithErrorHandling(ctx context.Context, fileToProcess *FileToProcess, result *ScanResult) (bool, error) { - if err := s.processFileStreaming(ctx, fileToProcess, result); err != nil { + err := s.processFileStreaming(ctx, fileToProcess, result) + if err != nil { // Handle files that were deleted between scan and process phases if errors.Is(err, os.ErrNotExist) { log.Warn("File was deleted during backup, skipping", "path", fileToProcess.Path) + result.FilesSkipped++ + return true, nil } // Skip file read errors if --skip-errors is enabled if s.skipErrors { log.Error("Failed to process file (skipping due to --skip-errors)", "path", fileToProcess.Path, "error", err) - s.ui.Error("Failed to process %s: %v. Skipping (--skip-errors).", s.ui.Path(fileToProcess.Path), err) + s.ui.Errorf("Failed to process %s: %v. Skipping (--skip-errors).", s.ui.Path(fileToProcess.Path), err) + result.FilesSkipped++ + return true, nil } + return false, fmt.Errorf("processing file %s: %w", fileToProcess.Path, err) } + return false, nil } @@ -1061,13 +1160,14 @@ func (s *Scanner) printProcessingProgress(filesProcessed, totalFiles int, bytesP // Calculate ETA based on bytes (more accurate than files) remainingBytes := totalBytes - bytesProcessed + var eta time.Duration if byteRate > 0 { eta = time.Duration(float64(remainingBytes)/byteRate) * time.Second } if eta > 0 { - s.ui.Progress("Snapshot backup: %s/%s files (%s), %s/%s, %s, %.0f files/sec, backup elapsed: %s, backup ETA: %s (est remain %s).", + s.ui.Progressf("Snapshot backup: %s/%s files (%s), %s/%s, %s, %.0f files/sec, backup elapsed: %s, backup ETA: %s (est remain %s).", s.ui.Count(filesProcessed), s.ui.Count(totalFiles), s.ui.Percent(pct), @@ -1079,7 +1179,7 @@ func (s *Scanner) printProcessingProgress(filesProcessed, totalFiles int, bytesP s.ui.Time(time.Now().Add(eta)), s.ui.Duration(eta)) } else { - s.ui.Progress("Snapshot backup: %s/%s files (%s), %s/%s, %s, %.0f files/sec, backup elapsed: %s.", + s.ui.Progressf("Snapshot backup: %s/%s files (%s), %s/%s, %s, %.0f files/sec, backup elapsed: %s.", s.ui.Count(filesProcessed), s.ui.Count(totalFiles), s.ui.Percent(pct), @@ -1097,15 +1197,19 @@ func (s *Scanner) finalizeProcessPhase(ctx context.Context, result *ScanResult) // Final packer flush first - this commits remaining chunks to DB // and handleBlobReady will flush files whose chunks are now committed s.packerMu.Lock() - if err := s.packer.Flush(); err != nil { + + err := s.packer.Flush() + if err != nil { s.packerMu.Unlock() + return fmt.Errorf("flushing packer: %w", err) } s.packerMu.Unlock() // Flush any remaining pending files (e.g., files with only pre-existing chunks // that didn't trigger a blob finalize) - if err := s.flushAllPending(ctx); err != nil { + err = s.flushAllPending(ctx) + if err != nil { return fmt.Errorf("flushing remaining pending files: %w", err) } @@ -1119,6 +1223,7 @@ func (s *Scanner) finalizeProcessPhase(ctx context.Context, result *ScanResult) if err != nil { return fmt.Errorf("parsing blob ID: %w", err) } + err = s.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { return s.repos.Snapshots.AddBlob(ctx, tx, s.snapshotID, blobID, types.BlobHash(b.Hash)) }) @@ -1126,6 +1231,7 @@ func (s *Scanner) finalizeProcessPhase(ctx context.Context, result *ScanResult) return fmt.Errorf("storing blob metadata: %w", err) } } + result.BlobsCreated += len(blobs) } @@ -1148,14 +1254,17 @@ func (s *Scanner) handleBlobReady(blobWithReader *blob.BlobWithReader) error { } blobPath := fmt.Sprintf("blobs/%s/%s/%s", finishedBlob.Hash[:2], finishedBlob.Hash[2:4], finishedBlob.Hash) + blobExists, err := s.uploadBlobIfNeeded(ctx, blobPath, blobWithReader, startTime) if err != nil { s.cleanupBlobTempFile(blobWithReader) + return fmt.Errorf("uploading blob %s: %w", finishedBlob.Hash, err) } if err := s.recordBlobMetadata(ctx, finishedBlob, blobExists, startTime); err != nil { s.cleanupBlobTempFile(blobWithReader) + return err } @@ -1181,25 +1290,28 @@ func (s *Scanner) uploadBlobIfNeeded(ctx context.Context, blobPath string, blobW if _, err := s.storage.Stat(ctx, blobPath); err == nil { log.Info("Blob already exists in storage, skipping upload", "hash", finishedBlob.Hash, "size", humanize.Bytes(uint64(finishedBlob.Compressed))) - s.ui.Info("Blob %s (%s) already exists at %s. Skipping upload.", + s.ui.Infof("Blob %s (%s) already exists at %s. Skipping upload.", s.ui.Hex(finishedBlob.Hash), s.ui.Size(finishedBlob.Compressed), s.ui.Path(destination)) + return true, nil } - s.ui.Begin("Uploading blob %s (%s) to %s.", + s.ui.Beginf("Uploading blob %s (%s) to %s.", s.ui.Hex(finishedBlob.Hash), s.ui.Size(finishedBlob.Compressed), s.ui.Path(destination)) progressCallback := s.makeUploadProgressCallback(ctx, finishedBlob, startTime) - if err := s.storage.PutWithProgress(ctx, blobPath, blobWithReader.Reader, finishedBlob.Compressed, progressCallback); err != nil { + err := s.storage.PutWithProgress(ctx, blobPath, blobWithReader.Reader, finishedBlob.Compressed, progressCallback) + if err != nil { log.Error("Failed to upload blob", "hash", finishedBlob.Hash, "error", err) + return false, fmt.Errorf("uploading blob to storage: %w", err) } uploadDuration := time.Since(startTime) uploadSpeedBps := float64(finishedBlob.Compressed) / uploadDuration.Seconds() - s.ui.Complete("Uploaded blob %s (%s) in %s at %s.", + s.ui.Completef("Uploaded blob %s (%s) in %s at %s.", s.ui.Hex(finishedBlob.Hash), s.ui.Size(finishedBlob.Compressed), s.ui.Duration(uploadDuration), @@ -1228,17 +1340,21 @@ func (s *Scanner) makeUploadProgressCallback(ctx context.Context, finishedBlob * lastProgressTime := time.Now() lastProgressBytes := int64(0) lastStdoutTime := time.Now() + const stdoutInterval = 15 * time.Second return func(uploaded int64) error { now := time.Now() + elapsed := now.Sub(lastProgressTime).Seconds() if elapsed > 0.5 { bytesSinceLastUpdate := uploaded - lastProgressBytes + speed := float64(bytesSinceLastUpdate) / elapsed if s.progress != nil { s.progress.ReportUploadProgress(finishedBlob.Hash, uploaded, finishedBlob.Compressed, speed) } + lastProgressTime = now lastProgressBytes = uploaded } @@ -1248,11 +1364,13 @@ func (s *Scanner) makeUploadProgressCallback(ctx context.Context, finishedBlob * totalElapsed := now.Sub(uploadStart) pct := float64(uploaded) / float64(finishedBlob.Compressed) * 100 avgSpeed := float64(uploaded) / totalElapsed.Seconds() + var eta time.Duration if avgSpeed > 0 { eta = time.Duration(float64(finishedBlob.Compressed-uploaded)/avgSpeed) * time.Second } - s.ui.Progress("Blob upload %s: %s / %s (%s) at %s, blob upload elapsed: %s, blob upload ETA: %s (est remain %s).", + + s.ui.Progressf("Blob upload %s: %s / %s (%s) at %s, blob upload elapsed: %s, blob upload ETA: %s (est remain %s).", s.ui.Hex(finishedBlob.Hash), s.ui.Size(uploaded), s.ui.Size(finishedBlob.Compressed), @@ -1283,11 +1401,13 @@ func (s *Scanner) recordBlobMetadata(ctx context.Context, finishedBlob *blob.Fin uploadDuration := time.Since(startTime) return s.repos.WithTx(ctx, func(txCtx context.Context, tx *sql.Tx) error { - if err := s.repos.Blobs.UpdateUploaded(txCtx, tx, finishedBlob.ID); err != nil { + err := s.repos.Blobs.UpdateUploaded(txCtx, tx, finishedBlob.ID) + if err != nil { return fmt.Errorf("updating blob upload timestamp: %w", err) } - if err := s.repos.Snapshots.AddBlob(txCtx, tx, s.snapshotID, finishedBlobID, types.BlobHash(finishedBlob.Hash)); err != nil { + err = s.repos.Snapshots.AddBlob(txCtx, tx, s.snapshotID, finishedBlobID, types.BlobHash(finishedBlob.Hash)) + if err != nil { return fmt.Errorf("adding blob to snapshot: %w", err) } @@ -1299,7 +1419,9 @@ func (s *Scanner) recordBlobMetadata(ctx context.Context, finishedBlob *blob.Fin Size: finishedBlob.Compressed, DurationMs: uploadDuration.Milliseconds(), } - if err := s.repos.Uploads.Create(txCtx, tx, upload); err != nil { + + err := s.repos.Uploads.Create(txCtx, tx, upload) + if err != nil { return fmt.Errorf("recording upload metrics: %w", err) } } @@ -1312,10 +1434,14 @@ func (s *Scanner) recordBlobMetadata(ctx context.Context, finishedBlob *blob.Fin func (s *Scanner) cleanupBlobTempFile(blobWithReader *blob.BlobWithReader) { if blobWithReader.TempFile != nil { tempName := blobWithReader.TempFile.Name() - if err := blobWithReader.TempFile.Close(); err != nil { + + err := blobWithReader.TempFile.Close() + if err != nil { log.Fatal("Failed to close temp file", "file", tempName, "error", err) } - if err := s.fs.Remove(tempName); err != nil { + + err = s.fs.Remove(tempName) + if err != nil { log.Fatal("Failed to remove temp file", "file", tempName, "error", err) } } @@ -1343,6 +1469,7 @@ func (s *Scanner) processFileStreaming(ctx context.Context, fileToProcess *FileT defer func() { _ = file.Close() }() var chunks []streamingChunkInfo + chunkIndex := 0 fileHash, err := s.chunker.ChunkReaderStreaming(file, func(chunk chunker.Chunk) error { @@ -1372,16 +1499,17 @@ func (s *Scanner) processFileStreaming(ctx context.Context, fileToProcess *FileT s.updateChunkStats(chunkExists, chunk.Size, result) if !chunkExists { - if err := s.addChunkToPacker(chunk); err != nil { + err := s.addChunkToPacker(chunk) + if err != nil { return err } } chunk.Data = nil chunkIndex++ + return nil }) - if err != nil { return fmt.Errorf("chunking file: %w", err) } @@ -1390,6 +1518,7 @@ func (s *Scanner) processFileStreaming(ctx context.Context, fileToProcess *FileT "path", fileToProcess.Path, "file_hash", fileHash, "chunks", len(chunks)) s.queueFileForBatchInsert(ctx, fileToProcess, chunks) + return nil } @@ -1397,6 +1526,7 @@ func (s *Scanner) processFileStreaming(ctx context.Context, fileToProcess *FileT func (s *Scanner) updateChunkStats(chunkExists bool, chunkSize int64, result *ScanResult) { if chunkExists { result.FilesSkipped++ + result.BytesSkipped += chunkSize if s.progress != nil { s.progress.GetStats().BytesSkipped.Add(chunkSize) @@ -1404,6 +1534,7 @@ func (s *Scanner) updateChunkStats(chunkExists bool, chunkSize int64, result *Sc } else { result.ChunksCreated++ result.BytesScanned += chunkSize + if s.progress != nil { s.progress.GetStats().ChunksCreated.Add(1) s.progress.GetStats().BytesProcessed.Add(chunkSize) @@ -1415,27 +1546,36 @@ func (s *Scanner) updateChunkStats(chunkExists bool, chunkSize int64, result *Sc // addChunkToPacker adds a chunk to the blob packer, finalizing the current blob if needed func (s *Scanner) addChunkToPacker(chunk chunker.Chunk) error { s.packerMu.Lock() + err := s.packer.AddChunk(&blob.ChunkRef{Hash: chunk.Hash, Data: chunk.Data}) - if err == blob.ErrBlobSizeLimitExceeded { - if err := s.packer.FinalizeBlob(); err != nil { + if errors.Is(err, blob.ErrBlobSizeLimitExceeded) { + err := s.packer.FinalizeBlob() + if err != nil { s.packerMu.Unlock() + return fmt.Errorf("finalizing blob: %w", err) } - if err := s.packer.AddChunk(&blob.ChunkRef{Hash: chunk.Hash, Data: chunk.Data}); err != nil { + + err = s.packer.AddChunk(&blob.ChunkRef{Hash: chunk.Hash, Data: chunk.Data}) + if err != nil { s.packerMu.Unlock() + return fmt.Errorf("adding chunk after finalize: %w", err) } } else if err != nil { s.packerMu.Unlock() + return fmt.Errorf("adding chunk to packer: %w", err) } s.packerMu.Unlock() + return nil } // queueFileForBatchInsert builds file/chunk associations and queues the file for batch DB insert func (s *Scanner) queueFileForBatchInsert(ctx context.Context, fileToProcess *FileToProcess, chunks []streamingChunkInfo) { fileChunks := make([]database.FileChunk, len(chunks)) + chunkFiles := make([]database.ChunkFile, len(chunks)) for i, ci := range chunks { fileChunks[i] = database.FileChunk{ @@ -1489,7 +1629,7 @@ func (s *Scanner) detectDeletedFilesFromMap(ctx context.Context, knownFiles map[ } if result.FilesDeleted > 0 { - s.ui.Info("Snapshot source files enumeration detected %s deleted files.", s.ui.Count(result.FilesDeleted)) + s.ui.Infof("Snapshot source files enumeration detected %s deleted files.", s.ui.Count(result.FilesDeleted)) } return nil @@ -1503,6 +1643,7 @@ func wrapPermissionError(path string, err error) error { if !errors.Is(err, os.ErrPermission) { return err } + if runtime.GOOS == "darwin" { return fmt.Errorf("cannot read %s: %w\n\n"+ "macOS is blocking access to this path. Grant Full Disk Access to your\n"+ @@ -1510,12 +1651,14 @@ func wrapPermissionError(path string, err error) error { " System Settings → Privacy & Security → Full Disk Access\n\n"+ "then quit and reopen the terminal and re-run the backup", path, err) } + return fmt.Errorf("cannot read %s: %w (check file permissions, or run with --skip-errors to continue past unreadable files)", path, err) } // compileExcludePatterns compiles the exclude patterns into glob matchers func compileExcludePatterns(patterns []string) []compiledPattern { var compiled []compiledPattern + for _, p := range patterns { if p == "" { continue @@ -1523,6 +1666,7 @@ func compileExcludePatterns(patterns []string) []compiledPattern { // Check if pattern is anchored (starts with /) anchored := strings.HasPrefix(p, "/") + pattern := p if anchored { pattern = p[1:] // Remove leading / @@ -1537,6 +1681,7 @@ func compileExcludePatterns(patterns []string) []compiledPattern { g, err := glob.Compile(pattern, '/') if err != nil { log.Warn("Invalid exclude pattern, skipping", "pattern", p, "error", err) + continue } @@ -1546,6 +1691,7 @@ func compileExcludePatterns(patterns []string) []compiledPattern { original: p, }) } + return compiled } diff --git a/internal/snapshot/scanner_test.go b/internal/snapshot/scanner_test.go index 4ceaf84..4ed2d3b 100644 --- a/internal/snapshot/scanner_test.go +++ b/internal/snapshot/scanner_test.go @@ -33,16 +33,22 @@ func TestScannerSimpleDirectory(t *testing.T) { // Create files with specific times testTime := time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC) + for path, content := range testFiles { dir := filepath.Dir(path) - if err := fs.MkdirAll(dir, 0755); err != nil { + + err := fs.MkdirAll(dir, 0755) + if err != nil { t.Fatalf("failed to create directory %s: %v", dir, err) } - if err := afero.WriteFile(fs, path, []byte(content), 0644); err != nil { + + err = afero.WriteFile(fs, path, []byte(content), 0644) + if err != nil { t.Fatalf("failed to write file %s: %v", path, err) } // Set times - if err := fs.Chtimes(path, testTime, testTime); err != nil { + err = fs.Chtimes(path, testTime, testTime) + if err != nil { t.Fatalf("failed to set times for %s: %v", path, err) } } @@ -53,7 +59,8 @@ func TestScannerSimpleDirectory(t *testing.T) { t.Fatalf("failed to create test database: %v", err) } defer func() { - if err := db.Close(); err != nil { + err := db.Close() + if err != nil { t.Errorf("failed to close database: %v", err) } }() @@ -73,6 +80,7 @@ func TestScannerSimpleDirectory(t *testing.T) { // Create a snapshot record for testing ctx := context.Background() snapshotID := "test-snapshot-001" + err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { snapshot := &database.Snapshot{ ID: types.SnapshotID(snapshotID), @@ -87,6 +95,7 @@ func TestScannerSimpleDirectory(t *testing.T) { BlobSize: 0, CompressionRatio: 1.0, } + return repos.Snapshots.Create(ctx, tx, snapshot) }) if err != nil { @@ -95,6 +104,7 @@ func TestScannerSimpleDirectory(t *testing.T) { // Scan the directory var result *snapshot.ScanResult + result, err = scanner.Scan(ctx, "/source", snapshotID) if err != nil { t.Fatalf("scan failed: %v", err) @@ -170,7 +180,7 @@ func TestScannerLargeFile(t *testing.T) { // Use random content to ensure good chunk boundaries largeContent := make([]byte, 1024*1024) // 1MB // Fill with pseudo-random data to ensure chunk boundaries - for i := 0; i < len(largeContent); i++ { + for i := range largeContent { // Simple pseudo-random generator for deterministic tests largeContent[i] = byte((i * 7919) ^ (i >> 3)) } @@ -178,6 +188,7 @@ func TestScannerLargeFile(t *testing.T) { if err := fs.MkdirAll("/source", 0755); err != nil { t.Fatal(err) } + if err := afero.WriteFile(fs, "/source/large.bin", largeContent, 0644); err != nil { t.Fatal(err) } @@ -188,7 +199,8 @@ func TestScannerLargeFile(t *testing.T) { t.Fatalf("failed to create test database: %v", err) } defer func() { - if err := db.Close(); err != nil { + err := db.Close() + if err != nil { t.Errorf("failed to close database: %v", err) } }() @@ -208,6 +220,7 @@ func TestScannerLargeFile(t *testing.T) { // Create a snapshot record for testing ctx := context.Background() snapshotID := "test-snapshot-001" + err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { snapshot := &database.Snapshot{ ID: types.SnapshotID(snapshotID), @@ -222,6 +235,7 @@ func TestScannerLargeFile(t *testing.T) { BlobSize: 0, CompressionRatio: 1.0, } + return repos.Snapshots.Create(ctx, tx, snapshot) }) if err != nil { @@ -230,6 +244,7 @@ func TestScannerLargeFile(t *testing.T) { // Scan the directory var result *snapshot.ScanResult + result, err = scanner.Scan(ctx, "/source", snapshotID) if err != nil { t.Fatalf("scan failed: %v", err) diff --git a/internal/snapshot/snapshot.go b/internal/snapshot/snapshot.go index f3f26d6..545d6e7 100644 --- a/internal/snapshot/snapshot.go +++ b/internal/snapshot/snapshot.go @@ -37,6 +37,8 @@ import ( "bytes" "context" "database/sql" + "encoding/hex" + "errors" "fmt" "io" "os/exec" @@ -97,12 +99,13 @@ func (sm *SnapshotManager) CreateSnapshot(ctx context.Context, hostname, version func (sm *SnapshotManager) CreateSnapshotWithName(ctx context.Context, hostname, name, version, gitRevision string) (string, error) { // Use short hostname (strip domain if present) shortHostname := hostname - if idx := strings.Index(hostname, "."); idx != -1 { - shortHostname = hostname[:idx] + if before, _, ok := strings.Cut(hostname, "."); ok { + shortHostname = before } // Build snapshot ID with optional name timestamp := time.Now().UTC().Format("2006-01-02T15:04:05Z") + var snapshotID string if name != "" { snapshotID = fmt.Sprintf("%s_%s_%s", shortHostname, name, timestamp) @@ -128,12 +131,12 @@ func (sm *SnapshotManager) CreateSnapshotWithName(ctx context.Context, hostname, err := sm.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { return sm.repos.Snapshots.Create(ctx, tx, snapshot) }) - if err != nil { return "", fmt.Errorf("creating snapshot: %w", err) } log.Info("Created snapshot", "snapshot_id", snapshotID) + return snapshotID, nil } @@ -148,7 +151,6 @@ func (sm *SnapshotManager) UpdateSnapshotStats(ctx context.Context, snapshotID s stats.BytesUploaded, ) }) - if err != nil { return fmt.Errorf("updating snapshot stats: %w", err) } @@ -161,13 +163,14 @@ func (sm *SnapshotManager) UpdateSnapshotStats(ctx context.Context, snapshotID s func (sm *SnapshotManager) UpdateSnapshotStatsExtended(ctx context.Context, snapshotID string, stats ExtendedBackupStats) error { return sm.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { // First update basic stats - if err := sm.repos.Snapshots.UpdateCounts(ctx, tx, snapshotID, + err := sm.repos.Snapshots.UpdateCounts(ctx, tx, snapshotID, int64(stats.FilesScanned), int64(stats.ChunksCreated), int64(stats.BlobsCreated), stats.BytesScanned, stats.BytesUploaded, - ); err != nil { + ) + if err != nil { return err } @@ -190,18 +193,20 @@ func (sm *SnapshotManager) CompleteSnapshot(ctx context.Context, snapshotID stri if err != nil { return err } + if added > 0 { log.Info("Populated snapshot_blobs with dedup-referenced blobs", "snapshot_id", snapshotID, "added", added) } + return sm.repos.Snapshots.MarkComplete(ctx, tx, snapshotID) }) - if err != nil { return fmt.Errorf("marking snapshot complete: %w", err) } log.Info("Completed snapshot", "snapshot_id", snapshotID) + return nil } @@ -229,10 +234,13 @@ func (sm *SnapshotManager) ExportSnapshotMetadata(ctx context.Context, dbPath st if err != nil { return fmt.Errorf("creating temp dir: %w", err) } + log.Debug("Created temporary directory", "path", tempDir) defer func() { log.Debug("Cleaning up temporary directory", "path", tempDir) - if err := sm.fs.RemoveAll(tempDir); err != nil { + + err := sm.fs.RemoveAll(tempDir) + if err != nil { log.Debug("Failed to remove temp dir", "path", tempDir, "error", err) } }() @@ -258,6 +266,7 @@ func (sm *SnapshotManager) ExportSnapshotMetadata(ctx context.Context, dbPath st "snapshot_id", snapshotID, "db_size", len(finalData), "manifest_size", len(blobManifest)) + return nil } @@ -268,17 +277,21 @@ func (sm *SnapshotManager) prepareExportDB(ctx context.Context, dbPath, snapshot // The main database should be closed at this point tempDBPath := filepath.Join(tempDir, "snapshot.db") log.Debug("Copying database to temporary location", "source", dbPath, "destination", tempDBPath) + if err := sm.copyFile(dbPath, tempDBPath); err != nil { return nil, "", fmt.Errorf("copying database: %w", err) } + log.Debug("Database copy complete", "size", sm.getFileSize(tempDBPath)) // Step 2: Clean the temp database to only contain current snapshot data log.Debug("Cleaning temporary database", "snapshot_id", snapshotID) + stats, err := sm.cleanSnapshotDB(ctx, tempDBPath, snapshotID) if err != nil { return nil, "", fmt.Errorf("cleaning snapshot database: %w", err) } + log.Info("Temporary database cleanup complete", "db_path", tempDBPath, "size_after_clean", humanize.Bytes(uint64(sm.getFileSize(tempDBPath))), @@ -294,6 +307,7 @@ func (sm *SnapshotManager) prepareExportDB(ctx context.Context, dbPath, snapshot if err := sm.vacuumDatabase(tempDBPath); err != nil { return nil, "", fmt.Errorf("vacuuming database: %w", err) } + log.Debug("Database vacuumed", "size", humanize.Bytes(uint64(sm.getFileSize(tempDBPath)))) // Step 4: Compress and encrypt the binary database file @@ -301,6 +315,7 @@ func (sm *SnapshotManager) prepareExportDB(ctx context.Context, dbPath, snapshot if err := sm.compressFile(tempDBPath, compressedPath); err != nil { return nil, "", fmt.Errorf("compressing database: %w", err) } + log.Debug("Compression complete", "original_size", humanize.Bytes(uint64(sm.getFileSize(tempDBPath))), "compressed_size", humanize.Bytes(uint64(sm.getFileSize(compressedPath)))) @@ -327,9 +342,12 @@ func (sm *SnapshotManager) uploadSnapshotArtifacts(ctx context.Context, snapshot dbKey := fmt.Sprintf("metadata/%s/db.zst.age", remoteKey) dbUploadStart := time.Now() - if err := sm.storage.Put(ctx, dbKey, bytes.NewReader(dbData)); err != nil { + + err := sm.storage.Put(ctx, dbKey, bytes.NewReader(dbData)) + if err != nil { return fmt.Errorf("uploading snapshot database: %w", err) } + dbUploadDuration := time.Since(dbUploadStart) dbUploadSpeed := float64(len(dbData)) * 8 / dbUploadDuration.Seconds() // bits per second log.Info("Uploaded snapshot database", @@ -341,9 +359,12 @@ func (sm *SnapshotManager) uploadSnapshotArtifacts(ctx context.Context, snapshot // Upload blob manifest (compressed only, not encrypted) manifestKey := fmt.Sprintf("metadata/%s/manifest.json.zst", remoteKey) manifestUploadStart := time.Now() - if err := sm.storage.Put(ctx, manifestKey, bytes.NewReader(manifestData)); err != nil { + + err = sm.storage.Put(ctx, manifestKey, bytes.NewReader(manifestData)) + if err != nil { return fmt.Errorf("uploading blob manifest: %w", err) } + manifestUploadDuration := time.Since(manifestUploadStart) manifestUploadSpeed := float64(len(manifestData)) * 8 / manifestUploadDuration.Seconds() // bits per second log.Info("Uploaded blob manifest", @@ -383,7 +404,8 @@ func (sm *SnapshotManager) cleanSnapshotDB(ctx context.Context, dbPath string, s return nil, fmt.Errorf("opening temp database: %w", err) } defer func() { - if err := db.Close(); err != nil { + err := db.Close() + if err != nil { log.Debug("Failed to close temp database", "error", err) } }() @@ -394,7 +416,8 @@ func (sm *SnapshotManager) cleanSnapshotDB(ctx context.Context, dbPath string, s return nil, fmt.Errorf("beginning transaction: %w", err) } defer func() { - if rbErr := tx.Rollback(); rbErr != nil && rbErr != sql.ErrTxDone { + rbErr := tx.Rollback() + if rbErr != nil && !errors.Is(rbErr, sql.ErrTxDone) { log.Debug("Failed to rollback transaction", "error", rbErr) } }() @@ -430,6 +453,7 @@ func (sm *SnapshotManager) cleanSnapshotDB(ctx context.Context, dbPath string, s // Commit transaction log.Debug("[Temp DB Cleanup] Committing cleanup transaction") + if err := tx.Commit(); err != nil { return nil, fmt.Errorf("committing transaction: %w", err) } @@ -439,23 +463,30 @@ func (sm *SnapshotManager) cleanSnapshotDB(ctx context.Context, dbPath string, s // Count files var fileCount int + err = db.QueryRowWithLog(ctx, "SELECT COUNT(*) FROM files").Scan(&fileCount) if err != nil { return nil, fmt.Errorf("counting files: %w", err) } + stats.FileCount = fileCount // Count chunks var chunkCount int + err = db.QueryRowWithLog(ctx, "SELECT COUNT(*) FROM chunks").Scan(&chunkCount) if err != nil { return nil, fmt.Errorf("counting chunks: %w", err) } + stats.ChunkCount = chunkCount // Count blobs and get sizes - var blobCount int - var compressedSize, uncompressedSize sql.NullInt64 + var ( + blobCount int + compressedSize, uncompressedSize sql.NullInt64 + ) + err = db.QueryRowWithLog(ctx, ` SELECT COUNT(*), COALESCE(SUM(compressed_size), 0), COALESCE(SUM(uncompressed_size), 0) FROM blobs @@ -464,6 +495,7 @@ func (sm *SnapshotManager) cleanSnapshotDB(ctx context.Context, dbPath string, s if err != nil { return nil, fmt.Errorf("counting blobs and sizes: %w", err) } + stats.BlobCount = blobCount stats.CompressedSize = compressedSize.Int64 stats.UncompressedSize = uncompressedSize.Int64 @@ -491,7 +523,8 @@ func (sm *SnapshotManager) compressFile(inputPath, outputPath string) error { return fmt.Errorf("opening input file: %w", err) } defer func() { - if err := input.Close(); err != nil { + err := input.Close() + if err != nil { log.Debug("Failed to close input file", "path", inputPath, "error", err) } }() @@ -501,13 +534,15 @@ func (sm *SnapshotManager) compressFile(inputPath, outputPath string) error { return fmt.Errorf("creating output file: %w", err) } defer func() { - if err := output.Close(); err != nil { + err := output.Close() + if err != nil { log.Debug("Failed to close output file", "path", outputPath, "error", err) } }() // Use blobgen for compression and encryption log.Debug("Compressing and encrypting data") + writer, err := blobgen.NewWriter(output, sm.config.CompressionLevel, sm.config.AgeRecipients) if err != nil { return fmt.Errorf("creating blobgen writer: %w", err) @@ -517,7 +552,8 @@ func (sm *SnapshotManager) compressFile(inputPath, outputPath string) error { writerClosed := false defer func() { if !writerClosed { - if err := writer.Close(); err != nil { + err := writer.Close() + if err != nil { log.Debug("Failed to close writer", "error", err) } } @@ -531,9 +567,10 @@ func (sm *SnapshotManager) compressFile(inputPath, outputPath string) error { if err := writer.Close(); err != nil { return fmt.Errorf("closing writer: %w", err) } + writerClosed = true - log.Debug("Compression complete", "hash", fmt.Sprintf("%x", writer.Sum256())) + log.Debug("Compression complete", "hash", hex.EncodeToString(writer.Sum256())) return nil } @@ -541,34 +578,42 @@ func (sm *SnapshotManager) compressFile(inputPath, outputPath string) error { // copyFile copies a file from src to dst func (sm *SnapshotManager) copyFile(src, dst string) error { log.Debug("Opening source file for copy", "path", src) + sourceFile, err := sm.fs.Open(src) if err != nil { return err } defer func() { log.Debug("Closing source file", "path", src) - if err := sourceFile.Close(); err != nil { + + err := sourceFile.Close() + if err != nil { log.Debug("Failed to close source file", "path", src, "error", err) } }() log.Debug("Creating destination file", "path", dst) + destFile, err := sm.fs.Create(dst) if err != nil { return err } defer func() { log.Debug("Closing destination file", "path", dst) - if err := destFile.Close(); err != nil { + + err := destFile.Close() + if err != nil { log.Debug("Failed to close destination file", "path", dst, "error", err) } }() log.Debug("Copying file data") + n, err := io.Copy(destFile, sourceFile) if err != nil { return err } + log.Debug("File copy complete", "bytes_copied", n) return nil @@ -576,7 +621,6 @@ func (sm *SnapshotManager) copyFile(src, dst string) error { // generateBlobManifest creates a compressed JSON list of all blobs in the snapshot func (sm *SnapshotManager) generateBlobManifest(ctx context.Context, dbPath string, snapshotID string) ([]byte, error) { - // Open the cleaned database using the database package db, err := database.New(ctx, dbPath) if err != nil { @@ -589,10 +633,12 @@ func (sm *SnapshotManager) generateBlobManifest(ctx context.Context, dbPath stri // Get all blobs for this snapshot log.Debug("Querying blobs for snapshot", "snapshot_id", snapshotID) + blobHashes, err := repos.Snapshots.GetBlobHashes(ctx, snapshotID) if err != nil { return nil, fmt.Errorf("getting snapshot blobs: %w", err) } + log.Debug("Found blobs", "count", len(blobHashes)) // Get blob details including sizes @@ -603,8 +649,10 @@ func (sm *SnapshotManager) generateBlobManifest(ctx context.Context, dbPath stri blob, err := repos.Blobs.GetByHash(ctx, hash) if err != nil { log.Warn("Failed to get blob details", "hash", hash, "error", err) + continue } + if blob != nil { blobs = append(blobs, BlobInfo{ Hash: hash, @@ -648,6 +696,7 @@ func (sm *SnapshotManager) getFileSize(path string) int64 { if err != nil { return -1 } + return info.Size() } @@ -663,6 +712,7 @@ type BackupStats struct { // ExtendedBackupStats contains additional statistics for comprehensive tracking type ExtendedBackupStats struct { BackupStats + BlobUncompressedSize int64 // Total uncompressed size of all referenced blobs CompressionLevel int // Compression level used for this snapshot UploadDurationMs int64 // Total milliseconds spent uploading to S3 @@ -682,6 +732,7 @@ func (sm *SnapshotManager) CleanupIncompleteSnapshots(ctx context.Context, hostn if len(incompleteSnapshots) == 0 { log.Debug("No incomplete snapshots found") + return nil } @@ -692,14 +743,15 @@ func (sm *SnapshotManager) CleanupIncompleteSnapshots(ctx context.Context, hostn // Check if metadata exists in storage (paths use the hashed // remote key so we don't leak host info to the listing). metadataKey := fmt.Sprintf("metadata/%s/db.zst", RemoteSnapshotKey(snapshot.ID.String())) - _, err := sm.storage.Stat(ctx, metadataKey) + _, err := sm.storage.Stat(ctx, metadataKey) if err != nil { // Metadata doesn't exist in S3 - this is an incomplete snapshot log.Info("Cleaning up incomplete snapshot record", "snapshot_id", snapshot.ID, "started_at", snapshot.StartedAt) // Delete the snapshot and all its associations - if err := sm.deleteSnapshot(ctx, snapshot.ID.String()); err != nil { + err := sm.deleteSnapshot(ctx, snapshot.ID.String()) + if err != nil { return fmt.Errorf("deleting incomplete snapshot %s: %w", snapshot.ID, err) } @@ -708,7 +760,9 @@ func (sm *SnapshotManager) CleanupIncompleteSnapshots(ctx context.Context, hostn // Metadata exists - this snapshot was completed but database wasn't updated // This shouldn't happen in normal operation, but mark it complete log.Warn("Found snapshot with remote metadata but incomplete in database", "snapshot_id", snapshot.ID) - if err := sm.repos.Snapshots.MarkComplete(ctx, nil, snapshot.ID.String()); err != nil { + + err := sm.repos.Snapshots.MarkComplete(ctx, nil, snapshot.ID.String()) + if err != nil { log.Error("Failed to mark snapshot as complete in database", "snapshot_id", snapshot.ID, "error", err) } } @@ -720,28 +774,34 @@ func (sm *SnapshotManager) CleanupIncompleteSnapshots(ctx context.Context, hostn // deleteSnapshot removes a snapshot and all its associations from the database func (sm *SnapshotManager) deleteSnapshot(ctx context.Context, snapshotID string) error { // Delete snapshot_files entries - if err := sm.repos.Snapshots.DeleteSnapshotFiles(ctx, snapshotID); err != nil { + err := sm.repos.Snapshots.DeleteSnapshotFiles(ctx, snapshotID) + if err != nil { return fmt.Errorf("deleting snapshot files: %w", err) } // Delete snapshot_blobs entries - if err := sm.repos.Snapshots.DeleteSnapshotBlobs(ctx, snapshotID); err != nil { + err = sm.repos.Snapshots.DeleteSnapshotBlobs(ctx, snapshotID) + if err != nil { return fmt.Errorf("deleting snapshot blobs: %w", err) } // Delete uploads entries (has foreign key to snapshots without CASCADE) - if err := sm.repos.Snapshots.DeleteSnapshotUploads(ctx, snapshotID); err != nil { + err = sm.repos.Snapshots.DeleteSnapshotUploads(ctx, snapshotID) + if err != nil { return fmt.Errorf("deleting snapshot uploads: %w", err) } // Delete the snapshot itself - if err := sm.repos.Snapshots.Delete(ctx, snapshotID); err != nil { + err = sm.repos.Snapshots.Delete(ctx, snapshotID) + if err != nil { return fmt.Errorf("deleting snapshot: %w", err) } // Clean up orphaned data log.Debug("Cleaning up orphaned records in main database") - if err := sm.CleanupOrphanedData(ctx); err != nil { + + err = sm.CleanupOrphanedData(ctx) + if err != nil { return fmt.Errorf("cleaning up orphaned data: %w", err) } @@ -759,28 +819,36 @@ func (sm *SnapshotManager) CleanupOrphanedData(ctx context.Context) error { // Delete orphaned files (files not in any snapshot) log.Debug("Deleting orphaned file records from database") - if err := sm.repos.Files.DeleteOrphaned(ctx); err != nil { + + err := sm.repos.Files.DeleteOrphaned(ctx) + if err != nil { return fmt.Errorf("deleting orphaned files: %w", err) } // Delete orphaned blobs (blobs not in any snapshot) // This will cascade delete blob_chunks for deleted blobs log.Debug("Deleting orphaned blob records from database") - if err := sm.repos.Blobs.DeleteOrphaned(ctx); err != nil { + + err = sm.repos.Blobs.DeleteOrphaned(ctx) + if err != nil { return fmt.Errorf("deleting orphaned blobs: %w", err) } // Delete orphaned blob_chunks entries // This handles cases where the blob still exists but chunks were deleted log.Debug("Deleting orphaned blob_chunks associations from database") - if err := sm.repos.BlobChunks.DeleteOrphaned(ctx); err != nil { + + err = sm.repos.BlobChunks.DeleteOrphaned(ctx) + if err != nil { return fmt.Errorf("deleting orphaned blob_chunks: %w", err) } // Delete orphaned chunks (chunks not referenced by any file) // This must come after cleaning up blob_chunks to avoid foreign key violations log.Debug("Deleting orphaned chunk records from database") - if err := sm.repos.Chunks.DeleteOrphaned(ctx); err != nil { + + err = sm.repos.Chunks.DeleteOrphaned(ctx) + if err != nil { return fmt.Errorf("deleting orphaned chunks: %w", err) } @@ -793,21 +861,26 @@ func (sm *SnapshotManager) deleteOtherSnapshots(ctx context.Context, tx *sql.Tx, // First delete uploads that reference other snapshots (no CASCADE DELETE on this FK) database.LogSQL("Execute", "DELETE FROM uploads WHERE snapshot_id != ?", currentSnapshotID) + uploadResult, err := tx.ExecContext(ctx, "DELETE FROM uploads WHERE snapshot_id != ?", currentSnapshotID) if err != nil { return fmt.Errorf("deleting uploads for other snapshots: %w", err) } + uploadsDeleted, _ := uploadResult.RowsAffected() log.Debug("[Temp DB Cleanup] Deleted upload records", "count", uploadsDeleted) // Now we can safely delete the snapshots database.LogSQL("Execute", "DELETE FROM snapshots WHERE id != ?", currentSnapshotID) + result, err := tx.ExecContext(ctx, "DELETE FROM snapshots WHERE id != ?", currentSnapshotID) if err != nil { return fmt.Errorf("deleting other snapshots: %w", err) } + rowsAffected, _ := result.RowsAffected() log.Debug("[Temp DB Cleanup] Deleted snapshot records from database", "count", rowsAffected) + return nil } @@ -816,22 +889,27 @@ func (sm *SnapshotManager) deleteOrphanedSnapshotAssociations(ctx context.Contex // Delete orphaned snapshot_files log.Debug("[Temp DB Cleanup] Deleting orphaned snapshot_files associations") database.LogSQL("Execute", "DELETE FROM snapshot_files WHERE snapshot_id != ?", currentSnapshotID) + result, err := tx.ExecContext(ctx, "DELETE FROM snapshot_files WHERE snapshot_id != ?", currentSnapshotID) if err != nil { return fmt.Errorf("deleting orphaned snapshot_files: %w", err) } + rowsAffected, _ := result.RowsAffected() log.Debug("[Temp DB Cleanup] Deleted snapshot_files associations", "count", rowsAffected) // Delete orphaned snapshot_blobs log.Debug("[Temp DB Cleanup] Deleting orphaned snapshot_blobs associations") database.LogSQL("Execute", "DELETE FROM snapshot_blobs WHERE snapshot_id != ?", currentSnapshotID) + result, err = tx.ExecContext(ctx, "DELETE FROM snapshot_blobs WHERE snapshot_id != ?", currentSnapshotID) if err != nil { return fmt.Errorf("deleting orphaned snapshot_blobs: %w", err) } + rowsAffected, _ = result.RowsAffected() log.Debug("[Temp DB Cleanup] Deleted snapshot_blobs associations", "count", rowsAffected) + return nil } @@ -839,6 +917,7 @@ func (sm *SnapshotManager) deleteOrphanedSnapshotAssociations(ctx context.Contex func (sm *SnapshotManager) deleteOrphanedFiles(ctx context.Context, tx *sql.Tx, currentSnapshotID string) error { log.Debug("[Temp DB Cleanup] Deleting file records not referenced by current snapshot") database.LogSQL("Execute", `DELETE FROM files WHERE NOT EXISTS (SELECT 1 FROM snapshot_files WHERE snapshot_files.file_id = files.id AND snapshot_files.snapshot_id = ?)`, currentSnapshotID) + result, err := tx.ExecContext(ctx, ` DELETE FROM files WHERE NOT EXISTS ( @@ -849,11 +928,13 @@ func (sm *SnapshotManager) deleteOrphanedFiles(ctx context.Context, tx *sql.Tx, if err != nil { return fmt.Errorf("deleting orphaned files: %w", err) } + rowsAffected, _ := result.RowsAffected() log.Debug("[Temp DB Cleanup] Deleted file records from database", "count", rowsAffected) // Note: file_chunks will be deleted via CASCADE log.Debug("[Temp DB Cleanup] file_chunks associations deleted via CASCADE") + return nil } @@ -861,6 +942,7 @@ func (sm *SnapshotManager) deleteOrphanedFiles(ctx context.Context, tx *sql.Tx, func (sm *SnapshotManager) deleteOrphanedChunkToFileMappings(ctx context.Context, tx *sql.Tx) error { log.Debug("[Temp DB Cleanup] Deleting orphaned chunk_files associations") database.LogSQL("Execute", `DELETE FROM chunk_files WHERE NOT EXISTS (SELECT 1 FROM files WHERE files.id = chunk_files.file_id)`) + result, err := tx.ExecContext(ctx, ` DELETE FROM chunk_files WHERE NOT EXISTS ( @@ -870,8 +952,10 @@ func (sm *SnapshotManager) deleteOrphanedChunkToFileMappings(ctx context.Context if err != nil { return fmt.Errorf("deleting orphaned chunk_files: %w", err) } + rowsAffected, _ := result.RowsAffected() log.Debug("[Temp DB Cleanup] Deleted chunk_files associations", "count", rowsAffected) + return nil } @@ -879,6 +963,7 @@ func (sm *SnapshotManager) deleteOrphanedChunkToFileMappings(ctx context.Context func (sm *SnapshotManager) deleteOrphanedBlobs(ctx context.Context, tx *sql.Tx, currentSnapshotID string) error { log.Debug("[Temp DB Cleanup] Deleting blob records not referenced by current snapshot") database.LogSQL("Execute", `DELETE FROM blobs WHERE NOT EXISTS (SELECT 1 FROM snapshot_blobs WHERE snapshot_blobs.blob_hash = blobs.blob_hash AND snapshot_blobs.snapshot_id = ?)`, currentSnapshotID) + result, err := tx.ExecContext(ctx, ` DELETE FROM blobs WHERE NOT EXISTS ( @@ -889,8 +974,10 @@ func (sm *SnapshotManager) deleteOrphanedBlobs(ctx context.Context, tx *sql.Tx, if err != nil { return fmt.Errorf("deleting orphaned blobs: %w", err) } + rowsAffected, _ := result.RowsAffected() log.Debug("[Temp DB Cleanup] Deleted blob records from database", "count", rowsAffected) + return nil } @@ -898,6 +985,7 @@ func (sm *SnapshotManager) deleteOrphanedBlobs(ctx context.Context, tx *sql.Tx, func (sm *SnapshotManager) deleteOrphanedBlobToChunkMappings(ctx context.Context, tx *sql.Tx) error { log.Debug("[Temp DB Cleanup] Deleting orphaned blob_chunks associations") database.LogSQL("Execute", `DELETE FROM blob_chunks WHERE NOT EXISTS (SELECT 1 FROM blobs WHERE blobs.id = blob_chunks.blob_id)`) + result, err := tx.ExecContext(ctx, ` DELETE FROM blob_chunks WHERE NOT EXISTS ( @@ -907,14 +995,17 @@ func (sm *SnapshotManager) deleteOrphanedBlobToChunkMappings(ctx context.Context if err != nil { return fmt.Errorf("deleting orphaned blob_chunks: %w", err) } + rowsAffected, _ := result.RowsAffected() log.Debug("[Temp DB Cleanup] Deleted blob_chunks associations", "count", rowsAffected) + return nil } // deleteOrphanedChunks deletes chunks not referenced by any file or blob func (sm *SnapshotManager) deleteOrphanedChunks(ctx context.Context, tx *sql.Tx) error { log.Debug("[Temp DB Cleanup] Deleting orphaned chunk records") + query := ` DELETE FROM chunks WHERE NOT EXISTS ( @@ -926,11 +1017,14 @@ func (sm *SnapshotManager) deleteOrphanedChunks(ctx context.Context, tx *sql.Tx) WHERE blob_chunks.chunk_hash = chunks.chunk_hash )` database.LogSQL("Execute", query) + result, err := tx.ExecContext(ctx, query) if err != nil { return fmt.Errorf("deleting orphaned chunks: %w", err) } + rowsAffected, _ := result.RowsAffected() log.Debug("[Temp DB Cleanup] Deleted chunk records from database", "count", rowsAffected) + return nil } diff --git a/internal/snapshot/snapshot_test.go b/internal/snapshot/snapshot_test.go index a47df4a..969aeed 100644 --- a/internal/snapshot/snapshot_test.go +++ b/internal/snapshot/snapshot_test.go @@ -33,6 +33,7 @@ func copyFile(fs afero.Fs, src, dst string) error { defer func() { _ = destFile.Close() }() _, err = io.Copy(destFile, sourceFile) + return err } @@ -46,6 +47,7 @@ func TestCleanSnapshotDBEmptySnapshot(t *testing.T) { // Create a test database tempDir := t.TempDir() dbPath := filepath.Join(tempDir, "test.db") + db, err := database.New(ctx, dbPath) if err != nil { t.Fatalf("failed to create database: %v", err) @@ -71,9 +73,11 @@ func TestCleanSnapshotDBEmptySnapshot(t *testing.T) { chunk := &database.Chunk{ChunkHash: "orphan-chunk", Size: 500} err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { - if err := repos.Files.Create(ctx, tx, file); err != nil { + err := repos.Files.Create(ctx, tx, file) + if err != nil { return err } + return repos.Chunks.Create(ctx, tx, chunk) }) if err != nil { @@ -111,7 +115,8 @@ func TestCleanSnapshotDBEmptySnapshot(t *testing.T) { t.Fatalf("failed to open cleaned database: %v", err) } defer func() { - if err := cleanedDB.Close(); err != nil { + err := cleanedDB.Close() + if err != nil { t.Errorf("failed to close database: %v", err) } }() @@ -123,6 +128,7 @@ func TestCleanSnapshotDBEmptySnapshot(t *testing.T) { if err != nil { t.Fatalf("failed to get snapshot: %v", err) } + if verifySnapshot == nil { t.Error("snapshot should exist") } @@ -132,6 +138,7 @@ func TestCleanSnapshotDBEmptySnapshot(t *testing.T) { if err != nil { t.Fatalf("failed to check file: %v", err) } + if f != nil { t.Error("orphan file should not exist") } @@ -141,6 +148,7 @@ func TestCleanSnapshotDBEmptySnapshot(t *testing.T) { if err != nil { t.Fatalf("failed to check chunk: %v", err) } + if c != nil { t.Error("orphan chunk should not exist") } @@ -156,6 +164,7 @@ func TestCleanSnapshotDBNonExistentSnapshot(t *testing.T) { // Create a test database tempDir := t.TempDir() dbPath := filepath.Join(tempDir, "test.db") + db, err := database.New(ctx, dbPath) if err != nil { t.Fatalf("failed to create database: %v", err) diff --git a/internal/storage/file.go b/internal/storage/file.go index 1c1fcf6..2d2ecf0 100644 --- a/internal/storage/file.go +++ b/internal/storage/file.go @@ -102,26 +102,32 @@ func (f *FileStorer) PutWithProgress(ctx context.Context, key string, data io.Re // Get retrieves data from the specified key. func (f *FileStorer) Get(ctx context.Context, key string) (io.ReadCloser, error) { path := f.fullPath(key) + file, err := f.fs.Open(path) if err != nil { if os.IsNotExist(err) { return nil, ErrNotFound } + return nil, fmt.Errorf("opening file: %w", err) } + return file, nil } // Stat returns metadata about an object without retrieving its contents. func (f *FileStorer) Stat(ctx context.Context, key string) (*ObjectInfo, error) { path := f.fullPath(key) + info, err := f.fs.Stat(path) if err != nil { if os.IsNotExist(err) { return nil, ErrNotFound } + return nil, fmt.Errorf("stat file: %w", err) } + return &ObjectInfo{ Key: key, Size: info.Size(), @@ -131,19 +137,23 @@ func (f *FileStorer) Stat(ctx context.Context, key string) (*ObjectInfo, error) // Delete removes an object. func (f *FileStorer) Delete(ctx context.Context, key string) error { path := f.fullPath(key) + err := f.fs.Remove(path) if os.IsNotExist(err) { return nil // Match S3 behavior: no error if doesn't exist } + if err != nil { return fmt.Errorf("removing file: %w", err) } + return nil } // List returns all keys with the given prefix. func (f *FileStorer) List(ctx context.Context, prefix string) ([]string, error) { var keys []string + basePath := f.fullPath(prefix) // Check if base path exists @@ -151,6 +161,7 @@ func (f *FileStorer) List(ctx context.Context, prefix string) ([]string, error) if err != nil { return nil, fmt.Errorf("checking path: %w", err) } + if !exists { return keys, nil // Empty list for non-existent prefix } @@ -177,9 +188,9 @@ func (f *FileStorer) List(ctx context.Context, prefix string) ([]string, error) relPath = strings.ReplaceAll(relPath, string(filepath.Separator), "/") keys = append(keys, relPath) } + return nil }) - if err != nil { return nil, fmt.Errorf("walking directory: %w", err) } @@ -192,14 +203,17 @@ func (f *FileStorer) ListStream(ctx context.Context, prefix string) <-chan Objec ch := make(chan ObjectInfo) go func() { defer close(ch) + basePath := f.fullPath(prefix) // Check if base path exists exists, err := afero.Exists(f.fs, basePath) if err != nil { ch <- ObjectInfo{Err: fmt.Errorf("checking path: %w", err)} + return } + if !exists { return // Empty channel for non-existent prefix } @@ -209,12 +223,14 @@ func (f *FileStorer) ListStream(ctx context.Context, prefix string) <-chan Objec select { case <-ctx.Done(): ch <- ObjectInfo{Err: ctx.Err()} + return ctx.Err() default: } if err != nil { ch <- ObjectInfo{Err: err} + return nil // Continue walking despite errors } @@ -222,6 +238,7 @@ func (f *FileStorer) ListStream(ctx context.Context, prefix string) <-chan Objec relPath, err := filepath.Rel(f.basePath, path) if err != nil { ch <- ObjectInfo{Err: fmt.Errorf("computing relative path: %w", err)} + return nil } // Normalize path separators @@ -231,9 +248,11 @@ func (f *FileStorer) ListStream(ctx context.Context, prefix string) <-chan Objec Size: info.Size(), } } + return nil }) }() + return ch } @@ -257,10 +276,12 @@ func (pw *progressWriter) Write(p []byte) (int, error) { if n > 0 { pw.written += int64(n) if pw.callback != nil { - if callbackErr := pw.callback(pw.written); callbackErr != nil { + callbackErr := pw.callback(pw.written) + if callbackErr != nil { return n, callbackErr } } } + return n, err } diff --git a/internal/storage/module.go b/internal/storage/module.go index bf830d6..42cc28a 100644 --- a/internal/storage/module.go +++ b/internal/storage/module.go @@ -24,6 +24,7 @@ func NewStorer(cfg *config.Config) (Storer, error) { if cfg.StorageURL != "" { return storerFromURL(cfg.StorageURL, cfg) } + return storerFromLegacyS3Config(cfg) } @@ -71,6 +72,7 @@ func storerFromURL(rawURL string, cfg *config.Config) (Storer, error) { if err != nil { return nil, fmt.Errorf("creating S3 client: %w", err) } + return NewS3Storer(client), nil case "rclone": @@ -109,5 +111,6 @@ func storerFromLegacyS3Config(cfg *config.Config) (Storer, error) { if err != nil { return nil, fmt.Errorf("creating S3 client: %w", err) } + return NewS3Storer(client), nil } diff --git a/internal/storage/rclone.go b/internal/storage/rclone.go index aff5ba6..19d39d4 100644 --- a/internal/storage/rclone.go +++ b/internal/storage/rclone.go @@ -49,6 +49,7 @@ func NewRcloneStorer(ctx context.Context, remote, path string) (*RcloneStorer, e strings.Contains(err.Error(), "failed to find remote") { return nil, fmt.Errorf("%w: %s", ErrRemoteNotFound, remote) } + return nil, fmt.Errorf("creating rclone filesystem: %w", err) } @@ -101,9 +102,11 @@ func (r *RcloneStorer) Get(ctx context.Context, key string) (io.ReadCloser, erro if errors.Is(err, fs.ErrorObjectNotFound) { return nil, ErrNotFound } + if errors.Is(err, fs.ErrorDirNotFound) { return nil, ErrNotFound } + return nil, fmt.Errorf("getting object: %w", err) } @@ -123,9 +126,11 @@ func (r *RcloneStorer) Stat(ctx context.Context, key string) (*ObjectInfo, error if errors.Is(err, fs.ErrorObjectNotFound) { return nil, ErrNotFound } + if errors.Is(err, fs.ErrorDirNotFound) { return nil, ErrNotFound } + return nil, fmt.Errorf("getting object: %w", err) } @@ -142,9 +147,11 @@ func (r *RcloneStorer) Delete(ctx context.Context, key string) error { if errors.Is(err, fs.ErrorObjectNotFound) { return nil // Match S3 behavior: no error if doesn't exist } + if errors.Is(err, fs.ErrorDirNotFound) { return nil } + return fmt.Errorf("getting object: %w", err) } @@ -209,6 +216,7 @@ func (r *RcloneStorer) Info() StorageInfo { if r.path != "" { location += ":" + r.path } + return StorageInfo{ Type: "rclone", Location: location, @@ -227,10 +235,12 @@ func (pr *progressReader) Read(p []byte) (int, error) { if n > 0 { pr.read += int64(n) if pr.callback != nil { - if callbackErr := pr.callback(pr.read); callbackErr != nil { + callbackErr := pr.callback(pr.read) + if callbackErr != nil { return n, callbackErr } } } + return n, err } diff --git a/internal/storage/s3.go b/internal/storage/s3.go index 534aed7..64b3ede 100644 --- a/internal/storage/s3.go +++ b/internal/storage/s3.go @@ -30,6 +30,7 @@ func (s *S3Storer) PutWithProgress(ctx context.Context, key string, data io.Read if progress != nil { s3Progress = s3.ProgressCallback(progress) } + return s.client.PutObjectWithProgress(ctx, key, data, size, s3Progress) } @@ -44,6 +45,7 @@ func (s *S3Storer) Stat(ctx context.Context, key string) (*ObjectInfo, error) { if err != nil { return nil, err } + return &ObjectInfo{ Key: info.Key, Size: info.Size, @@ -65,6 +67,7 @@ func (s *S3Storer) ListStream(ctx context.Context, prefix string) <-chan ObjectI ch := make(chan ObjectInfo) go func() { defer close(ch) + for info := range s.client.ListObjectsStream(ctx, prefix, false) { ch <- ObjectInfo{ Key: info.Key, @@ -73,6 +76,7 @@ func (s *S3Storer) ListStream(ctx context.Context, prefix string) <-chan ObjectI } } }() + return ch } diff --git a/internal/storage/url.go b/internal/storage/url.go index 2dcca35..4ab169d 100644 --- a/internal/storage/url.go +++ b/internal/storage/url.go @@ -1,6 +1,7 @@ package storage import ( + "errors" "fmt" "net/url" "strings" @@ -24,15 +25,16 @@ type StorageURL struct { // - rclone://remote/path/to/backups func ParseStorageURL(rawURL string) (*StorageURL, error) { if rawURL == "" { - return nil, fmt.Errorf("storage URL is empty") + return nil, errors.New("storage URL is empty") } // Handle file:// URLs - if strings.HasPrefix(rawURL, "file://") { - path := strings.TrimPrefix(rawURL, "file://") + if after, ok := strings.CutPrefix(rawURL, "file://"); ok { + path := after if path == "" { - return nil, fmt.Errorf("file URL path is empty") + return nil, errors.New("file URL path is empty") } + return &StorageURL{ Scheme: "file", Prefix: path, @@ -48,12 +50,13 @@ func ParseStorageURL(rawURL string) (*StorageURL, error) { bucket := u.Host if bucket == "" { - return nil, fmt.Errorf("s3 URL missing bucket name") + return nil, errors.New("s3 URL missing bucket name") } prefix := strings.TrimPrefix(u.Path, "/") query := u.Query() + useSSL := true if query.Get("ssl") == "false" { useSSL = false @@ -78,7 +81,7 @@ func ParseStorageURL(rawURL string) (*StorageURL, error) { remote := u.Host if remote == "" { - return nil, fmt.Errorf("rclone URL missing remote name") + return nil, errors.New("rclone URL missing remote name") } path := strings.TrimPrefix(u.Path, "/") @@ -90,29 +93,32 @@ func ParseStorageURL(rawURL string) (*StorageURL, error) { }, nil } - return nil, fmt.Errorf("unsupported URL scheme: must start with s3://, file://, or rclone://") + return nil, errors.New("unsupported URL scheme: must start with s3://, file://, or rclone://") } // String returns a human-readable representation of the storage URL. func (u *StorageURL) String() string { switch u.Scheme { case "file": - return fmt.Sprintf("file://%s", u.Prefix) + return "file://" + u.Prefix case "s3": endpoint := u.Endpoint if endpoint == "" { endpoint = "s3.amazonaws.com" } + if u.Prefix != "" { return fmt.Sprintf("s3://%s/%s (endpoint: %s)", u.Bucket, u.Prefix, endpoint) } + return fmt.Sprintf("s3://%s (endpoint: %s)", u.Bucket, endpoint) case "rclone": if u.Prefix != "" { return fmt.Sprintf("rclone://%s/%s", u.RcloneRemote, u.Prefix) } - return fmt.Sprintf("rclone://%s", u.RcloneRemote) + + return "rclone://" + u.RcloneRemote default: - return fmt.Sprintf("%s://?", u.Scheme) + return u.Scheme + "://?" } } diff --git a/internal/types/types.go b/internal/types/types.go index 51636e1..9abc02c 100644 --- a/internal/types/types.go +++ b/internal/types/types.go @@ -24,6 +24,7 @@ func ParseFileID(s string) (FileID, error) { if err != nil { return FileID{}, err } + return FileID(id), nil } @@ -38,13 +39,15 @@ func (id FileID) Value() (driver.Value, error) { } // Scan implements sql.Scanner for database deserialization. -func (id *FileID) Scan(src interface{}) error { +func (id *FileID) Scan(src any) error { if src == nil { *id = FileID{} + return nil } var s string + switch v := src.(type) { case string: s = v @@ -58,7 +61,9 @@ func (id *FileID) Scan(src interface{}) error { if err != nil { return fmt.Errorf("invalid FileID: %w", err) } + *id = FileID(parsed) + return nil } @@ -77,6 +82,7 @@ func ParseBlobID(s string) (BlobID, error) { if err != nil { return BlobID{}, err } + return BlobID(id), nil } @@ -91,13 +97,15 @@ func (id BlobID) Value() (driver.Value, error) { } // Scan implements sql.Scanner for database deserialization. -func (id *BlobID) Scan(src interface{}) error { +func (id *BlobID) Scan(src any) error { if src == nil { *id = BlobID{} + return nil } var s string + switch v := src.(type) { case string: s = v @@ -111,7 +119,9 @@ func (id *BlobID) Scan(src interface{}) error { if err != nil { return fmt.Errorf("invalid BlobID: %w", err) } + *id = BlobID(parsed) + return nil } diff --git a/internal/ui/ui.go b/internal/ui/ui.go index 6855b41..83abf4d 100644 --- a/internal/ui/ui.go +++ b/internal/ui/ui.go @@ -94,10 +94,12 @@ func shouldColor(w io.Writer) bool { if os.Getenv("NO_COLOR") != "" { return false } + f, ok := w.(*os.File) if !ok { return false } + return term.IsTerminal(int(f.Fd())) } @@ -106,67 +108,73 @@ func (w *Writer) paint(color, s string) string { if !w.color { return s } + return color + s + ansiReset } // ───────────────────────── message methods ───────────────────────── -// Begin prints an operation-start line, left-aligned with a white marker. -func (w *Writer) Begin(format string, args ...any) { +// Beginf prints an operation-start line, left-aligned with a white marker. +func (w *Writer) Beginf(format string, args ...any) { if w.quiet { return } + w.emit(ansiWhite, Marker, "", format, args) } -// Complete prints an operation-completion line in green, left-aligned. -func (w *Writer) Complete(format string, args ...any) { +// Completef prints an operation-completion line in green, left-aligned. +func (w *Writer) Completef(format string, args ...any) { if w.quiet { return } + w.emit(ansiGreen, Marker, ansiGreen, format, args) } -// Info prints a neutral status line, left-aligned with a white marker. -func (w *Writer) Info(format string, args ...any) { +// Infof prints a neutral status line, left-aligned with a white marker. +func (w *Writer) Infof(format string, args ...any) { if w.quiet { return } + w.emit(ansiWhite, Marker, "", format, args) } -// Notice prints an attention-worthy informational line, marker in cyan. -func (w *Writer) Notice(format string, args ...any) { +// Noticef prints an attention-worthy informational line, marker in cyan. +func (w *Writer) Noticef(format string, args ...any) { if w.quiet { return } + w.emit(ansiCyan, Marker, "", format, args) } -// Warning prints "⚠️ Warning: " in orange/yellow followed by the message. -func (w *Writer) Warning(format string, args ...any) { +// Warningf prints "⚠️ Warning: " in orange/yellow followed by the message. +func (w *Writer) Warningf(format string, args ...any) { w.warnings++ prefix := "⚠️ " + w.paint(ansiYellow+ansiBold, "Warning: ") _, _ = fmt.Fprintln(w.out, prefix+fmt.Sprintf(format, args...)) } -// Error prints "🛑 ERROR: " in red followed by the message. Goes to the +// Errorf prints "🛑 ERROR: " in red followed by the message. Goes to the // same writer as everything else; callers that want stderr should // construct a separate Writer for it. -func (w *Writer) Error(format string, args ...any) { +func (w *Writer) Errorf(format string, args ...any) { w.errors++ prefix := "🛑 " + w.paint(ansiRed+ansiBold, "ERROR: ") _, _ = fmt.Fprintln(w.out, prefix+fmt.Sprintf(format, args...)) } -// Detail prints an indented continuation line under a preceding Complete +// Detailf prints an indented continuation line under a preceding Completef // (or other top-level message). Marker " 》" (white) at column 2. -// Distinct from Progress (semantically a "heartbeat") in usage but +// Distinct from Progressf (semantically a "heartbeat") in usage but // visually identical. -func (w *Writer) Detail(format string, args ...any) { +func (w *Writer) Detailf(format string, args ...any) { if w.quiet { return } + w.emit(ansiWhite, " "+Marker, "", format, args) } @@ -176,24 +184,27 @@ func (w *Writer) WarningCount() int { return w.warnings } // ErrorCount returns the number of Error() calls this writer has emitted. func (w *Writer) ErrorCount() int { return w.errors } -// Progress prints an indented heartbeat / per-item update, marker in white. -func (w *Writer) Progress(format string, args ...any) { +// Progressf prints an indented heartbeat / per-item update, marker in white. +func (w *Writer) Progressf(format string, args ...any) { if w.quiet { return } + w.emit(ansiWhite, " "+Marker, "", format, args) } -// Banner prints a line with no marker, left-aligned. Bold when color +// Bannerf prints a line with no marker, left-aligned. Bold when color // is enabled. Used for the application startup banner only. -func (w *Writer) Banner(format string, args ...any) { +func (w *Writer) Bannerf(format string, args ...any) { if w.quiet { return } + body := fmt.Sprintf(format, args...) if w.color { body = ansiBold + body + ansiReset } + _, _ = fmt.Fprintln(w.out, body) } @@ -204,6 +215,7 @@ func (w *Writer) emit(prefixColor, prefix, bodyColor, format string, args []any) if bodyColor != "" { body = w.paint(bodyColor, body) } + _, _ = fmt.Fprintln(w.out, w.paint(prefixColor, prefix)+" "+body) } @@ -219,6 +231,7 @@ func (w *Writer) Hex(s string) string { if len(s) > 12 { short = s[:12] + "..." } + return w.paint(ansiCyan, short) } @@ -244,8 +257,11 @@ func (w *Writer) Speed(bytesPerSec float64) string { if bytesPerSec <= 0 { return w.paint(ansiMagenta, "N/A") } + bitsPerSec := bytesPerSec * 8 + var s string + switch { case bitsPerSec >= 1e9: s = fmt.Sprintf("%.1f Gbit/sec", bitsPerSec/1e9) @@ -256,6 +272,7 @@ func (w *Writer) Speed(bytesPerSec float64) string { default: s = fmt.Sprintf("%.0f bit/sec", bitsPerSec) } + return w.paint(ansiMagenta, s) } @@ -270,10 +287,12 @@ func (w *Writer) Duration(d time.Duration) string { // displayed in the process's local zone. func (w *Writer) Time(t time.Time) string { t = t.Local() + now := time.Now() if t.Year() == now.Year() && t.YearDay() == now.YearDay() { return w.paint(ansiYellow, t.Format("15:04:05")) } + return w.paint(ansiYellow, t.Format("2006-01-02 15:04:05")) } diff --git a/internal/ui/ui_test.go b/internal/ui/ui_test.go index ac9ad73..0f06b86 100644 --- a/internal/ui/ui_test.go +++ b/internal/ui/ui_test.go @@ -9,6 +9,7 @@ import ( func newTestWriter(color bool) (*Writer, *bytes.Buffer) { buf := &bytes.Buffer{} + return NewWithColor(buf, color), buf } @@ -18,21 +19,22 @@ func TestMessageMethodsPlain(t *testing.T) { fn func(*Writer) want string }{ - {"Begin", func(w *Writer) { w.Begin("starting %s", "thing") }, "》 starting thing\n"}, - {"Complete", func(w *Writer) { w.Complete("done %s", "thing") }, "》 done thing\n"}, - {"Info", func(w *Writer) { w.Info("status") }, "》 status\n"}, - {"Notice", func(w *Writer) { w.Notice("note") }, "》 note\n"}, - {"Warning", func(w *Writer) { w.Warning("oops") }, "⚠️ Warning: oops\n"}, - {"Error", func(w *Writer) { w.Error("boom") }, "🛑 ERROR: boom\n"}, - {"Progress", func(w *Writer) { w.Progress("p") }, " 》 p\n"}, - {"Detail", func(w *Writer) { w.Detail("d") }, " 》 d\n"}, - {"Banner", func(w *Writer) { w.Banner("hello") }, "hello\n"}, // plain mode, no bold + {"Begin", func(w *Writer) { w.Beginf("starting %s", "thing") }, "》 starting thing\n"}, + {"Complete", func(w *Writer) { w.Completef("done %s", "thing") }, "》 done thing\n"}, + {"Info", func(w *Writer) { w.Infof("status") }, "》 status\n"}, + {"Notice", func(w *Writer) { w.Noticef("note") }, "》 note\n"}, + {"Warning", func(w *Writer) { w.Warningf("oops") }, "⚠️ Warning: oops\n"}, + {"Error", func(w *Writer) { w.Errorf("boom") }, "🛑 ERROR: boom\n"}, + {"Progress", func(w *Writer) { w.Progressf("p") }, " 》 p\n"}, + {"Detail", func(w *Writer) { w.Detailf("d") }, " 》 d\n"}, + {"Banner", func(w *Writer) { w.Bannerf("hello") }, "hello\n"}, // plain mode, no bold } for _, tt := range tests { t.Run(tt.method, func(t *testing.T) { w, buf := newTestWriter(false) tt.fn(w) + if got := buf.String(); got != tt.want { t.Errorf("got %q, want %q", got, tt.want) } @@ -45,13 +47,16 @@ func TestWarningErrorCounters(t *testing.T) { if w.WarningCount() != 0 || w.ErrorCount() != 0 { t.Fatalf("expected fresh writer to have zero counts") } - w.Info("normal") - w.Warning("first warn") - w.Warning("second warn") - w.Error("only error") + + w.Infof("normal") + w.Warningf("first warn") + w.Warningf("second warn") + w.Errorf("only error") + if got, want := w.WarningCount(), 2; got != want { t.Errorf("WarningCount: got %d, want %d", got, want) } + if got, want := w.ErrorCount(), 1; got != want { t.Errorf("ErrorCount: got %d, want %d", got, want) } @@ -59,11 +64,13 @@ func TestWarningErrorCounters(t *testing.T) { func TestColorOutputContainsANSI(t *testing.T) { w, buf := newTestWriter(true) - w.Error("boom") + w.Errorf("boom") + out := buf.String() if !strings.Contains(out, "\033[") { t.Errorf("expected ANSI escapes in color output, got %q", out) } + if !strings.Contains(out, "ERROR: ") { t.Errorf("expected 'ERROR: ' text in output, got %q", out) } @@ -71,7 +78,8 @@ func TestColorOutputContainsANSI(t *testing.T) { func TestBannerBoldWhenColor(t *testing.T) { w, buf := newTestWriter(true) - w.Banner("hello") + w.Bannerf("hello") + out := buf.String() if !strings.Contains(out, "\033[1m") { t.Errorf("expected bold ANSI escape in colored Banner output, got %q", out) @@ -84,18 +92,23 @@ func TestValueFormattersPlain(t *testing.T) { if got := w.Hex("0123456789abcdef0123"); got != "0123456789ab..." { t.Errorf("Hex long: got %q", got) } + if got := w.Hex("short"); got != "short" { t.Errorf("Hex short: got %q", got) } + if got := w.Size(1024); got != "1.0 kB" { t.Errorf("Size: got %q", got) } + if got := w.Duration(90 * time.Second); got != "1m30s" { t.Errorf("Duration: got %q", got) } + if got := w.Count(12345); got != "12,345" { t.Errorf("Count: got %q", got) } + if got := w.Percent(12.34); got != "12.3%" { t.Errorf("Percent: got %q", got) } @@ -104,9 +117,11 @@ func TestValueFormattersPlain(t *testing.T) { if got := w.Speed(0); got != "N/A" { t.Errorf("Speed(0): got %q, want N/A", got) } + if got := w.Speed(125_000_000); got != "1.0 Gbit/sec" { // 1 Gbit/s = 125 MB/s t.Errorf("Speed(125e6): got %q", got) } + if got := w.Speed(125_000); got != "1 Mbit/sec" { t.Errorf("Speed(125e3): got %q", got) } @@ -116,6 +131,7 @@ func TestValueFormattersPlain(t *testing.T) { if got := w.Time(today); got != "14:30:45" { t.Errorf("Time today: got %q, want 14:30:45", got) } + other := time.Date(2030, 1, 2, 3, 4, 5, 0, time.Local) if got := w.Time(other); got != "2030-01-02 03:04:05" { t.Errorf("Time other day: got %q", got) @@ -124,10 +140,12 @@ func TestValueFormattersPlain(t *testing.T) { func TestValueFormattersColored(t *testing.T) { w, _ := newTestWriter(true) + hex := w.Hex("0123456789abcdef0123") if !strings.Contains(hex, "\033[") { t.Errorf("expected ANSI in colored Hex output, got %q", hex) } + if !strings.Contains(hex, "0123456789ab") { t.Errorf("expected hex content in output, got %q", hex) } diff --git a/internal/vaultik/blob_fetch.go b/internal/vaultik/blob_fetch.go index dde5f34..1bd5931 100644 --- a/internal/vaultik/blob_fetch.go +++ b/internal/vaultik/blob_fetch.go @@ -4,6 +4,7 @@ import ( "context" "crypto/sha256" "encoding/hex" + "errors" "fmt" "io" "time" @@ -26,9 +27,10 @@ type hashVerifyReader struct { func (h *hashVerifyReader) Read(p []byte) (int, error) { n, err := h.reader.Read(p) - if err == io.EOF { + if errors.Is(err, io.EOF) { h.done = true } + return n, err } @@ -41,6 +43,7 @@ func (h *hashVerifyReader) Close() error { firstHash := h.reader.Sum256() secondHasher := sha256.New() secondHasher.Write(firstHash) + actualHashHex := hex.EncodeToString(secondHasher.Sum(nil)) if actualHashHex != h.blobHash { return fmt.Errorf("blob hash mismatch: expected %s, got %s", h.blobHash[:16], actualHashHex[:16]) @@ -50,6 +53,7 @@ func (h *hashVerifyReader) Close() error { if readerErr != nil { return readerErr } + return fetcherErr } @@ -66,6 +70,7 @@ func (v *Vaultik) FetchAndDecryptBlob(ctx context.Context, blobHash string, expe reader, err := blobgen.NewReader(rc, identity) if err != nil { _ = rc.Close() + return nil, fmt.Errorf("creating blob reader: %w", err) } @@ -86,6 +91,7 @@ func (v *Vaultik) FetchBlob(ctx context.Context, blobHash string, expectedSize i t0 := time.Now() rc, err := v.Storage.Get(ctx, blobPath) getDur := time.Since(t0) + if err != nil { return nil, 0, fmt.Errorf("downloading blob %s: %w", blobHash[:16], err) } @@ -93,8 +99,10 @@ func (v *Vaultik) FetchBlob(ctx context.Context, blobHash string, expectedSize i t0 = time.Now() info, err := v.Storage.Stat(ctx, blobPath) statDur := time.Since(t0) + if err != nil { _ = rc.Close() + return nil, 0, fmt.Errorf("stat blob %s: %w", blobHash[:16], err) } diff --git a/internal/vaultik/blob_fetch_hash_test.go b/internal/vaultik/blob_fetch_hash_test.go index c8f6099..6dee0b6 100644 --- a/internal/vaultik/blob_fetch_hash_test.go +++ b/internal/vaultik/blob_fetch_hash_test.go @@ -24,17 +24,22 @@ func TestFetchAndDecryptBlobVerifiesHash(t *testing.T) { // Create test data and encrypt it using blobgen.Writer plaintext := []byte("hello world test data for blob hash verification") + var encBuf bytes.Buffer + writer, err := blobgen.NewWriter(&encBuf, 1, []string{identity.Recipient().String()}) if err != nil { t.Fatalf("creating blobgen writer: %v", err) } + if _, err := writer.Write(plaintext); err != nil { t.Fatalf("writing plaintext: %v", err) } + if err := writer.Close(); err != nil { t.Fatalf("closing writer: %v", err) } + encryptedData := encBuf.Bytes() // Compute correct double-SHA-256 hash of the plaintext (matches blobgen.Writer.Sum256) @@ -51,6 +56,7 @@ func TestFetchAndDecryptBlobVerifiesHash(t *testing.T) { // Set up mock storage with the blob at the correct path mockStorage := NewMockStorer() blobPath := "blobs/" + correctHash[:2] + "/" + correctHash[2:4] + "/" + correctHash + mockStorage.mu.Lock() mockStorage.data[blobPath] = encryptedData mockStorage.mu.Unlock() @@ -63,13 +69,16 @@ func TestFetchAndDecryptBlobVerifiesHash(t *testing.T) { if err != nil { t.Fatalf("expected success, got error: %v", err) } + data, err := io.ReadAll(rc) if err != nil { t.Fatalf("reading stream: %v", err) } + if err := rc.Close(); err != nil { t.Fatalf("close (hash verification) failed: %v", err) } + if !bytes.Equal(data, plaintext) { t.Fatalf("decrypted data mismatch: got %q, want %q", data, plaintext) } @@ -79,6 +88,7 @@ func TestFetchAndDecryptBlobVerifiesHash(t *testing.T) { // Use a fake hash that doesn't match the actual plaintext fakeHash := strings.Repeat("ab", 32) // 64 hex chars fakePath := "blobs/" + fakeHash[:2] + "/" + fakeHash[2:4] + "/" + fakeHash + mockStorage.mu.Lock() mockStorage.data[fakePath] = encryptedData mockStorage.mu.Unlock() @@ -89,10 +99,12 @@ func TestFetchAndDecryptBlobVerifiesHash(t *testing.T) { } // Read all data — hash is verified on Close _, _ = io.ReadAll(rc) + err = rc.Close() if err == nil { t.Fatal("expected error for mismatched hash, got nil") } + if !strings.Contains(err.Error(), "hash mismatch") { t.Fatalf("expected hash mismatch error, got: %v", err) } diff --git a/internal/vaultik/blobcache.go b/internal/vaultik/blobcache.go index 7ebbefe..2a28f18 100644 --- a/internal/vaultik/blobcache.go +++ b/internal/vaultik/blobcache.go @@ -53,6 +53,7 @@ func newBlobDiskCache(maxBytes int64) (*blobDiskCache, error) { if err != nil { return nil, fmt.Errorf("creating blob cache dir: %w", err) } + return &blobDiskCache{ dir: dir, maxBytes: maxBytes, @@ -70,21 +71,25 @@ func (c *blobDiskCache) unlink(e *blobDiskCacheEntry) { } else { c.head = e.next } + if e.next != nil { e.next.prev = e.prev } else { c.tail = e.prev } + e.prev = nil e.next = nil } func (c *blobDiskCache) pushFront(e *blobDiskCacheEntry) { e.prev = nil + e.next = c.head if c.head != nil { c.head.prev = e } + c.head = e if c.tail == nil { c.tail = e @@ -95,6 +100,7 @@ func (c *blobDiskCache) evictLRU() { if c.tail == nil { return } + victim := c.tail c.unlink(victim) delete(c.items, victim.key) @@ -121,7 +127,8 @@ func (c *blobDiskCache) Put(key string, data []byte) error { delete(c.items, key) } - if err := os.WriteFile(c.path(key), data, 0600); err != nil { + err := os.WriteFile(c.path(key), data, 0600) + if err != nil { return fmt.Errorf("writing blob to cache: %w", err) } @@ -163,14 +170,19 @@ func (c *blobDiskCache) PutFromReader(key string, r io.Reader) (int64, error) { if err != nil { return 0, fmt.Errorf("creating cache file: %w", err) } + written, copyErr := io.Copy(f, r) closeErr := f.Close() + if copyErr != nil { _ = os.Remove(c.path(key)) + return written, fmt.Errorf("streaming to cache file: %w", copyErr) } + if closeErr != nil { _ = os.Remove(c.path(key)) + return written, fmt.Errorf("closing cache file: %w", closeErr) } @@ -182,6 +194,7 @@ func (c *blobDiskCache) PutFromReader(key string, r io.Reader) (int64, error) { // so this branch is effectively unreachable there. if written > c.maxBytes { _ = os.Remove(c.path(key)) + return written, nil } @@ -205,11 +218,14 @@ func (c *blobDiskCache) PutFromReader(key string, r io.Reader) (int64, error) { func (c *blobDiskCache) Get(key string) ([]byte, bool) { c.mu.Lock() c.getCalls++ + e, ok := c.items[key] if !ok { c.mu.Unlock() + return nil, false } + c.unlink(e) c.pushFront(e) c.mu.Unlock() @@ -223,8 +239,10 @@ func (c *blobDiskCache) Get(key string) ([]byte, bool) { c.curBytes -= e.size } c.mu.Unlock() + return nil, false } + return data, true } @@ -232,15 +250,20 @@ func (c *blobDiskCache) Get(key string) ([]byte, bool) { func (c *blobDiskCache) ReadAt(key string, offset, length int64) ([]byte, error) { c.mu.Lock() c.readAtCalls++ + e, ok := c.items[key] if !ok { c.mu.Unlock() + return nil, fmt.Errorf("key %q not in cache", key) } + if offset+length > e.size { c.mu.Unlock() + return nil, fmt.Errorf("read beyond blob size: offset=%d length=%d size=%d", offset, length, e.size) } + c.unlink(e) c.pushFront(e) c.mu.Unlock() @@ -255,6 +278,7 @@ func (c *blobDiskCache) ReadAt(key string, offset, length int64) ([]byte, error) if _, err := f.ReadAt(buf, offset); err != nil { return nil, err } + return buf, nil } @@ -262,7 +286,9 @@ func (c *blobDiskCache) ReadAt(key string, offset, length int64) ([]byte, error) func (c *blobDiskCache) Has(key string) bool { c.mu.Lock() defer c.mu.Unlock() + _, ok := c.items[key] + return ok } @@ -272,10 +298,12 @@ func (c *blobDiskCache) Has(key string) bool { func (c *blobDiskCache) Delete(key string) { c.mu.Lock() defer c.mu.Unlock() + e, ok := c.items[key] if !ok { return } + c.unlink(e) delete(c.items, key) c.curBytes -= e.size @@ -287,10 +315,12 @@ func (c *blobDiskCache) Delete(key string) { func (c *blobDiskCache) Keys() []string { c.mu.Lock() defer c.mu.Unlock() + keys := make([]string, 0, len(c.items)) for k := range c.items { keys = append(keys, k) } + return keys } @@ -298,6 +328,7 @@ func (c *blobDiskCache) Keys() []string { func (c *blobDiskCache) Size() int64 { c.mu.Lock() defer c.mu.Unlock() + return c.curBytes } @@ -305,6 +336,7 @@ func (c *blobDiskCache) Size() int64 { func (c *blobDiskCache) Len() int { c.mu.Lock() defer c.mu.Unlock() + return len(c.items) } @@ -312,6 +344,7 @@ func (c *blobDiskCache) Len() int { func (c *blobDiskCache) GetCalls() int { c.mu.Lock() defer c.mu.Unlock() + return c.getCalls } @@ -319,6 +352,7 @@ func (c *blobDiskCache) GetCalls() int { func (c *blobDiskCache) ReadAtCalls() int { c.mu.Lock() defer c.mu.Unlock() + return c.readAtCalls } @@ -327,6 +361,7 @@ func (c *blobDiskCache) ReadAtCalls() int { func (c *blobDiskCache) PeakLen() int { c.mu.Lock() defer c.mu.Unlock() + return c.peakLen } @@ -334,9 +369,11 @@ func (c *blobDiskCache) PeakLen() int { func (c *blobDiskCache) Close() error { c.mu.Lock() defer c.mu.Unlock() + c.items = nil c.head = nil c.tail = nil c.curBytes = 0 + return os.RemoveAll(c.dir) } diff --git a/internal/vaultik/blobcache_test.go b/internal/vaultik/blobcache_test.go index 778aadd..7467eff 100644 --- a/internal/vaultik/blobcache_test.go +++ b/internal/vaultik/blobcache_test.go @@ -23,6 +23,7 @@ func TestBlobDiskCache_BasicGetPut(t *testing.T) { if !ok { t.Fatal("expected cache hit") } + if !bytes.Equal(got, data) { t.Fatalf("got %q, want %q", got, data) } @@ -35,15 +36,19 @@ func TestBlobDiskCache_BasicGetPut(t *testing.T) { func TestBlobDiskCache_EvictionUnderPressure(t *testing.T) { maxBytes := int64(1000) + cache, err := newBlobDiskCache(maxBytes) if err != nil { t.Fatal(err) } + defer func() { _ = cache.Close() }() - for i := 0; i < 5; i++ { + for i := range 5 { data := make([]byte, 300) - if err := cache.Put(fmt.Sprintf("key%d", i), data); err != nil { + + err := cache.Put(fmt.Sprintf("key%d", i), data) + if err != nil { t.Fatal(err) } } @@ -55,6 +60,7 @@ func TestBlobDiskCache_EvictionUnderPressure(t *testing.T) { if !cache.Has("key4") { t.Fatal("expected key4 to be cached") } + if cache.Has("key0") { t.Fatal("expected key0 to be evicted") } @@ -87,6 +93,7 @@ func TestBlobDiskCache_UpdateInPlace(t *testing.T) { if err := cache.Put("key1", []byte("v1")); err != nil { t.Fatal(err) } + if err := cache.Put("key1", []byte("version2")); err != nil { t.Fatal(err) } @@ -95,12 +102,15 @@ func TestBlobDiskCache_UpdateInPlace(t *testing.T) { if !ok { t.Fatal("expected hit") } + if string(got) != "version2" { t.Fatalf("got %q, want %q", got, "version2") } + if cache.Len() != 1 { t.Fatalf("expected 1 entry, got %d", cache.Len()) } + if cache.Size() != int64(len("version2")) { t.Fatalf("expected size %d, got %d", len("version2"), cache.Size()) } @@ -117,6 +127,7 @@ func TestBlobDiskCache_ReadAt(t *testing.T) { if _, err := rand.Read(data); err != nil { t.Fatal(err) } + if err := cache.Put("blob1", data); err != nil { t.Fatal(err) } @@ -125,6 +136,7 @@ func TestBlobDiskCache_ReadAt(t *testing.T) { if err != nil { t.Fatal(err) } + if !bytes.Equal(chunk, data[100:300]) { t.Fatal("ReadAt returned wrong data") } @@ -149,6 +161,7 @@ func TestBlobDiskCache_Close(t *testing.T) { if err := cache.Put("key1", []byte("data")); err != nil { t.Fatal(err) } + if err := cache.Close(); err != nil { t.Fatal(err) } @@ -165,6 +178,7 @@ func TestBlobDiskCache_LRUOrder(t *testing.T) { if err := cache.Put("a", d); err != nil { t.Fatal(err) } + if err := cache.Put("b", d); err != nil { t.Fatal(err) } @@ -180,9 +194,11 @@ func TestBlobDiskCache_LRUOrder(t *testing.T) { if !cache.Has("a") { t.Fatal("expected 'a' to survive") } + if !cache.Has("c") { t.Fatal("expected 'c' to be present") } + if cache.Has("b") { t.Fatal("expected 'b' to be evicted") } diff --git a/internal/vaultik/helpers.go b/internal/vaultik/helpers.go index deb0d3b..b10e161 100644 --- a/internal/vaultik/helpers.go +++ b/internal/vaultik/helpers.go @@ -1,6 +1,7 @@ package vaultik import ( + "errors" "fmt" "regexp" "strconv" @@ -29,11 +30,13 @@ func formatBytes(bytes int64) string { if bytes < unit { return fmt.Sprintf("%d B", bytes) } + div, exp := int64(unit), 0 for n := bytes / unit; n >= unit; n /= unit { div *= unit exp++ } + return fmt.Sprintf("%.1f %cB", float64(bytes)/float64(div), "KMGTPE"[exp]) } @@ -42,11 +45,12 @@ func formatBytes(bytes int64) string { func parseSnapshotTimestamp(snapshotID string) (time.Time, error) { parts := strings.Split(snapshotID, "_") if len(parts) < 2 { - return time.Time{}, fmt.Errorf("invalid snapshot ID format: expected hostname_snapshotname_timestamp") + return time.Time{}, errors.New("invalid snapshot ID format: expected hostname_snapshotname_timestamp") } // Last part is the RFC3339 timestamp timestampStr := parts[len(parts)-1] + timestamp, err := time.Parse(time.RFC3339, timestampStr) if err != nil { return time.Time{}, fmt.Errorf("invalid timestamp: %w", err) @@ -80,17 +84,20 @@ func parseDuration(s string) (time.Duration, error) { } re := regexp.MustCompile(`(\d+)\s*([a-zA-Z]+)`) + matches := re.FindAllStringSubmatch(s, -1) if len(matches) == 0 { return 0, fmt.Errorf("invalid duration: %q", s) } var total time.Duration + for _, match := range matches { n, err := strconv.Atoi(match[1]) if err != nil { return 0, fmt.Errorf("invalid number %q: %w", match[1], err) } + unit := strings.ToLower(match[2]) switch unit { case "d", "day", "days": @@ -105,5 +112,6 @@ func parseDuration(s string) (time.Duration, error) { return 0, fmt.Errorf("unknown time unit %q", unit) } } + return total, nil } diff --git a/internal/vaultik/helpers_test.go b/internal/vaultik/helpers_test.go index 76a3ea3..e648bec 100644 --- a/internal/vaultik/helpers_test.go +++ b/internal/vaultik/helpers_test.go @@ -61,11 +61,14 @@ func TestParseDuration(t *testing.T) { if err == nil { t.Fatalf("expected error for %q, got %v", tt.input, got) } + return } + if err != nil { t.Fatalf("unexpected error for %q: %v", tt.input, err) } + if got != tt.want { t.Errorf("parseDuration(%q) = %v, want %v", tt.input, got, tt.want) } diff --git a/internal/vaultik/info.go b/internal/vaultik/info.go index ac5f76f..2046bf2 100644 --- a/internal/vaultik/info.go +++ b/internal/vaultik/info.go @@ -15,102 +15,122 @@ import ( // ShowInfo displays system and configuration information func (v *Vaultik) ShowInfo() error { // System Information - v.printfStdout("=== System Information ===\n") - v.printfStdout("OS/Architecture: %s/%s\n", runtime.GOOS, runtime.GOARCH) - v.printfStdout("Version: %s\n", v.Globals.Version) - v.printfStdout("Commit: %s\n", v.Globals.Commit) - v.printfStdout("Go Version: %s\n", runtime.Version()) + v.stdoutf("=== System Information ===\n") + v.stdoutf("OS/Architecture: %s/%s\n", runtime.GOOS, runtime.GOARCH) + v.stdoutf("Version: %s\n", v.Globals.Version) + v.stdoutf("Commit: %s\n", v.Globals.Commit) + v.stdoutf("Go Version: %s\n", runtime.Version()) v.printlnStdout() // Storage Configuration. The backend is selected by storage_url // (s3://, file://, rclone://); the legacy s3.* fields are only // printed when they're actually populated, since the URL scheme // is the primary configuration. - v.printfStdout("=== Storage Configuration ===\n") + v.stdoutf("=== Storage Configuration ===\n") storageInfo := v.Storage.Info() - v.printfStdout("Type: %s\n", storageInfo.Type) - v.printfStdout("Location: %s\n", storageInfo.Location) + v.stdoutf("Type: %s\n", storageInfo.Type) + v.stdoutf("Location: %s\n", storageInfo.Location) + if v.Config.StorageURL != "" { - v.printfStdout("Storage URL: %s\n", v.Config.StorageURL) + v.stdoutf("Storage URL: %s\n", v.Config.StorageURL) } + if v.Config.S3.Bucket != "" { - v.printfStdout("S3 Bucket: %s\n", v.Config.S3.Bucket) + v.stdoutf("S3 Bucket: %s\n", v.Config.S3.Bucket) } + if v.Config.S3.Prefix != "" { - v.printfStdout("S3 Prefix: %s\n", v.Config.S3.Prefix) + v.stdoutf("S3 Prefix: %s\n", v.Config.S3.Prefix) } + if v.Config.S3.Endpoint != "" { - v.printfStdout("S3 Endpoint: %s\n", v.Config.S3.Endpoint) + v.stdoutf("S3 Endpoint: %s\n", v.Config.S3.Endpoint) } + if v.Config.S3.Region != "" { - v.printfStdout("S3 Region: %s\n", v.Config.S3.Region) + v.stdoutf("S3 Region: %s\n", v.Config.S3.Region) } + v.printlnStdout() // Backup Settings - v.printfStdout("=== Backup Settings ===\n") + v.stdoutf("=== Backup Settings ===\n") // Show configured snapshots - v.printfStdout("Snapshots:\n") + v.stdoutf("Snapshots:\n") + for _, name := range v.Config.SnapshotNames() { snap := v.Config.Snapshots[name] - v.printfStdout(" %s:\n", name) + v.stdoutf(" %s:\n", name) + for _, path := range snap.Paths { - v.printfStdout(" - %s\n", path) + v.stdoutf(" - %s\n", path) } + if len(snap.Exclude) > 0 { - v.printfStdout(" exclude: %s\n", strings.Join(snap.Exclude, ", ")) + v.stdoutf(" exclude: %s\n", strings.Join(snap.Exclude, ", ")) } } // Global exclude patterns if len(v.Config.Exclude) > 0 { - v.printfStdout("Global Exclude: %s\n", strings.Join(v.Config.Exclude, ", ")) + v.stdoutf("Global Exclude: %s\n", strings.Join(v.Config.Exclude, ", ")) } - v.printfStdout("Compression: zstd level %d\n", v.Config.CompressionLevel) - v.printfStdout("Chunk Size: %s\n", humanize.Bytes(uint64(v.Config.ChunkSize))) - v.printfStdout("Blob Size Limit: %s\n", humanize.Bytes(uint64(v.Config.BlobSizeLimit))) + v.stdoutf("Compression: zstd level %d\n", v.Config.CompressionLevel) + v.stdoutf("Chunk Size: %s\n", humanize.Bytes(uint64(v.Config.ChunkSize))) + v.stdoutf("Blob Size Limit: %s\n", humanize.Bytes(uint64(v.Config.BlobSizeLimit))) v.printlnStdout() // Encryption Configuration - v.printfStdout("=== Encryption Configuration ===\n") - v.printfStdout("Recipients:\n") + v.stdoutf("=== Encryption Configuration ===\n") + v.stdoutf("Recipients:\n") + for _, recipient := range v.Config.AgeRecipients { - v.printfStdout(" - %s\n", recipient) + v.stdoutf(" - %s\n", recipient) } + v.printlnStdout() // Local Database - v.printfStdout("=== Local Database ===\n") - v.printfStdout("Index Path: %s\n", v.Config.IndexPath) + v.stdoutf("=== Local Database ===\n") + v.stdoutf("Index Path: %s\n", v.Config.IndexPath) // Check if index file exists and get its size if info, err := v.Fs.Stat(v.Config.IndexPath); err == nil { - v.printfStdout("Index Size: %s\n", humanize.Bytes(uint64(info.Size()))) + v.stdoutf("Index Size: %s\n", humanize.Bytes(uint64(info.Size()))) // Get snapshot count from database query := `SELECT COUNT(*) FROM snapshots WHERE completed_at IS NOT NULL` + var snapshotCount int - if err := v.DB.Conn().QueryRowContext(v.ctx, query).Scan(&snapshotCount); err == nil { - v.printfStdout("Snapshots: %d\n", snapshotCount) + + err := v.DB.Conn().QueryRowContext(v.ctx, query).Scan(&snapshotCount) + if err == nil { + v.stdoutf("Snapshots: %d\n", snapshotCount) } // Get blob count from database query = `SELECT COUNT(*) FROM blobs` + var blobCount int - if err := v.DB.Conn().QueryRowContext(v.ctx, query).Scan(&blobCount); err == nil { - v.printfStdout("Blobs: %d\n", blobCount) + + err = v.DB.Conn().QueryRowContext(v.ctx, query).Scan(&blobCount) + if err == nil { + v.stdoutf("Blobs: %d\n", blobCount) } // Get file count from database query = `SELECT COUNT(*) FROM files` + var fileCount int - if err := v.DB.Conn().QueryRowContext(v.ctx, query).Scan(&fileCount); err == nil { - v.printfStdout("Files: %d\n", fileCount) + + err = v.DB.Conn().QueryRowContext(v.ctx, query).Scan(&fileCount) + if err == nil { + v.stdoutf("Files: %d\n", fileCount) } } else { - v.printfStdout("Index Size: (not created)\n") + v.stdoutf("Index Size: (not created)\n") } return nil @@ -153,6 +173,7 @@ type RemoteInfoResult struct { // RemoteInfo displays information about remote storage func (v *Vaultik) RemoteInfo(jsonOutput bool) error { log.Info("Starting remote storage info gathering") + result := &RemoteInfoResult{} storageInfo := v.Storage.Info() @@ -160,11 +181,11 @@ func (v *Vaultik) RemoteInfo(jsonOutput bool) error { result.StorageLocation = storageInfo.Location if !jsonOutput { - v.printfStdout("=== Remote Storage ===\n") - v.printfStdout("Type: %s\n", storageInfo.Type) - v.printfStdout("Location: %s\n", storageInfo.Location) + v.stdoutf("=== Remote Storage ===\n") + v.stdoutf("Type: %s\n", storageInfo.Type) + v.stdoutf("Location: %s\n", storageInfo.Location) v.printlnStdout() - v.printfStdout("Scanning snapshot metadata...\n") + v.stdoutf("Scanning snapshot metadata...\n") } snapshotMetadata, snapshotIDs, err := v.collectSnapshotMetadata() @@ -173,7 +194,7 @@ func (v *Vaultik) RemoteInfo(jsonOutput bool) error { } if !jsonOutput { - v.printfStdout("Downloading %d manifest(s)...\n", len(snapshotIDs)) + v.stdoutf("Downloading %d manifest(s)...\n", len(snapshotIDs)) } referencedBlobs := v.collectReferencedBlobsFromManifests(snapshotIDs, snapshotMetadata) @@ -193,10 +214,12 @@ func (v *Vaultik) RemoteInfo(jsonOutput bool) error { if jsonOutput { enc := json.NewEncoder(v.Stdout) enc.SetIndent("", " ") + return enc.Encode(result) } v.printRemoteInfoTable(result) + return nil } @@ -214,6 +237,7 @@ func (v *Vaultik) collectSnapshotMetadata() (map[string]*SnapshotMetadataInfo, [ if len(parts) < 3 { continue } + snapshotID := parts[1] if _, exists := snapshotMetadata[snapshotID]; !exists { @@ -221,12 +245,14 @@ func (v *Vaultik) collectSnapshotMetadata() (map[string]*SnapshotMetadataInfo, [ } info := snapshotMetadata[snapshotID] + filename := parts[2] if strings.HasPrefix(filename, "manifest") { info.ManifestSize = obj.Size } else if strings.HasPrefix(filename, "db") { info.DatabaseSize = obj.Size } + info.TotalSize = info.ManifestSize + info.DatabaseSize } @@ -234,6 +260,7 @@ func (v *Vaultik) collectSnapshotMetadata() (map[string]*SnapshotMetadataInfo, [ for id := range snapshotMetadata { snapshotIDs = append(snapshotIDs, id) } + sort.Strings(snapshotIDs) return snapshotMetadata, snapshotIDs, nil @@ -245,26 +272,33 @@ func (v *Vaultik) collectReferencedBlobsFromManifests(snapshotIDs []string, snap for _, snapshotID := range snapshotIDs { manifestKey := fmt.Sprintf("metadata/%s/manifest.json.zst", snapshotID) + reader, err := v.Storage.Get(v.ctx, manifestKey) if err != nil { log.Warn("Failed to get manifest", "snapshot", snapshotID, "error", err) + continue } manifest, err := snapshot.DecodeManifest(reader) _ = reader.Close() + if err != nil { log.Warn("Failed to decode manifest", "snapshot", snapshotID, "error", err) + continue } info := snapshotMetadata[snapshotID] info.BlobCount = manifest.BlobCount + var blobsSize int64 + for _, blob := range manifest.Blobs { referencedBlobs[blob.Hash] = blob.CompressedSize blobsSize += blob.CompressedSize } + info.BlobsSize = blobsSize } @@ -274,11 +308,13 @@ func (v *Vaultik) collectReferencedBlobsFromManifests(snapshotIDs []string, snap // populateRemoteInfoResult fills in the result's snapshot and referenced blob stats func (v *Vaultik) populateRemoteInfoResult(result *RemoteInfoResult, snapshotMetadata map[string]*SnapshotMetadataInfo, snapshotIDs []string, referencedBlobs map[string]int64) { var totalMetadataSize int64 + for _, id := range snapshotIDs { info := snapshotMetadata[id] result.Snapshots = append(result.Snapshots, *info) totalMetadataSize += info.TotalSize } + result.TotalMetadataSize = totalMetadataSize result.TotalMetadataCount = len(snapshotIDs) @@ -291,7 +327,7 @@ func (v *Vaultik) populateRemoteInfoResult(result *RemoteInfoResult, snapshotMet // scanRemoteBlobStorage lists all blobs on remote and computes orphan stats func (v *Vaultik) scanRemoteBlobStorage(result *RemoteInfoResult, referencedBlobs map[string]int64, jsonOutput bool) error { if !jsonOutput { - v.printfStdout("Scanning blobs...\n") + v.stdoutf("Scanning blobs...\n") } blobCh := v.Storage.ListStream(v.ctx, "blobs/") @@ -301,10 +337,12 @@ func (v *Vaultik) scanRemoteBlobStorage(result *RemoteInfoResult, referencedBlob if obj.Err != nil { return fmt.Errorf("listing blobs: %w", obj.Err) } + parts := strings.Split(obj.Key, "/") if len(parts) < 4 { continue } + hash := parts[3] allBlobs[hash] = obj.Size result.TotalBlobCount++ @@ -323,14 +361,16 @@ func (v *Vaultik) scanRemoteBlobStorage(result *RemoteInfoResult, referencedBlob // printRemoteInfoTable renders the human-readable remote info output func (v *Vaultik) printRemoteInfoTable(result *RemoteInfoResult) { - v.printfStdout("\n=== Snapshot Metadata ===\n") + v.stdoutf("\n=== Snapshot Metadata ===\n") + if len(result.Snapshots) == 0 { - v.printfStdout("No snapshots found\n") + v.stdoutf("No snapshots found\n") } else { - v.printfStdout("%-45s %12s %12s %12s %10s %12s\n", "SNAPSHOT", "MANIFEST", "DATABASE", "TOTAL", "BLOBS", "BLOB SIZE") - v.printfStdout("%-45s %12s %12s %12s %10s %12s\n", strings.Repeat("-", 45), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 10), strings.Repeat("-", 12)) + v.stdoutf("%-45s %12s %12s %12s %10s %12s\n", "SNAPSHOT", "MANIFEST", "DATABASE", "TOTAL", "BLOBS", "BLOB SIZE") + v.stdoutf("%-45s %12s %12s %12s %10s %12s\n", strings.Repeat("-", 45), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 10), strings.Repeat("-", 12)) + for _, info := range result.Snapshots { - v.printfStdout("%-45s %12s %12s %12s %10s %12s\n", + v.stdoutf("%-45s %12s %12s %12s %10s %12s\n", truncateString(info.SnapshotID, 45), humanize.Bytes(uint64(info.ManifestSize)), humanize.Bytes(uint64(info.DatabaseSize)), @@ -339,20 +379,21 @@ func (v *Vaultik) printRemoteInfoTable(result *RemoteInfoResult) { humanize.Bytes(uint64(info.BlobsSize)), ) } - v.printfStdout("%-45s %12s %12s %12s %10s %12s\n", strings.Repeat("-", 45), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 10), strings.Repeat("-", 12)) - v.printfStdout("%-45s %12s %12s %12s\n", fmt.Sprintf("Total (%d snapshots)", result.TotalMetadataCount), "", "", humanize.Bytes(uint64(result.TotalMetadataSize))) + + v.stdoutf("%-45s %12s %12s %12s %10s %12s\n", strings.Repeat("-", 45), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 10), strings.Repeat("-", 12)) + v.stdoutf("%-45s %12s %12s %12s\n", fmt.Sprintf("Total (%d snapshots)", result.TotalMetadataCount), "", "", humanize.Bytes(uint64(result.TotalMetadataSize))) } - v.printfStdout("\n=== Blob Storage ===\n") - v.printfStdout("Total blobs on remote: %s (%s)\n", + v.stdoutf("\n=== Blob Storage ===\n") + v.stdoutf("Total blobs on remote: %s (%s)\n", humanize.Comma(int64(result.TotalBlobCount)), humanize.Bytes(uint64(result.TotalBlobSize))) - v.printfStdout("Referenced by snapshots: %s (%s)\n", + v.stdoutf("Referenced by snapshots: %s (%s)\n", humanize.Comma(int64(result.ReferencedBlobCount)), humanize.Bytes(uint64(result.ReferencedBlobSize))) - v.printfStdout("Orphaned (unreferenced): %s (%s)\n", + v.stdoutf("Orphaned (unreferenced): %s (%s)\n", humanize.Comma(int64(result.OrphanedBlobCount)), humanize.Bytes(uint64(result.OrphanedBlobSize))) if result.OrphanedBlobCount > 0 { - v.printfStdout("\nRun 'vaultik prune' to remove orphaned blobs.\n") + v.stdoutf("\nRun 'vaultik prune' to remove orphaned blobs.\n") } } @@ -361,8 +402,10 @@ func truncateString(s string, maxLen int) string { if len(s) <= maxLen { return s } + if maxLen <= 3 { return s[:maxLen] } + return s[:maxLen-3] + "..." } diff --git a/internal/vaultik/integration_test.go b/internal/vaultik/integration_test.go index 945d448..78a1c18 100644 --- a/internal/vaultik/integration_test.go +++ b/internal/vaultik/integration_test.go @@ -43,11 +43,14 @@ func (m *MockStorer) Put(ctx context.Context, key string, reader io.Reader) erro defer m.mu.Unlock() m.calls = append(m.calls, "Put:"+key) + data, err := io.ReadAll(reader) if err != nil { return err } + m.data[key] = data + return nil } @@ -60,10 +63,12 @@ func (m *MockStorer) Get(ctx context.Context, key string) (io.ReadCloser, error) defer m.mu.Unlock() m.calls = append(m.calls, "Get:"+key) + data, exists := m.data[key] if !exists { return nil, storage.ErrNotFound } + return io.NopCloser(bytes.NewReader(data)), nil } @@ -72,10 +77,12 @@ func (m *MockStorer) Stat(ctx context.Context, key string) (*storage.ObjectInfo, defer m.mu.Unlock() m.calls = append(m.calls, "Stat:"+key) + data, exists := m.data[key] if !exists { return nil, storage.ErrNotFound } + return &storage.ObjectInfo{ Key: key, Size: int64(len(data)), @@ -88,6 +95,7 @@ func (m *MockStorer) Delete(ctx context.Context, key string) error { m.calls = append(m.calls, "Delete:"+key) delete(m.data, key) + return nil } @@ -96,12 +104,15 @@ func (m *MockStorer) List(ctx context.Context, prefix string) ([]string, error) defer m.mu.Unlock() m.calls = append(m.calls, "List:"+prefix) + var keys []string + for key := range m.data { if len(prefix) == 0 || (len(key) >= len(prefix) && key[:len(prefix)] == prefix) { keys = append(keys, key) } } + return keys, nil } @@ -109,6 +120,7 @@ func (m *MockStorer) ListStream(ctx context.Context, prefix string) <-chan stora ch := make(chan storage.ObjectInfo) go func() { defer close(ch) + m.mu.Lock() defer m.mu.Unlock() @@ -121,6 +133,7 @@ func (m *MockStorer) ListStream(ctx context.Context, prefix string) <-chan stora } } }() + return ch } @@ -138,6 +151,7 @@ func (m *MockStorer) GetCalls() []string { calls := make([]string, len(m.calls)) copy(calls, m.calls) + return calls } @@ -172,14 +186,16 @@ func TestEndToEndBackup(t *testing.T) { "/home/user/code", } for _, dir := range dirs { - if err := fs.MkdirAll(dir, 0755); err != nil { + err := fs.MkdirAll(dir, 0755) + if err != nil { t.Fatalf("failed to create directory %s: %v", dir, err) } } // Create test files for path, content := range testFiles { - if err := afero.WriteFile(fs, path, []byte(content), 0644); err != nil { + err := afero.WriteFile(fs, path, []byte(content), 0644) + if err != nil { t.Fatalf("failed to create test file %s: %v", path, err) } } @@ -216,9 +232,11 @@ func TestEndToEndBackup(t *testing.T) { // Create in-memory database db, err := database.New(ctx, ":memory:") + require.NoError(t, err) defer func() { - if err := db.Close(); err != nil { + err := db.Close() + if err != nil { t.Errorf("failed to close database: %v", err) } }() @@ -246,6 +264,7 @@ func TestEndToEndBackup(t *testing.T) { VaultikVersion: "test-version", StartedAt: time.Now(), } + return repos.Snapshots.Create(ctx, tx, snapshot) }) require.NoError(t, err) @@ -258,9 +277,9 @@ func TestEndToEndBackup(t *testing.T) { // The scanner counts both files and directories, so we have: // 4 files + 4 directories (/home, /home/user, /home/user/documents, /home/user/pictures, /home/user/code) assert.GreaterOrEqual(t, result.FilesScanned, 4, "Should scan at least 4 files") - assert.Greater(t, result.BytesScanned, int64(0), "Should scan some bytes") - assert.Greater(t, result.ChunksCreated, 0, "Should create chunks") - assert.Greater(t, result.BlobsCreated, 0, "Should create blobs") + assert.Positive(t, result.BytesScanned, "Should scan some bytes") + assert.Positive(t, result.ChunksCreated, "Should create chunks") + assert.Positive(t, result.BlobsCreated, "Should create blobs") // Verify storage operations calls := mockStorage.GetCalls() @@ -268,6 +287,7 @@ func TestEndToEndBackup(t *testing.T) { // Should have uploaded at least one blob blobUploads := 0 + for _, call := range calls { if len(call) > 4 && call[:4] == "Put:" { if len(call) > 10 && call[4:10] == "blobs/" { @@ -275,27 +295,30 @@ func TestEndToEndBackup(t *testing.T) { } } } - assert.Greater(t, blobUploads, 0, "Should upload at least one blob") + + assert.Positive(t, blobUploads, "Should upload at least one blob") // Verify files in database files, err := repos.Files.ListByPrefix(ctx, "/home/user") require.NoError(t, err) // Count only regular files (not directories) regularFiles := 0 + for _, f := range files { if f.Mode&0x80000000 == 0 { // Check if regular file (not directory) regularFiles++ } } + assert.Equal(t, 4, regularFiles, "Should have 4 regular files in database") // Verify chunks were created by checking a specific file fileChunks, err := repos.FileChunks.GetByPath(ctx, "/home/user/documents/file1.txt") require.NoError(t, err) - assert.Greater(t, len(fileChunks), 0, "Should have chunks for file1.txt") + assert.NotEmpty(t, fileChunks, "Should have chunks for file1.txt") // Verify blobs were uploaded to storage - assert.Greater(t, mockStorage.GetStorageSize(), 0, "Should have blobs in storage") + assert.Positive(t, mockStorage.GetStorageSize(), "Should have blobs in storage") // Complete the snapshot - just verify we got results // In a real integration test, we'd update the snapshot record @@ -337,9 +360,11 @@ func TestBackupAndVerify(t *testing.T) { // Create test database ctx := context.Background() db, err := database.New(ctx, ":memory:") + require.NoError(t, err) defer func() { - if err := db.Close(); err != nil { + err := db.Close() + if err != nil { t.Errorf("failed to close database: %v", err) } }() @@ -366,6 +391,7 @@ func TestBackupAndVerify(t *testing.T) { VaultikVersion: "test-version", StartedAt: time.Now(), } + return repos.Snapshots.Create(ctx, tx, snapshot) }) require.NoError(t, err) @@ -375,7 +401,7 @@ func TestBackupAndVerify(t *testing.T) { require.NoError(t, err) // Verify backup created blobs - assert.Greater(t, result.BlobsCreated, 0, "Should create at least one blob") + assert.Positive(t, result.BlobsCreated, "Should create at least one blob") assert.Equal(t, mockStorage.GetStorageSize(), result.BlobsCreated, "Storage should have the blobs") // Verify we can retrieve the blob from storage @@ -391,18 +417,19 @@ func TestBackupAndVerify(t *testing.T) { // Get blob info blobInfo, err := mockStorage.Stat(ctx, blobKey) require.NoError(t, err) - assert.Greater(t, blobInfo.Size, int64(0), "Blob should have content") + assert.Positive(t, blobInfo.Size, "Blob should have content") // Get blob content reader, err := mockStorage.Get(ctx, blobKey) require.NoError(t, err) + defer func() { _ = reader.Close() }() // Verify blob data is encrypted (should not contain plaintext) blobData, err := io.ReadAll(reader) require.NoError(t, err) assert.NotContains(t, string(blobData), testContent, "Blob should be encrypted") - assert.Greater(t, len(blobData), 0, "Blob should have data") + assert.NotEmpty(t, blobData, "Blob should have data") } t.Logf("Backup and verify test completed successfully") @@ -418,6 +445,7 @@ func TestBackupAndRestore(t *testing.T) { // Create real temp directory for the database (SQLite needs real filesystem) realTempDir, err := os.MkdirTemp("", "vaultik-test-") require.NoError(t, err) + defer func() { _ = os.RemoveAll(realTempDir) }() // Use real OS filesystem for this test @@ -434,10 +462,14 @@ func TestBackupAndRestore(t *testing.T) { // Create directories and files for path, content := range testFiles { dir := filepath.Dir(path) - if err := fs.MkdirAll(dir, 0755); err != nil { + + err := fs.MkdirAll(dir, 0755) + if err != nil { t.Fatalf("failed to create directory %s: %v", dir, err) } - if err := afero.WriteFile(fs, path, []byte(content), 0644); err != nil { + + err = afero.WriteFile(fs, path, []byte(content), 0644) + if err != nil { t.Fatalf("failed to create test file %s: %v", path, err) } } @@ -455,6 +487,7 @@ func TestBackupAndRestore(t *testing.T) { dbPath := filepath.Join(realTempDir, "test.db") db, err := database.New(ctx, dbPath) require.NoError(t, err) + defer func() { _ = db.Close() }() repos := database.NewRepositories(db) @@ -558,6 +591,7 @@ func TestEndToEndFileStorage(t *testing.T) { fs := afero.NewOsFs() tempDir, err := os.MkdirTemp("", "vaultik-e2e-") require.NoError(t, err) + defer func() { _ = os.RemoveAll(tempDir) }() dataDir := filepath.Join(tempDir, "source") @@ -618,6 +652,7 @@ func TestEndToEndFileStorage(t *testing.T) { db, err := database.New(ctx, dbPath) require.NoError(t, err) + defer func() { _ = db.Close() }() repos := database.NewRepositories(db) @@ -644,8 +679,8 @@ func TestEndToEndFileStorage(t *testing.T) { scanResult, err := scanner.Scan(ctx, dataDir, snapshotID) require.NoError(t, err) - require.Greater(t, scanResult.FilesScanned, 0) - require.Greater(t, scanResult.BlobsCreated, 0) + require.Positive(t, scanResult.FilesScanned) + require.Positive(t, scanResult.BlobsCreated) require.NoError(t, sm.CompleteSnapshot(ctx, snapshotID)) require.NoError(t, sm.ExportSnapshotMetadata(ctx, dbPath, snapshotID)) @@ -656,6 +691,7 @@ func TestEndToEndFileStorage(t *testing.T) { blobInfo, err := os.Stat(filepath.Join(storeDir, "blobs")) require.NoError(t, err) require.True(t, blobInfo.IsDir()) + metaInfo, err := os.Stat(filepath.Join(storeDir, "metadata", snapshot.RemoteSnapshotKey(snapshotID))) require.NoError(t, err) require.True(t, metaInfo.IsDir()) @@ -721,6 +757,7 @@ func TestDedupOnlySnapshotRestores(t *testing.T) { fs := afero.NewOsFs() tempDir, err := os.MkdirTemp("", "vaultik-dedup-") require.NoError(t, err) + defer func() { _ = os.RemoveAll(tempDir) }() dataDir := filepath.Join(tempDir, "source") @@ -756,7 +793,9 @@ func TestDedupOnlySnapshotRestores(t *testing.T) { ctx := context.Background() db, err := database.New(ctx, dbPath) require.NoError(t, err) + defer func() { _ = db.Close() }() + repos := database.NewRepositories(db) makeScanner := func() *snapshot.Scanner { @@ -780,13 +819,14 @@ func TestDedupOnlySnapshotRestores(t *testing.T) { require.NoError(t, err) r1, err := makeScanner().Scan(ctx, dataDir, id1) require.NoError(t, err) - require.Greater(t, r1.BlobsCreated, 0, "first snapshot should upload at least one blob") + require.Positive(t, r1.BlobsCreated, "first snapshot should upload at least one blob") require.NoError(t, sm.CompleteSnapshot(ctx, id1)) require.NoError(t, sm.ExportSnapshotMetadata(ctx, dbPath, id1)) // Second snapshot — same data, every chunk dedups. Sleep past the // second-precision timestamp so the snapshot IDs differ. time.Sleep(1100 * time.Millisecond) + id2, err := sm.CreateSnapshotWithName(ctx, cfg.Hostname, "dedup", "v", "g") require.NoError(t, err) r2, err := makeScanner().Scan(ctx, dataDir, id2) @@ -833,5 +873,6 @@ func bytesPattern(tag string, n int) []byte { for i := range out { out[i] = byte(tag[i%len(tag)] ^ byte(i&0xff)) } + return out } diff --git a/internal/vaultik/prune.go b/internal/vaultik/prune.go index 8b3cfac..67cd06c 100644 --- a/internal/vaultik/prune.go +++ b/internal/vaultik/prune.go @@ -2,6 +2,7 @@ package vaultik import ( "encoding/json" + "errors" "fmt" "strings" @@ -23,20 +24,24 @@ type PruneOptions struct { // confirming with the user. func (v *Vaultik) NukeRemote(force bool) error { if !force { - return fmt.Errorf("nuke requires --force (this deletes ALL remote snapshots and blobs)") + return errors.New("nuke requires --force (this deletes ALL remote snapshots and blobs)") } - v.UI.Begin("Removing all snapshot metadata from backup destination store.") + v.UI.Beginf("Removing all snapshot metadata from backup destination store.") + if _, err := v.RemoveAllSnapshots(&RemoveOptions{Force: true}); err != nil { return fmt.Errorf("removing all snapshots: %w", err) } - v.UI.Begin("Removing any blobs still present in backup destination store.") - if err := v.PruneBlobs(&PruneOptions{Force: true}); err != nil { + v.UI.Beginf("Removing any blobs still present in backup destination store.") + + err := v.PruneBlobs(&PruneOptions{Force: true}) + if err != nil { return fmt.Errorf("pruning blobs: %w", err) } - v.UI.Complete("Backup destination store is now empty.") + v.UI.Completef("Backup destination store is now empty.") + return nil } @@ -55,7 +60,8 @@ type PruneBlobsResult struct { // prefer this method over PruneDatabase or PruneBlobs individually // unless it specifically wants one half. func (v *Vaultik) Prune(opts *PruneOptions) error { - if err := v.EnsureStorageBinding(); err != nil { + err := v.EnsureStorageBinding() + if err != nil { return err } // First reconcile local snapshot records against remote metadata: @@ -63,12 +69,15 @@ func (v *Vaultik) Prune(opts *PruneOptions) error { // store is treated as gone. This used to be the separate 'snapshot // cleanup' command and is now folded in so a single 'vaultik prune' // gets the local index fully back in sync with the destination. - if err := v.CleanupLocalSnapshots(); err != nil { + err = v.CleanupLocalSnapshots() + if err != nil { return fmt.Errorf("reconciling local snapshots with remote: %w", err) } + if _, err := v.PruneDatabase(); err != nil { return fmt.Errorf("pruning local database: %w", err) } + return v.PruneBlobs(opts) } @@ -92,27 +101,35 @@ func (v *Vaultik) PruneBlobs(opts *PruneOptions) error { if len(unreferencedBlobs) == 0 { log.Info("No unreferenced blobs found") + if opts.JSON { return v.outputPruneBlobsJSON(result) } + v.printlnStdout("No unreferenced blobs to remove.") + return nil } log.Info("Found unreferenced blobs", "count", len(unreferencedBlobs), "total_size", humanize.Bytes(uint64(totalSize))) + if !opts.JSON { - v.printfStdout("Found %d unreferenced blob(s) totaling %s\n", len(unreferencedBlobs), humanize.Bytes(uint64(totalSize))) + v.stdoutf("Found %d unreferenced blob(s) totaling %s\n", len(unreferencedBlobs), humanize.Bytes(uint64(totalSize))) } if !opts.Force && !opts.JSON { - v.printfStdout("\nDelete %d unreferenced blob(s)? [y/N] ", len(unreferencedBlobs)) + v.stdoutf("\nDelete %d unreferenced blob(s)? [y/N] ", len(unreferencedBlobs)) + var confirm string if _, err := v.scanStdin(&confirm); err != nil { v.printlnStdout("Cancelled") + return nil } + if strings.ToLower(confirm) != "y" { v.printlnStdout("Cancelled") + return nil } } @@ -123,9 +140,10 @@ func (v *Vaultik) PruneBlobs(opts *PruneOptions) error { return v.outputPruneBlobsJSON(result) } - v.printfStdout("\nDeleted %d blob(s) totaling %s\n", result.BlobsDeleted, humanize.Bytes(uint64(result.BytesFreed))) + v.stdoutf("\nDeleted %d blob(s) totaling %s\n", result.BlobsDeleted, humanize.Bytes(uint64(result.BytesFreed))) + if result.BlobsFailed > 0 { - v.printfStdout("Failed to delete %d blob(s)\n", result.BlobsFailed) + v.stdoutf("Failed to delete %d blob(s)\n", result.BlobsFailed) } return nil @@ -140,6 +158,7 @@ func (v *Vaultik) collectReferencedBlobs() (map[string]bool, error) { if err != nil { return nil, fmt.Errorf("listing snapshot keys: %w", err) } + log.Info("Found manifests in remote storage", "count", len(remoteKeys)) allBlobsReferenced := make(map[string]bool) @@ -147,18 +166,23 @@ func (v *Vaultik) collectReferencedBlobs() (map[string]bool, error) { for _, remoteKey := range remoteKeys { log.Debug("Processing manifest", "remote_key", remoteKey) + manifest, err := v.downloadManifestByKey(remoteKey) if err != nil { log.Error("Failed to download manifest", "remote_key", remoteKey, "error", err) + continue } + for _, blob := range manifest.Blobs { allBlobsReferenced[blob.Hash] = true } + manifestCount++ } log.Info("Processed manifests", "count", manifestCount, "unique_blobs_referenced", len(allBlobsReferenced)) + return allBlobsReferenced, nil } @@ -166,12 +190,14 @@ func (v *Vaultik) collectReferencedBlobs() (map[string]bool, error) { func (v *Vaultik) listUniqueSnapshotIDs() ([]string, error) { objectCh := v.Storage.ListStream(v.ctx, "metadata/") seen := make(map[string]bool) + var snapshotIDs []string for object := range objectCh { if object.Err != nil { return nil, fmt.Errorf("listing metadata objects: %w", object.Err) } + parts := strings.Split(object.Key, "/") if len(parts) >= 2 && parts[0] == "metadata" && parts[1] != "" { if strings.HasSuffix(object.Key, "/") || strings.Contains(object.Key, "/manifest.json.zst") { @@ -183,12 +209,14 @@ func (v *Vaultik) listUniqueSnapshotIDs() ([]string, error) { } } } + return snapshotIDs, nil } // listAllRemoteBlobs returns a map of all blob hashes to their sizes in remote storage func (v *Vaultik) listAllRemoteBlobs() (map[string]int64, error) { log.Info("Listing all blobs in storage") + allBlobs := make(map[string]int64) blobObjectCh := v.Storage.ListStream(v.ctx, "blobs/") @@ -196,6 +224,7 @@ func (v *Vaultik) listAllRemoteBlobs() (map[string]int64, error) { if object.Err != nil { return nil, fmt.Errorf("listing blobs: %w", object.Err) } + parts := strings.Split(object.Key, "/") if len(parts) == 4 && parts[0] == "blobs" { allBlobs[parts[3]] = object.Size @@ -203,19 +232,24 @@ func (v *Vaultik) listAllRemoteBlobs() (map[string]int64, error) { } log.Info("Found blobs in storage", "count", len(allBlobs)) + return allBlobs, nil } // findUnreferencedBlobs returns blob hashes not referenced by any manifest and their total size func (v *Vaultik) findUnreferencedBlobs(allBlobs map[string]int64, referenced map[string]bool) ([]string, int64) { - var unreferenced []string - var totalSize int64 + var ( + unreferenced []string + totalSize int64 + ) + for hash, size := range allBlobs { if !referenced[hash] { unreferenced = append(unreferenced, hash) totalSize += size } } + return unreferenced, totalSize } @@ -226,8 +260,10 @@ func (v *Vaultik) deleteUnreferencedBlobs(unreferencedBlobs []string, allBlobs m for i, hash := range unreferencedBlobs { blobPath := fmt.Sprintf("blobs/%s/%s/%s", hash[:2], hash[2:4], hash) - if err := v.Storage.Delete(v.ctx, blobPath); err != nil { + err := v.Storage.Delete(v.ctx, blobPath) + if err != nil { log.Error("Failed to delete blob", "hash", hash, "error", err) + continue } @@ -256,5 +292,6 @@ func (v *Vaultik) deleteUnreferencedBlobs(unreferencedBlobs []string, allBlobs m func (v *Vaultik) outputPruneBlobsJSON(result *PruneBlobsResult) error { encoder := json.NewEncoder(v.Stdout) encoder.SetIndent("", " ") + return encoder.Encode(result) } diff --git a/internal/vaultik/purge_per_name_test.go b/internal/vaultik/purge_per_name_test.go index ff86be3..b8c4782 100644 --- a/internal/vaultik/purge_per_name_test.go +++ b/internal/vaultik/purge_per_name_test.go @@ -79,16 +79,19 @@ func setupPurgeTest(t *testing.T, snapshotIDs []string) *vaultik.Vaultik { // listRemainingSnapshots returns IDs of all completed snapshots in the database. func listRemainingSnapshots(t *testing.T, v *vaultik.Vaultik) []string { t.Helper() + ctx := context.Background() dbSnaps, err := v.Repositories.Snapshots.ListRecent(ctx, 10000) require.NoError(t, err) var ids []string + for _, s := range dbSnaps { if s.CompletedAt != nil { ids = append(ids, s.ID.String()) } } + return ids } diff --git a/internal/vaultik/remove_snapshot_test.go b/internal/vaultik/remove_snapshot_test.go index 5dd03ec..2bbf33f 100644 --- a/internal/vaultik/remove_snapshot_test.go +++ b/internal/vaultik/remove_snapshot_test.go @@ -37,7 +37,9 @@ func (s *testStorer) Put(ctx context.Context, key string, reader io.Reader) erro if err != nil { return err } + s.data[key] = data + return nil } @@ -53,6 +55,7 @@ func (s *testStorer) Get(ctx context.Context, key string) (io.ReadCloser, error) if !exists { return nil, storage.ErrNotFound } + return io.NopCloser(bytes.NewReader(data)), nil } @@ -64,6 +67,7 @@ func (s *testStorer) Stat(ctx context.Context, key string) (*storage.ObjectInfo, if !exists { return nil, storage.ErrNotFound } + return &storage.ObjectInfo{ Key: key, Size: int64(len(data)), @@ -75,6 +79,7 @@ func (s *testStorer) Delete(ctx context.Context, key string) error { defer s.mu.Unlock() delete(s.data, key) + return nil } @@ -83,11 +88,13 @@ func (s *testStorer) List(ctx context.Context, prefix string) ([]string, error) defer s.mu.Unlock() var keys []string + for key := range s.data { if prefix == "" || strings.HasPrefix(key, prefix) { keys = append(keys, key) } } + return keys, nil } @@ -96,6 +103,7 @@ func (s *testStorer) ListStream(ctx context.Context, prefix string) <-chan stora go func() { defer close(ch) + s.mu.Lock() defer s.mu.Unlock() @@ -115,13 +123,16 @@ func (s *testStorer) ListStream(ctx context.Context, prefix string) <-chan stora func (s *testStorer) hasKey(key string) bool { s.mu.Lock() defer s.mu.Unlock() + _, exists := s.data[key] + return exists } func (s *testStorer) keyCount() int { s.mu.Lock() defer s.mu.Unlock() + return len(s.data) } @@ -175,6 +186,7 @@ func addBlob(t *testing.T, store *testStorer, hash string) { // Create zstd compressed data var buf bytes.Buffer + writer, _ := zstd.NewWriter(&buf) _, _ = writer.Write([]byte("blob data")) _ = writer.Close() @@ -366,7 +378,7 @@ func TestRemoveAllSnapshots_NoSnapshots(t *testing.T) { result, err := tv.RemoveAllSnapshots(opts) require.NoError(t, err) - assert.Len(t, result.SnapshotsRemoved, 0) + assert.Empty(t, result.SnapshotsRemoved) // Verify output assert.Contains(t, tv.Stdout.String(), "No snapshots found") diff --git a/internal/vaultik/restore.go b/internal/vaultik/restore.go index bb7715c..7eebf4f 100644 --- a/internal/vaultik/restore.go +++ b/internal/vaultik/restore.go @@ -5,6 +5,7 @@ import ( "context" "crypto/sha256" "encoding/hex" + "errors" "fmt" "io" "math" @@ -62,16 +63,20 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error { // Step 1: Download and decrypt the snapshot metadata database log.Info("Downloading snapshot metadata...") + tempDB, err := v.downloadSnapshotDB(opts.SnapshotID, identity) if err != nil { return fmt.Errorf("downloading snapshot database: %w", err) } + defer func() { - if err := tempDB.Close(); err != nil { + err := tempDB.Close() + if err != nil { log.Debug("Failed to close temp database", "error", err) } // Clean up temp file - if err := v.Fs.Remove(tempDB.Path()); err != nil { + err = v.Fs.Remove(tempDB.Path()) + if err != nil { log.Debug("Failed to remove temp database", "error", err) } }() @@ -86,12 +91,13 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error { if len(files) == 0 { log.Warn("No files found to restore") - v.UI.Warning("No files found to restore.") + v.UI.Warningf("No files found to restore.") + return nil } log.Info("Found files to restore", "count", len(files)) - v.UI.Info("Found %s files to restore.", v.UI.Count(len(files))) + v.UI.Infof("Found %s files to restore.", v.UI.Count(len(files))) // Step 3: Create target directory if err := v.Fs.MkdirAll(opts.TargetDir, 0755); err != nil { @@ -120,26 +126,28 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error { "duration", result.Duration, ) - v.UI.Complete("Restored %s files (%s) in %s.", + v.UI.Completef("Restored %s files (%s) in %s.", v.UI.Count(result.FilesRestored), v.UI.Size(result.BytesRestored), v.UI.Duration(result.Duration), ) if os.Geteuid() != 0 { - v.UI.Warning("Restore did not preserve file ownership: chown(2) requires root. Re-run as root (e.g. with sudo) if you need original UID/GID preserved.") + v.UI.Warningf("Restore did not preserve file ownership: chown(2) requires root. Re-run as root (e.g. with sudo) if you need original UID/GID preserved.") } if result.FilesFailed > 0 { - v.UI.Warning("%d file(s) failed to restore:", result.FilesFailed) + v.UI.Warningf("%d file(s) failed to restore:", result.FilesFailed) + for _, path := range result.FailedFiles { - v.UI.Detail("%s", v.UI.Path(path)) + v.UI.Detailf("%s", v.UI.Path(path)) } } // Run verification if requested if opts.Verify { - if err := v.handleRestoreVerification(repos, files, opts, result); err != nil { + err := v.handleRestoreVerification(repos, files, opts, result) + if err != nil { return err } } @@ -154,13 +162,14 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error { // prepareRestoreIdentity validates that an age secret key is configured and parses it func (v *Vaultik) prepareRestoreIdentity() (age.Identity, error) { if v.Config.AgeSecretKey == "" { - return nil, fmt.Errorf("decryption key required for restore\n\nSet the VAULTIK_AGE_SECRET_KEY environment variable to your age private key:\n export VAULTIK_AGE_SECRET_KEY='AGE-SECRET-KEY-...'") + return nil, errors.New("decryption key required for restore\n\nSet the VAULTIK_AGE_SECRET_KEY environment variable to your age private key:\n export VAULTIK_AGE_SECRET_KEY='AGE-SECRET-KEY-...'") } identity, err := age.ParseX25519Identity(v.Config.AgeSecretKey) if err != nil { return nil, fmt.Errorf("parsing age secret key: %w", err) } + return identity, nil } @@ -187,13 +196,16 @@ func (v *Vaultik) restoreAllFiles( if err != nil { return nil, fmt.Errorf("creating blob cache: %w", err) } + if v.restoreCacheObserver != nil { v.restoreCacheObserver(blobCache) } + defer func() { if v.restoreCacheObserver != nil { v.restoreCacheObserver(blobCache) } + _ = blobCache.Close() }() @@ -208,7 +220,9 @@ func (v *Vaultik) restoreAllFiles( if err != nil { return nil, fmt.Errorf("fetching blob index: %w", err) } + blobIDToHash := make(map[string]string, len(blobsByID)) + blobByHash := make(map[string]*database.Blob, len(blobsByID)) for id, blob := range blobsByID { hash := blob.Hash.String() @@ -234,7 +248,7 @@ func (v *Vaultik) restoreAllFiles( totalBytesExpected += file.Size } - v.UI.Begin("Restoring %s files (%s) to %s.", + v.UI.Beginf("Restoring %s files (%s) to %s.", v.UI.Count(len(files)), v.UI.Size(totalBytesExpected), v.UI.Path(opts.TargetDir)) @@ -257,9 +271,11 @@ func (v *Vaultik) restoreAllFiles( // Periodic progress output, matching the snapshot create cadence. startTime := time.Now() lastStatusTime := startTime + const statusInterval = 15 * time.Second processed := 0 + for plan.hasPending() { if v.ctx.Err() != nil { return nil, v.ctx.Err() @@ -282,31 +298,44 @@ func (v *Vaultik) restoreAllFiles( if next.IsZero() { break } + for _, hash := range plan.blobsNeeded(next) { blob, ok := blobByHash[hash] if !ok { return nil, fmt.Errorf("blob hash %s missing from blob index", hash[:16]) } - if err := session.downloadBlobToCache(hash, blob.CompressedSize); err != nil { + + err := session.downloadBlobToCache(hash, blob.CompressedSize) + if err != nil { return nil, fmt.Errorf("downloading blob %s: %w", hash[:16], err) } + result.BlobsDownloaded++ result.BytesDownloaded += blob.CompressedSize + plan.markBlobCached(hash) } + continue } file := filesByID[fileID] - if err := session.restoreFile(file); err != nil { + + err := session.restoreFile(file) + if err != nil { log.Error("Failed to restore file", "path", file.Path, "error", err) + if !opts.SkipErrors { return nil, fmt.Errorf("restoring %s: %w (pass --skip-errors to continue past restore failures)", file.Path, err) } - v.UI.Error("Failed to restore %s: %v. Skipping (--skip-errors).", v.UI.Path(file.Path.String()), err) + + v.UI.Errorf("Failed to restore %s: %v. Skipping (--skip-errors).", v.UI.Path(file.Path.String()), err) + result.FilesFailed++ result.FailedFiles = append(result.FailedFiles, file.Path.String()) + plan.finishFile(fileID) + continue } @@ -315,10 +344,12 @@ func (v *Vaultik) restoreAllFiles( // plan's indexes so future picks ignore it. sweeper.fileRestored(fileID.String()) plan.finishFile(fileID) + processed++ if time.Since(lastStatusTime) >= statusInterval { v.printRestoreProgress(processed, len(files), result.BytesRestored, totalBytesExpected, startTime) + lastStatusTime = time.Now() } @@ -344,13 +375,14 @@ func (v *Vaultik) printRestoreProgress(filesDone, totalFiles int, bytesDone, tot fileRate := float64(filesDone) / elapsed.Seconds() remainingBytes := totalBytes - bytesDone + var eta time.Duration if byteRate > 0 && remainingBytes > 0 { eta = time.Duration(float64(remainingBytes)/byteRate) * time.Second } if eta > 0 { - v.UI.Progress("Restore: %s/%s files (%s), %s/%s, %s, %.0f files/sec, restore elapsed: %s, restore ETA: %s (est remain %s).", + v.UI.Progressf("Restore: %s/%s files (%s), %s/%s, %s, %.0f files/sec, restore elapsed: %s, restore ETA: %s (est remain %s).", v.UI.Count(filesDone), v.UI.Count(totalFiles), v.UI.Percent(pct), @@ -361,9 +393,11 @@ func (v *Vaultik) printRestoreProgress(filesDone, totalFiles int, bytesDone, tot v.UI.Duration(elapsed), v.UI.Time(time.Now().Add(eta)), v.UI.Duration(eta)) + return } - v.UI.Progress("Restore: %s/%s files (%s), %s/%s, %s, %.0f files/sec, restore elapsed: %s.", + + v.UI.Progressf("Restore: %s/%s files (%s), %s/%s, %s, %.0f files/sec, restore elapsed: %s.", v.UI.Count(filesDone), v.UI.Count(totalFiles), v.UI.Percent(pct), @@ -381,22 +415,26 @@ func (v *Vaultik) handleRestoreVerification( opts *RestoreOptions, result *RestoreResult, ) error { - if err := v.verifyRestoredFiles(v.ctx, repos, files, opts.TargetDir, result); err != nil { + err := v.verifyRestoredFiles(v.ctx, repos, files, opts.TargetDir, result) + if err != nil { return fmt.Errorf("verification failed: %w", err) } if result.FilesFailed > 0 { - v.UI.Error("Verification failed: %s files did not match expected checksums.", + v.UI.Errorf("Verification failed: %s files did not match expected checksums.", v.UI.Count(result.FilesFailed)) + for _, path := range result.FailedFiles { - v.UI.Detail("%s", v.UI.Path(path)) + v.UI.Detailf("%s", v.UI.Path(path)) } + return fmt.Errorf("%d files failed verification", result.FilesFailed) } - v.UI.Complete("Verified %s files (%s).", + v.UI.Completef("Verified %s files (%s).", v.UI.Count(result.FilesVerified), v.UI.Size(result.BytesVerified)) + return nil } @@ -418,6 +456,7 @@ func (v *Vaultik) downloadSnapshotDB(snapshotID string, identity age.Identity) ( if err != nil { return nil, fmt.Errorf("reading encrypted data: %w", err) } + log.Debug("Downloaded encrypted database", "size", humanize.Bytes(uint64(len(encryptedData)))) // Decrypt and decompress using blobgen.Reader @@ -432,6 +471,7 @@ func (v *Vaultik) downloadSnapshotDB(snapshotID string, identity age.Identity) ( if err != nil { return nil, fmt.Errorf("decrypting and decompressing: %w", err) } + log.Debug("Decrypted database", "size", humanize.Bytes(uint64(len(dbData)))) // Create a temporary database file and write the binary SQLite data directly @@ -439,18 +479,23 @@ func (v *Vaultik) downloadSnapshotDB(snapshotID string, identity age.Identity) ( if err != nil { return nil, fmt.Errorf("creating temp file: %w", err) } + tempPath := tempFile.Name() // Write the binary SQLite database directly if _, err := tempFile.Write(dbData); err != nil { _ = tempFile.Close() _ = v.Fs.Remove(tempPath) + return nil, fmt.Errorf("writing database file: %w", err) } + if err := tempFile.Close(); err != nil { _ = v.Fs.Remove(tempPath) + return nil, fmt.Errorf("closing temp file: %w", err) } + log.Debug("Created restore database", "path", tempPath) // Open the database @@ -471,6 +516,7 @@ func (v *Vaultik) getFilesToRestore(ctx context.Context, repos *database.Reposit // Get files matching the path filters var result []*database.File + seen := make(map[string]bool) for _, filter := range pathFilters { @@ -498,23 +544,30 @@ func (v *Vaultik) getFilesToRestore(ctx context.Context, repos *database.Reposit func (v *Vaultik) buildChunkToBlobMap(ctx context.Context, repos *database.Repositories) (map[string]*database.BlobChunk, error) { // Query all blob_chunks query := `SELECT blob_id, chunk_hash, offset, length FROM blob_chunks` + rows, err := repos.DB().Conn().QueryContext(ctx, query) if err != nil { return nil, fmt.Errorf("querying blob_chunks: %w", err) } + defer func() { _ = rows.Close() }() result := make(map[string]*database.BlobChunk) + for rows.Next() { - var bc database.BlobChunk - var blobIDStr, chunkHashStr string + var ( + bc database.BlobChunk + blobIDStr, chunkHashStr string + ) if err := rows.Scan(&blobIDStr, &chunkHashStr, &bc.Offset, &bc.Length); err != nil { return nil, fmt.Errorf("scanning blob_chunk: %w", err) } + blobID, err := types.ParseBlobID(blobIDStr) if err != nil { return nil, fmt.Errorf("parsing blob ID: %w", err) } + bc.BlobID = blobID bc.ChunkHash = types.ChunkHash(chunkHashStr) result[chunkHashStr] = &bc @@ -553,16 +606,22 @@ type restoreSession struct { // restoreFile dispatches to the right per-kind restorer. func (s *restoreSession) restoreFile(file *database.File) error { targetPath := filepath.Join(s.opts.TargetDir, file.Path.String()) + parentDir := filepath.Dir(targetPath) - if err := s.v.Fs.MkdirAll(parentDir, 0755); err != nil { + + err := s.v.Fs.MkdirAll(parentDir, 0755) + if err != nil { return fmt.Errorf("creating parent directory: %w", err) } + if file.IsSymlink() { return s.restoreSymlink(file, targetPath) } + if file.Mode&uint32(os.ModeDir) != 0 { return s.restoreDirectory(file, targetPath) } + return s.restoreRegularFile(file, targetPath) } @@ -572,37 +631,50 @@ func (s *restoreSession) restoreSymlink(file *database.File, targetPath string) // afero.MemMapFs doesn't support symlinks, so route real-FS // symlinks through os. if _, ok := s.v.Fs.(*afero.OsFs); ok { - if err := os.Symlink(file.LinkTarget.String(), targetPath); err != nil { + err := os.Symlink(file.LinkTarget.String(), targetPath) + if err != nil { return fmt.Errorf("creating symlink: %w", err) } } else { log.Debug("Symlink creation not supported on this filesystem", "path", file.Path, "target", file.LinkTarget) } + s.result.FilesRestored++ + log.Debug("Restored symlink", "path", file.Path, "target", file.LinkTarget) + return nil } // restoreDirectory restores a directory with its permissions, mtime, // and (on real filesystems, with sufficient privileges) ownership. func (s *restoreSession) restoreDirectory(file *database.File, targetPath string) error { - if err := s.v.Fs.MkdirAll(targetPath, os.FileMode(file.Mode)); err != nil { + err := s.v.Fs.MkdirAll(targetPath, os.FileMode(file.Mode)) + if err != nil { return fmt.Errorf("creating directory: %w", err) } - if err := s.v.Fs.Chmod(targetPath, os.FileMode(file.Mode)); err != nil { + + err = s.v.Fs.Chmod(targetPath, os.FileMode(file.Mode)) + if err != nil { log.Debug("Failed to set directory permissions", "path", targetPath, "error", err) } + if s.runningAsRoot { if _, ok := s.v.Fs.(*afero.OsFs); ok { - if err := os.Chown(targetPath, int(file.UID), int(file.GID)); err != nil { + err := os.Chown(targetPath, int(file.UID), int(file.GID)) + if err != nil { log.Debug("Failed to set directory ownership", "path", targetPath, "error", err) } } } - if err := s.v.Fs.Chtimes(targetPath, file.MTime, file.MTime); err != nil { + + err = s.v.Fs.Chtimes(targetPath, file.MTime, file.MTime) + if err != nil { log.Debug("Failed to set directory mtime", "path", targetPath, "error", err) } + s.result.FilesRestored++ + return nil } @@ -617,16 +689,20 @@ func (s *restoreSession) restoreRegularFile(file *database.File, targetPath stri t0 := time.Now() fileChunks, err := s.repos.FileChunks.GetByFileID(s.ctx, file.ID) fileChunksQueryDur := time.Since(t0) + if err != nil { return fmt.Errorf("getting file chunks: %w", err) } t0 = time.Now() + outFile, err := s.v.Fs.Create(targetPath) createDur := time.Since(t0) + if err != nil { return fmt.Errorf("creating output file: %w", err) } + defer func() { _ = outFile.Close() }() var ( @@ -638,10 +714,12 @@ func (s *restoreSession) restoreRegularFile(file *database.File, targetPath stri for _, fc := range fileChunks { chunkHashStr := fc.ChunkHash.String() + blobChunk, ok := s.chunkToBlobMap[chunkHashStr] if !ok { return fmt.Errorf("chunk %s not found in any blob", chunkHashStr[:16]) } + blobHash, ok := s.blobIDToHash[blobChunk.BlobID.String()] if !ok { return fmt.Errorf("blob id %s missing from hash index", blobChunk.BlobID) @@ -650,6 +728,7 @@ func (s *restoreSession) restoreRegularFile(file *database.File, targetPath stri t0 = time.Now() chunkData, err := s.blobCache.ReadAt(blobHash, blobChunk.Offset, blobChunk.Length) readAtDur += time.Since(t0) + if err != nil { return fmt.Errorf("reading chunk %s from cached blob %s: %w", fc.ChunkHash[:16], blobHash[:16], err) } @@ -657,13 +736,17 @@ func (s *restoreSession) restoreRegularFile(file *database.File, targetPath stri t0 = time.Now() n, err := outFile.Write(chunkData) writeDur += time.Since(t0) + if err != nil { return fmt.Errorf("writing chunk: %w", err) } + bytesWritten += int64(n) t0 = time.Now() + s.sweeper.chunkRestored(int64(n)) + sweeperDur += time.Since(t0) } @@ -682,16 +765,20 @@ func (s *restoreSession) restoreRegularFile(file *database.File, targetPath stri if err := outFile.Close(); err != nil { return fmt.Errorf("closing output file: %w", err) } + if err := s.v.Fs.Chmod(targetPath, os.FileMode(file.Mode)); err != nil { log.Debug("Failed to set file permissions", "path", targetPath, "error", err) } + if s.runningAsRoot { if _, ok := s.v.Fs.(*afero.OsFs); ok { - if err := os.Chown(targetPath, int(file.UID), int(file.GID)); err != nil { + err := os.Chown(targetPath, int(file.UID), int(file.GID)) + if err != nil { log.Debug("Failed to set file ownership", "path", targetPath, "error", err) } } } + if err := s.v.Fs.Chtimes(targetPath, file.MTime, file.MTime); err != nil { log.Debug("Failed to set file mtime", "path", targetPath, "error", err) } @@ -700,6 +787,7 @@ func (s *restoreSession) restoreRegularFile(file *database.File, targetPath stri s.result.BytesRestored += bytesWritten log.Debug("Restored file", "path", file.Path, "size", humanize.Bytes(uint64(bytesWritten))) + return nil } @@ -715,6 +803,7 @@ func (s *restoreSession) downloadBlobToCache(blobHash string, expectedSize int64 t0 := time.Now() rc, err := s.v.FetchAndDecryptBlob(s.ctx, blobHash, expectedSize, s.identity) fetchSetupDur := time.Since(t0) + if err != nil { return err } @@ -723,9 +812,11 @@ func (s *restoreSession) downloadBlobToCache(blobHash string, expectedSize int64 written, copyErr := s.blobCache.PutFromReader(blobHash, rc) streamDur := time.Since(t0) closeErr := rc.Close() + if copyErr != nil { return copyErr } + if closeErr != nil { return closeErr } @@ -738,6 +829,7 @@ func (s *restoreSession) downloadBlobToCache(blobHash string, expectedSize int64 "ms_fetch_setup", fetchSetupDur.Milliseconds(), "ms_stream_decrypt_decompress", streamDur.Milliseconds(), ) + return nil } @@ -751,18 +843,21 @@ func (v *Vaultik) verifyRestoredFiles( ) error { // Calculate total bytes to verify for progress bar var totalBytes int64 + regularFiles := make([]*database.File, 0, len(files)) for _, file := range files { // Skip symlinks and directories - only verify regular files if file.IsSymlink() || file.Mode&uint32(os.ModeDir) != 0 { continue } + regularFiles = append(regularFiles, file) totalBytes += file.Size } if len(regularFiles) == 0 { log.Info("No regular files to verify") + return nil } @@ -770,34 +865,40 @@ func (v *Vaultik) verifyRestoredFiles( "files", len(regularFiles), "bytes", humanize.Bytes(uint64(totalBytes)), ) - v.UI.Begin("Verifying %s files (%s).", + v.UI.Beginf("Verifying %s files (%s).", v.UI.Count(len(regularFiles)), v.UI.Size(totalBytes)) startTime := time.Now() lastStatusTime := startTime + const statusInterval = 15 * time.Second var bytesProcessed int64 + for i, file := range regularFiles { if ctx.Err() != nil { return ctx.Err() } targetPath := filepath.Join(targetDir, file.Path.String()) + bytesVerified, err := v.verifyFile(ctx, repos, file, targetPath) if err != nil { log.Error("File verification failed", "path", file.Path, "error", err) + result.FilesFailed++ result.FailedFiles = append(result.FailedFiles, file.Path.String()) } else { result.FilesVerified++ result.BytesVerified += bytesVerified } + bytesProcessed += file.Size if time.Since(lastStatusTime) >= statusInterval { v.printVerifyProgress(i+1, len(regularFiles), bytesProcessed, totalBytes, startTime) + lastStatusTime = time.Now() } } @@ -821,13 +922,14 @@ func (v *Vaultik) printVerifyProgress(filesDone, totalFiles int, bytesDone, tota fileRate := float64(filesDone) / elapsed.Seconds() remainingBytes := totalBytes - bytesDone + var eta time.Duration if byteRate > 0 && remainingBytes > 0 { eta = time.Duration(float64(remainingBytes)/byteRate) * time.Second } if eta > 0 { - v.UI.Progress("Verify: %s/%s files (%s), %s/%s, %s, %.0f files/sec, verify elapsed: %s, verify ETA: %s (est remain %s).", + v.UI.Progressf("Verify: %s/%s files (%s), %s/%s, %s, %.0f files/sec, verify elapsed: %s, verify ETA: %s (est remain %s).", v.UI.Count(filesDone), v.UI.Count(totalFiles), v.UI.Percent(pct), @@ -838,9 +940,11 @@ func (v *Vaultik) printVerifyProgress(filesDone, totalFiles int, bytesDone, tota v.UI.Duration(elapsed), v.UI.Time(time.Now().Add(eta)), v.UI.Duration(eta)) + return } - v.UI.Progress("Verify: %s/%s files (%s), %s/%s, %s, %.0f files/sec, verify elapsed: %s.", + + v.UI.Progressf("Verify: %s/%s files (%s), %s/%s, %s, %.0f files/sec, verify elapsed: %s.", v.UI.Count(filesDone), v.UI.Count(totalFiles), v.UI.Percent(pct), @@ -873,6 +977,7 @@ func (v *Vaultik) verifyFile( // Verify each chunk var bytesVerified int64 + for _, fc := range fileChunks { // Get chunk size from database chunk, err := repos.Chunks.GetByHash(ctx, fc.ChunkHash.String()) @@ -882,10 +987,12 @@ func (v *Vaultik) verifyFile( // Read chunk data from file chunkData := make([]byte, chunk.Size) + n, err := io.ReadFull(f, chunkData) if err != nil { return bytesVerified, fmt.Errorf("reading chunk data: %w", err) } + if int64(n) != chunk.Size { return bytesVerified, fmt.Errorf("short read: expected %d bytes, got %d", chunk.Size, n) } @@ -904,5 +1011,6 @@ func (v *Vaultik) verifyFile( } log.Debug("File verified", "path", file.Path, "bytes", bytesVerified, "chunks", len(fileChunks)) + return bytesVerified, nil } diff --git a/internal/vaultik/restore_locality_test.go b/internal/vaultik/restore_locality_test.go index 4a29bc2..0e7c52d 100644 --- a/internal/vaultik/restore_locality_test.go +++ b/internal/vaultik/restore_locality_test.go @@ -6,6 +6,7 @@ import ( "crypto/rand" "fmt" "io" + "maps" "os" "path/filepath" "sort" @@ -55,6 +56,7 @@ func TestRestoreLocalityAndReadAt(t *testing.T) { fs := afero.NewOsFs() tempDir, err := os.MkdirTemp("", "vaultik-locality-") require.NoError(t, err) + defer func() { _ = os.RemoveAll(tempDir) }() dataDir := filepath.Join(tempDir, "source") @@ -86,8 +88,9 @@ func TestRestoreLocalityAndReadAt(t *testing.T) { path string data []byte } + sources := make([]*source, srcCount) - for i := 0; i < srcCount; i++ { + for i := range srcCount { s := &source{ path: fmt.Sprintf("src-%03d.bin", i+1), data: randomBytes(t, srcBytes), @@ -105,11 +108,14 @@ func TestRestoreLocalityAndReadAt(t *testing.T) { sourceBlob int // 0, 1, or 2 sourceIndex int // index into sources slice } + groupReps := []int{0, perBlob, 2 * perBlob} // 0, 5, 10 letters := []byte{'A', 'B', 'C'} + var copies []copyFile - for i := 0; i < 3; i++ { - for j := 0; j < blobsCount; j++ { + + for i := range 3 { + for j := range blobsCount { seq := i*blobsCount + j + 1 name := fmt.Sprintf("cp-%03d-%c.bin", seq, letters[j]) path := filepath.Join(dataDir, name) @@ -143,6 +149,7 @@ func TestRestoreLocalityAndReadAt(t *testing.T) { db, err := database.New(ctx, dbPath) require.NoError(t, err) + defer func() { _ = db.Close() }() repos := database.NewRepositories(db) @@ -187,6 +194,7 @@ func TestRestoreLocalityAndReadAt(t *testing.T) { // immediately before close) so we read PeakLen and call counters // from the same instance the production code used. var cacheRef *blobDiskCache + v := &Vaultik{ Config: cfg, Storage: counter, @@ -214,6 +222,7 @@ func TestRestoreLocalityAndReadAt(t *testing.T) { require.NoErrorf(t, err, "source missing after restore: %s", s.path) require.Truef(t, bytes.Equal(got, s.data), "byte mismatch for source %s", s.path) } + for _, c := range copies { restored := filepath.Join(restoreDir, c.path) got, err := afero.ReadFile(fs, restored) @@ -226,6 +235,7 @@ func TestRestoreLocalityAndReadAt(t *testing.T) { if !filterBlobKey(key) { continue } + assert.Equalf(t, 1, n, "blob %s fetched %d times, want exactly 1", key, n) } @@ -248,9 +258,11 @@ func TestRestoreLocalityAndReadAt(t *testing.T) { // chunker picks non-degenerate FastCDC boundaries. func randomBytes(t *testing.T, n int) []byte { t.Helper() + b := make([]byte, n) _, err := rand.Read(b) require.NoError(t, err) + return b } @@ -258,21 +270,27 @@ func randomBytes(t *testing.T, n int) []byte { // relative keys for every blob file present. func listBlobKeys(t *testing.T, storeDir string) []string { t.Helper() + var keys []string + root := filepath.Join(storeDir, "blobs") err := filepath.Walk(root, func(p string, info os.FileInfo, err error) error { if err != nil { return err } + if info.IsDir() { return nil } + rel, _ := filepath.Rel(storeDir, p) keys = append(keys, rel) + return nil }) require.NoError(t, err) sort.Strings(keys) + return keys } @@ -289,6 +307,7 @@ func filterBlobKey(key string) bool { // vaultik for access to unexported cache internals. type countingStorerInternal struct { storage.Storer + mu sync.Mutex counts map[string]int } @@ -301,15 +320,16 @@ func (c *countingStorerInternal) Get(ctx context.Context, key string) (io.ReadCl c.mu.Lock() c.counts[key]++ c.mu.Unlock() + return c.Storer.Get(ctx, key) } func (c *countingStorerInternal) snapshot() map[string]int { c.mu.Lock() defer c.mu.Unlock() + out := make(map[string]int, len(c.counts)) - for k, v := range c.counts { - out[k] = v - } + maps.Copy(out, c.counts) + return out } diff --git a/internal/vaultik/restore_plan.go b/internal/vaultik/restore_plan.go index 5b53680..c6cf5cb 100644 --- a/internal/vaultik/restore_plan.go +++ b/internal/vaultik/restore_plan.go @@ -53,26 +53,33 @@ func newRestorePlan( // No chunks to fetch — restore can run immediately. p.fileBlobs[f.ID] = nil p.ready = append(p.ready, f.ID) + continue } + fileChunks, err := repos.FileChunks.GetByFileID(ctx, f.ID) if err != nil { return nil, fmt.Errorf("planning %s: %w", f.Path, err) } + blobs := make(map[string]struct{}) + for _, fc := range fileChunks { bc, ok := chunkToBlobMap[fc.ChunkHash.String()] if !ok { return nil, fmt.Errorf("planning %s: chunk %s missing from blob map", f.Path, fc.ChunkHash.String()[:16]) } + hash, ok := blobIDToHash[bc.BlobID.String()] if !ok { return nil, fmt.Errorf("planning %s: blob id %s missing from id-to-hash map", f.Path, bc.BlobID) } + blobs[hash] = struct{}{} } + p.fileBlobs[f.ID] = blobs for hash := range blobs { set, ok := p.blobFiles[hash] @@ -80,12 +87,15 @@ func newRestorePlan( set = make(map[types.FileID]struct{}) p.blobFiles[hash] = set } + set[f.ID] = struct{}{} } + if len(blobs) == 0 { p.ready = append(p.ready, f.ID) } } + return p, nil } @@ -96,10 +106,12 @@ func (p *restorePlan) markBlobCached(blobHash string) { if _, already := p.cached[blobHash]; already { return } + p.cached[blobHash] = struct{}{} for fileID := range p.blobFiles[blobHash] { blobs := p.fileBlobs[fileID] delete(blobs, blobHash) + if len(blobs) == 0 { p.ready = append(p.ready, fileID) } @@ -112,8 +124,10 @@ func (p *restorePlan) popReady() (types.FileID, bool) { if len(p.ready) == 0 { return types.FileID{}, false } + id := p.ready[0] p.ready = p.ready[1:] + return id, true } @@ -123,17 +137,20 @@ func (p *restorePlan) finishFile(fileID types.FileID) { for hash := range p.fileBlobs[fileID] { if set, ok := p.blobFiles[hash]; ok { delete(set, fileID) + if len(set) == 0 { delete(p.blobFiles, hash) } } } + delete(p.fileBlobs, fileID) // Also scrub the file from any blobFiles entries where it might // still appear even after its uncached-blob set was emptied. for hash, set := range p.blobFiles { if _, ok := set[fileID]; ok { delete(set, fileID) + if len(set) == 0 { delete(p.blobFiles, hash) } @@ -150,8 +167,11 @@ func (p *restorePlan) finishFile(fileID types.FileID) { // The zero FileID return means nothing is pending. func (p *restorePlan) pickNextDownload() types.FileID { var best types.FileID + bestCount := math.MaxInt + var bestID string + for id, blobs := range p.fileBlobs { n := len(blobs) if n == 0 { @@ -159,6 +179,7 @@ func (p *restorePlan) pickNextDownload() types.FileID { // popReady; ignore here just in case. continue } + idStr := id.String() if n < bestCount || (n == bestCount && (best.IsZero() || idStr < bestID)) { best = id @@ -166,16 +187,19 @@ func (p *restorePlan) pickNextDownload() types.FileID { bestID = idStr } } + return best } // blobsNeeded returns the uncached blob hashes for fileID in any order. func (p *restorePlan) blobsNeeded(fileID types.FileID) []string { blobs := p.fileBlobs[fileID] + out := make([]string, 0, len(blobs)) for h := range blobs { out = append(out, h) } + return out } diff --git a/internal/vaultik/restore_sweeper.go b/internal/vaultik/restore_sweeper.go index 2594844..f3a47f0 100644 --- a/internal/vaultik/restore_sweeper.go +++ b/internal/vaultik/restore_sweeper.go @@ -42,6 +42,7 @@ func newRestoreSweeper(ctx context.Context, repos *database.Repositories, cache if threshold <= 0 { threshold = 1 } + return &restoreSweeper{ ctx: ctx, repos: repos, @@ -65,6 +66,7 @@ func (s *restoreSweeper) chunkRestored(n int64) { if s.bytesAccum < s.threshold { return } + s.bytesAccum = 0 s.sweep() } @@ -77,8 +79,10 @@ func (s *restoreSweeper) sweep() { needed, err := s.blobStillNeeded(blobHash) if err != nil { log.Debug("sweeper referencing-files query failed", "blob_hash", blobHash[:16], "error", err) + continue } + if !needed { s.cache.Delete(blobHash) } @@ -104,15 +108,20 @@ func (s *restoreSweeper) blobStillNeeded(blobHash string) (bool, error) { for rows.Next() { var fileID string - if err := rows.Scan(&fileID); err != nil { + + err := rows.Scan(&fileID) + if err != nil { return true, fmt.Errorf("scanning file_id: %w", err) } + if _, ok := s.restored[fileID]; !ok { return true, nil } } + if err := rows.Err(); err != nil { return true, err } + return false, nil } diff --git a/internal/vaultik/restore_sweeper_integration_test.go b/internal/vaultik/restore_sweeper_integration_test.go index 044f3a2..d5b21e0 100644 --- a/internal/vaultik/restore_sweeper_integration_test.go +++ b/internal/vaultik/restore_sweeper_integration_test.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "io" + "maps" "math/rand" "os" "path/filepath" @@ -45,6 +46,7 @@ func TestRestoreSweeperEvictsBlobs(t *testing.T) { fs := afero.NewOsFs() tempDir, err := os.MkdirTemp("", "vaultik-sweeper-") require.NoError(t, err) + defer func() { _ = os.RemoveAll(tempDir) }() dataDir := filepath.Join(tempDir, "source") @@ -63,19 +65,22 @@ func TestRestoreSweeperEvictsBlobs(t *testing.T) { duplicateFiles = 10 fileSize = 1 * 1024 * 1024 ) + rng := rand.New(rand.NewSource(42)) type sourceFile struct { path string data []byte } + uniques := make([]sourceFile, 0, uniqueFiles) expected := make(map[string][]byte, uniqueFiles+duplicateFiles) - for i := 0; i < uniqueFiles; i++ { + for i := range uniqueFiles { data := make([]byte, fileSize) _, err := rng.Read(data) require.NoError(t, err) + path := filepath.Join(dataDir, fmt.Sprintf("unique-%02d.bin", i)) require.NoError(t, afero.WriteFile(fs, path, data, 0o644)) uniques = append(uniques, sourceFile{path: path, data: data}) @@ -112,6 +117,7 @@ func TestRestoreSweeperEvictsBlobs(t *testing.T) { db, err := database.New(ctx, dbPath) require.NoError(t, err) + defer func() { _ = db.Close() }() repos := database.NewRepositories(db) @@ -184,14 +190,18 @@ func TestRestoreSweeperEvictsBlobs(t *testing.T) { // sweeper evicted a still-needed blob; 0 means the cache silently // stopped being consulted. blobDownloads := 0 + for key, count := range counter.snapshot() { if !strings.HasPrefix(key, "blobs/") { continue } + assert.Equalf(t, 1, count, "blob %s should have been downloaded exactly once during restore, got %d", key, count) + blobDownloads++ } + assert.Equal(t, blobCount, blobDownloads, "every blob on disk should have been fetched exactly once during restore") t.Logf("restore downloaded %d blobs, each exactly once", blobDownloads) @@ -202,6 +212,7 @@ func TestRestoreSweeperEvictsBlobs(t *testing.T) { // re-downloading blobs that are evicted while still needed. type countingStorer struct { storage.Storer + mu sync.Mutex counts map[string]int } @@ -214,16 +225,17 @@ func (c *countingStorer) Get(ctx context.Context, key string) (io.ReadCloser, er c.mu.Lock() c.counts[key]++ c.mu.Unlock() + return c.Storer.Get(ctx, key) } func (c *countingStorer) snapshot() map[string]int { c.mu.Lock() defer c.mu.Unlock() + out := make(map[string]int, len(c.counts)) - for k, v := range c.counts { - out[k] = v - } + maps.Copy(out, c.counts) + return out } @@ -232,17 +244,21 @@ func (c *countingStorer) snapshot() map[string]int { // expected number of restore-time downloads. func countBlobsOnDisk(t *testing.T, storeDir string) int { t.Helper() + count := 0 root := filepath.Join(storeDir, "blobs") err := filepath.Walk(root, func(_ string, info os.FileInfo, err error) error { if err != nil { return err } + if !info.IsDir() { count++ } + return nil }) require.NoError(t, err) + return count } diff --git a/internal/vaultik/snapshot.go b/internal/vaultik/snapshot.go index ba5c017..3f9b040 100644 --- a/internal/vaultik/snapshot.go +++ b/internal/vaultik/snapshot.go @@ -2,6 +2,7 @@ package vaultik import ( "encoding/json" + "errors" "fmt" "os" "path/filepath" @@ -36,7 +37,8 @@ func (v *Vaultik) CreateSnapshot(opts *SnapshotCreateOptions) error { "index_path", v.Config.IndexPath, ) - if err := v.EnsureStorageBinding(); err != nil { + err := v.EnsureStorageBinding() + if err != nil { return err } @@ -72,23 +74,25 @@ func (v *Vaultik) CreateSnapshot(opts *SnapshotCreateOptions) error { } if len(snapshotNames) == 0 { - return fmt.Errorf("no snapshots configured") + return errors.New("no snapshots configured") } // Process each named snapshot for snapIdx, snapName := range snapshotNames { - if err := v.createNamedSnapshot(opts, hostname, snapName, snapIdx+1, len(snapshotNames)); err != nil { + err := v.createNamedSnapshot(opts, hostname, snapName, snapIdx+1, len(snapshotNames)) + if err != nil { return err } } // Print overall summary if multiple snapshots if len(snapshotNames) > 1 { - v.UI.Complete("All %d snapshots completed in %s.", len(snapshotNames), v.UI.Duration(time.Since(overallStartTime))) + v.UI.Completef("All %d snapshots completed in %s.", len(snapshotNames), v.UI.Duration(time.Since(overallStartTime))) } if opts.Prune { - if err := v.runPostBackupPrune(snapshotNames, opts.KeepNewerThan); err != nil { + err := v.runPostBackupPrune(snapshotNames, opts.KeepNewerThan) + if err != nil { return fmt.Errorf("post-backup prune: %w", err) } } @@ -99,9 +103,9 @@ func (v *Vaultik) CreateSnapshot(opts *SnapshotCreateOptions) error { // emit the summary via UI.Warning so cron actually delivers // something for the user to look at. if v.UI.WarningCount() > 0 { - v.UI.Warning("Finished with %d warning(s) — review the output above.", v.UI.WarningCount()) + v.UI.Warningf("Finished with %d warning(s) — review the output above.", v.UI.WarningCount()) } else { - v.UI.Complete("Finished successfully.") + v.UI.Completef("Finished successfully.") } return nil @@ -113,7 +117,7 @@ func (v *Vaultik) CreateSnapshot(opts *SnapshotCreateOptions) error { // snapshot of each name is kept. func (v *Vaultik) runPostBackupPrune(snapshotNames []string, keepNewerThan string) error { log.Info("Running post-backup prune", "snapshots", snapshotNames, "keep_newer_than", keepNewerThan) - v.UI.Begin("Running post-backup prune.") + v.UI.Beginf("Running post-backup prune.") purgeOpts := &SnapshotPurgeOptions{ Force: true, @@ -127,11 +131,13 @@ func (v *Vaultik) runPostBackupPrune(snapshotNames []string, keepNewerThan strin purgeOpts.KeepLatest = true } - if err := v.PurgeSnapshotsWithOptions(purgeOpts); err != nil { + err := v.PurgeSnapshotsWithOptions(purgeOpts) + if err != nil { return fmt.Errorf("purging old snapshots: %w", err) } - if err := v.PruneBlobs(&PruneOptions{Force: true}); err != nil { + err = v.PruneBlobs(&PruneOptions{Force: true}) + if err != nil { return fmt.Errorf("pruning orphaned blobs: %w", err) } @@ -158,7 +164,7 @@ func (v *Vaultik) createNamedSnapshot(opts *SnapshotCreateOptions, hostname, sna snapshotStartTime := time.Now() if total > 1 { - v.UI.Info("Snapshot %d/%d: %s.", idx, total, snapName) + v.UI.Infof("Snapshot %d/%d: %s.", idx, total, snapName) } resolvedDirs, err := v.resolveSnapshotPaths(snapName) @@ -178,8 +184,9 @@ func (v *Vaultik) createNamedSnapshot(opts *SnapshotCreateOptions, hostname, sna if err != nil { return fmt.Errorf("creating snapshot: %w", err) } + log.Info("Beginning snapshot", "snapshot_id", snapshotID, "name", snapName) - v.UI.Begin("Creating snapshot %s.", v.UI.Snapshot(snapshotID)) + v.UI.Beginf("Creating snapshot %s.", v.UI.Snapshot(snapshotID)) stats, err := v.scanAllDirectories(scanner, resolvedDirs, snapshotID) if err != nil { @@ -201,6 +208,7 @@ func (v *Vaultik) createNamedSnapshot(opts *SnapshotCreateOptions, hostname, sna "duration", time.Since(snapshotStartTime)) v.printSnapshotSummary(snapshotID, snapshotStartTime, stats) + return nil } @@ -238,12 +246,14 @@ func (v *Vaultik) scanAllDirectories(scanner *snapshot.Scanner, resolvedDirs []s select { case <-v.ctx.Done(): log.Info("Snapshot creation cancelled") + return nil, v.ctx.Err() default: } log.Info("Scanning directory", "path", dir) - v.UI.Begin("Enumerating snapshot source files in %s (%d of %d).", v.UI.Path(dir), i+1, len(resolvedDirs)) + v.UI.Beginf("Enumerating snapshot source files in %s (%d of %d).", v.UI.Path(dir), i+1, len(resolvedDirs)) + result, err := scanner.Scan(v.ctx, dir, snapshotID) if err != nil { return nil, fmt.Errorf("failed to scan %s: %w", dir, err) @@ -297,15 +307,18 @@ func (v *Vaultik) finalizeSnapshotMetadata(snapshotID string, stats *snapshotSta UploadDurationMs: stats.uploadDuration.Milliseconds(), } - if err := v.SnapshotManager.UpdateSnapshotStatsExtended(v.ctx, snapshotID, extStats); err != nil { + err := v.SnapshotManager.UpdateSnapshotStatsExtended(v.ctx, snapshotID, extStats) + if err != nil { return fmt.Errorf("updating snapshot stats: %w", err) } - if err := v.SnapshotManager.CompleteSnapshot(v.ctx, snapshotID); err != nil { + err = v.SnapshotManager.CompleteSnapshot(v.ctx, snapshotID) + if err != nil { return fmt.Errorf("completing snapshot: %w", err) } - if err := v.SnapshotManager.ExportSnapshotMetadata(v.ctx, v.Config.IndexPath, snapshotID); err != nil { + err = v.SnapshotManager.ExportSnapshotMetadata(v.ctx, v.Config.IndexPath, snapshotID) + if err != nil { return fmt.Errorf("exporting snapshot metadata: %w", err) } @@ -318,6 +331,7 @@ func (v *Vaultik) uploadSpeed(bytesUploaded int64, duration time.Duration) strin if bytesUploaded <= 0 || duration <= 0 { return v.UI.Speed(0) } + return v.UI.Speed(float64(bytesUploaded) / duration.Seconds()) } @@ -337,7 +351,8 @@ func (v *Vaultik) printSnapshotSummary(snapshotID string, startTime time.Time, s compressionRatio = 1.0 } - v.UI.Complete("Created snapshot %s.", v.UI.Snapshot(snapshotID)) + v.UI.Completef("Created snapshot %s.", v.UI.Snapshot(snapshotID)) + filesMsg := fmt.Sprintf("Files: %s examined, %s backed up, %s unchanged", v.UI.Count(stats.totalFiles), v.UI.Count(totalFilesChanged), @@ -345,7 +360,8 @@ func (v *Vaultik) printSnapshotSummary(snapshotID string, startTime time.Time, s if stats.totalFilesDeleted > 0 { filesMsg += fmt.Sprintf(", %s deleted", v.UI.Count(stats.totalFilesDeleted)) } - v.UI.Detail("%s.", filesMsg) + + v.UI.Detailf("%s.", filesMsg) dataMsg := fmt.Sprintf("Data: %s total (%s backed up)", v.UI.Size(totalBytesAll), @@ -353,20 +369,22 @@ func (v *Vaultik) printSnapshotSummary(snapshotID string, startTime time.Time, s if stats.totalBytesDeleted > 0 { dataMsg += fmt.Sprintf(", %s deleted", v.UI.Size(stats.totalBytesDeleted)) } - v.UI.Detail("%s.", dataMsg) + + v.UI.Detailf("%s.", dataMsg) if stats.totalBlobsUploaded > 0 { - v.UI.Detail("Storage: %s compressed from %s (%.2fx ratio).", + v.UI.Detailf("Storage: %s compressed from %s (%.2fx ratio).", v.UI.Size(totalBlobSizeCompressed), v.UI.Size(totalBlobSizeUncompressed), compressionRatio) - v.UI.Detail("Upload: %d blobs, %s in %s (%s).", + v.UI.Detailf("Upload: %d blobs, %s in %s (%s).", stats.totalBlobsUploaded, v.UI.Size(stats.totalBytesUploaded), v.UI.Duration(stats.uploadDuration), v.uploadSpeed(stats.totalBytesUploaded, stats.uploadDuration)) } - v.UI.Detail("Snapshot create duration: %s.", v.UI.Duration(snapshotDuration)) + + v.UI.Detailf("Snapshot create duration: %s.", v.UI.Duration(snapshotDuration)) } // getSnapshotBlobSizes returns total compressed and uncompressed blob sizes for a snapshot @@ -375,12 +393,14 @@ func (v *Vaultik) getSnapshotBlobSizes(snapshotID string) (compressed int64, unc if err != nil { return 0, 0 } + for _, hash := range blobHashes { if blob, err := v.Repositories.Blobs.GetByHash(v.ctx, hash); err == nil && blob != nil { compressed += blob.CompressedSize uncompressed += blob.UncompressedSize } } + return compressed, uncompressed } @@ -418,6 +438,7 @@ func (v *Vaultik) ListSnapshots(jsonOutput bool) error { if ls.CompletedAt == nil { continue } + snapshots = append(snapshots, v.snapshotInfoFromLocal(ls)) } @@ -428,6 +449,7 @@ func (v *Vaultik) ListSnapshots(jsonOutput bool) error { if jsonOutput { encoder := json.NewEncoder(v.Stdout) encoder.SetIndent("", " ") + return encoder.Encode(snapshots) } @@ -441,7 +463,8 @@ func (v *Vaultik) ListSnapshots(jsonOutput bool) error { remoteKeys, err := v.listAllRemoteSnapshotKeys() if err != nil { - v.UI.Warning("Could not list backup destination store: %v.", err) + v.UI.Warningf("Could not list backup destination store: %v.", err) + return nil } @@ -450,20 +473,25 @@ func (v *Vaultik) ListSnapshots(jsonOutput bool) error { if ls.CompletedAt == nil { continue } + localKeys[snapshot.RemoteSnapshotKey(ls.ID.String())] = ls.ID.String() } + remoteSet := make(map[string]bool, len(remoteKeys)) for _, k := range remoteKeys { remoteSet[k] = true } var localOnly []string + for key, humanID := range localKeys { if !remoteSet[key] { localOnly = append(localOnly, humanID) } } + var remoteOnlyCount int + for key := range remoteSet { if _, ok := localKeys[key]; !ok { remoteOnlyCount++ @@ -471,14 +499,17 @@ func (v *Vaultik) ListSnapshots(jsonOutput bool) error { } if len(localOnly) > 0 { - v.UI.Warning("%d local snapshot record(s) not found in backup destination store:", len(localOnly)) + v.UI.Warningf("%d local snapshot record(s) not found in backup destination store:", len(localOnly)) + for _, id := range localOnly { - v.UI.Info("%s", v.UI.Snapshot(id)) + v.UI.Infof("%s", v.UI.Snapshot(id)) } - v.UI.Info("Run 'vaultik snapshot cleanup' to remove stale local records.") + + v.UI.Infof("Run 'vaultik snapshot cleanup' to remove stale local records.") } + if remoteOnlyCount > 0 { - v.UI.Notice("NOTE: %d remote snapshot(s) found in backup destination store but not in local database.", remoteOnlyCount) + v.UI.Noticef("NOTE: %d remote snapshot(s) found in backup destination store but not in local database.", remoteOnlyCount) } return nil @@ -493,6 +524,7 @@ func (v *Vaultik) snapshotInfoFromLocal(ls *database.Snapshot) SnapshotInfo { totalSize, err := v.Repositories.Snapshots.GetSnapshotTotalCompressedSize(v.ctx, idStr) if err != nil { log.Warn("Failed to get total compressed size", "id", idStr, "error", err) + totalSize = ls.BlobSize } @@ -523,19 +555,24 @@ func (v *Vaultik) printSnapshotTable(snapshots []SnapshotInfo) error { if _, err := fmt.Fprintln(w, "CONFIGURED SNAPSHOTS:"); err != nil { return err } + if _, err := fmt.Fprintln(w, "NAME\tPATHS"); err != nil { return err } + if _, err := fmt.Fprintln(w, "────\t─────"); err != nil { return err } + for _, name := range v.Config.SnapshotNames() { snap := v.Config.Snapshots[name] + paths := strings.Join(snap.Paths, ", ") if _, err := fmt.Fprintf(w, "%s\t%s\n", name, paths); err != nil { return err } } + if _, err := fmt.Fprintln(w); err != nil { return err } @@ -543,9 +580,11 @@ func (v *Vaultik) printSnapshotTable(snapshots []SnapshotInfo) error { if _, err := fmt.Fprintln(w, "REMOTE SNAPSHOTS:"); err != nil { return err } + if _, err := fmt.Fprintln(w, "SNAPSHOT ID\tTIMESTAMP\tCOMPRESSED SIZE\tUNCOMPRESSED SIZE\tNEW CHUNK SIZE"); err != nil { return err } + if _, err := fmt.Fprintln(w, "───────────\t─────────\t───────────────\t─────────────────\t──────────────"); err != nil { return err } @@ -554,10 +593,12 @@ func (v *Vaultik) printSnapshotTable(snapshots []SnapshotInfo) error { for _, snap := range snapshots { uncompressed := remoteOnlyCell newChunks := remoteOnlyCell + if snap.LocallyTracked { uncompressed = formatBytes(snap.UncompressedSize) newChunks = formatBytes(snap.NewChunkSize) } + if _, err := fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n", snap.ID, snap.Timestamp.Format("2006-01-02 15:04:05"), @@ -611,11 +652,13 @@ func (v *Vaultik) PurgeSnapshotsWithOptions(opts *SnapshotPurgeOptions) error { if s.CompletedAt == nil { continue } + if len(nameFilter) > 0 { if _, ok := nameFilter[parseSnapshotName(s.ID.String())]; !ok { continue } } + snapshots = append(snapshots, SnapshotInfo{ ID: s.ID, Timestamp: s.StartedAt, @@ -634,12 +677,15 @@ func (v *Vaultik) PurgeSnapshotsWithOptions(opts *SnapshotPurgeOptions) error { // Keep the latest snapshot per snapshot name. Snapshots are sorted // newest-first, so the first occurrence of each name is kept. seen := make(map[string]bool) + for _, snap := range snapshots { name := parseSnapshotName(snap.ID.String()) if seen[name] { toDelete = append(toDelete, snap) + continue } + seen[name] = true } } else if opts.OlderThan != "" { @@ -660,6 +706,7 @@ func (v *Vaultik) PurgeSnapshotsWithOptions(opts *SnapshotPurgeOptions) error { if !opts.Quiet { v.printlnStdout("No snapshots to delete") } + return nil } @@ -669,9 +716,10 @@ func (v *Vaultik) PurgeSnapshotsWithOptions(opts *SnapshotPurgeOptions) error { // confirmAndExecutePurge shows deletion candidates, confirms with user, and deletes snapshots func (v *Vaultik) confirmAndExecutePurge(toDelete []SnapshotInfo, force, quiet bool) error { if !quiet { - v.printfStdout("The following snapshots will be deleted:\n\n") + v.stdoutf("The following snapshots will be deleted:\n\n") + for _, snap := range toDelete { - v.printfStdout(" %s (%s, %s)\n", + v.stdoutf(" %s (%s, %s)\n", snap.ID, snap.Timestamp.Format("2006-01-02 15:04:05"), formatBytes(snap.CompressedSize)) @@ -680,29 +728,37 @@ func (v *Vaultik) confirmAndExecutePurge(toDelete []SnapshotInfo, force, quiet b // Confirm unless --force is used if !force { - v.printfStdout("\nDelete %d snapshot(s)? [y/N] ", len(toDelete)) + v.stdoutf("\nDelete %d snapshot(s)? [y/N] ", len(toDelete)) + var confirm string if _, err := v.scanStdin(&confirm); err != nil { // Treat EOF or error as "no" v.printlnStdout("Cancelled") + return nil } + if strings.ToLower(confirm) != "y" { v.printlnStdout("Cancelled") + return nil } } else if !quiet { - v.printfStdout("\nDeleting %d snapshot(s) (--force specified)\n", len(toDelete)) + v.stdoutf("\nDeleting %d snapshot(s) (--force specified)\n", len(toDelete)) } // Delete snapshots (both local and remote) for _, snap := range toDelete { snapshotID := snap.ID.String() log.Info("Deleting snapshot", "id", snapshotID) - if err := v.deleteSnapshotFromLocalDB(snapshotID); err != nil { + + err := v.deleteSnapshotFromLocalDB(snapshotID) + if err != nil { log.Error("Failed to delete from local database", "snapshot_id", snapshotID, "error", err) } - if err := v.deleteRemoteSnapshotByKey(snapshot.RemoteSnapshotKey(snapshotID)); err != nil { + + err = v.deleteRemoteSnapshotByKey(snapshot.RemoteSnapshotKey(snapshotID)) + if err != nil { return fmt.Errorf("deleting snapshot %s from remote: %w", snapshotID, err) } } @@ -711,13 +767,14 @@ func (v *Vaultik) confirmAndExecutePurge(toDelete []SnapshotInfo, force, quiet b // separate command after a purge. Guarded against nil for tests // that don't wire up a SnapshotManager. if v.SnapshotManager != nil { - if err := v.SnapshotManager.CleanupOrphanedData(v.ctx); err != nil { + err := v.SnapshotManager.CleanupOrphanedData(v.ctx) + if err != nil { log.Warn("Failed to clean up orphaned local data after purge", "error", err) } } if !quiet { - v.printfStdout("Deleted %d snapshot(s)\n", len(toDelete)) + v.stdoutf("Deleted %d snapshot(s)\n", len(toDelete)) v.printlnStdout("\nNote: Run 'vaultik prune' to clean up unreferenced remote blobs.") } @@ -730,6 +787,7 @@ func (v *Vaultik) VerifySnapshot(snapshotID string, deep bool) error { if deep { return v.RunDeepVerify(snapshotID, opts) } + return v.VerifySnapshotWithOptions(snapshotID, opts) } @@ -740,6 +798,7 @@ func (v *Vaultik) VerifySnapshotWithOptions(snapshotID string, opts *VerifyOptio if opts.Deep { return v.RunDeepVerify(snapshotID, opts) } + result := &VerifyResult{ SnapshotID: snapshotID, Mode: "shallow", @@ -754,8 +813,10 @@ func (v *Vaultik) VerifySnapshotWithOptions(snapshotID string, opts *VerifyOptio if opts.JSON { result.Status = "failed" result.ErrorMessage = fmt.Sprintf("downloading manifest: %v", err) + return v.outputVerifyJSON(result) } + return fmt.Errorf("downloading manifest: %w", err) } @@ -763,18 +824,20 @@ func (v *Vaultik) VerifySnapshotWithOptions(snapshotID string, opts *VerifyOptio result.TotalSize = manifest.TotalCompressedSize if !opts.JSON { - v.printfStdout("Snapshot information:\n") - v.printfStdout(" Blob count: %d\n", manifest.BlobCount) - v.printfStdout(" Total size: %s\n", humanize.Bytes(uint64(manifest.TotalCompressedSize))) + v.stdoutf("Snapshot information:\n") + v.stdoutf(" Blob count: %d\n", manifest.BlobCount) + v.stdoutf(" Total size: %s\n", humanize.Bytes(uint64(manifest.TotalCompressedSize))) + if manifest.Timestamp != "" { if t, err := time.Parse(time.RFC3339, manifest.Timestamp); err == nil { - v.printfStdout(" Created: %s\n", t.Format("2006-01-02 15:04:05 MST")) + v.stdoutf(" Created: %s\n", t.Format("2006-01-02 15:04:05 MST")) } } + v.printlnStdout() // Check each blob exists - v.printfStdout("Checking blob existence...\n") + v.stdoutf("Checking blob existence...\n") } result.Verified, result.Missing, result.MissingSize = v.verifyManifestBlobsExist(manifest, opts) @@ -791,10 +854,12 @@ func (v *Vaultik) printVerifyHeader(snapshotID string, opts *VerifyOptions) { } if !opts.JSON { - v.printfStdout("Verifying snapshot %s\n", snapshotID) + v.stdoutf("Verifying snapshot %s\n", snapshotID) + if !snapshotTime.IsZero() { - v.printfStdout("Snapshot time: %s\n", snapshotTime.Format("2006-01-02 15:04:05 MST")) + v.stdoutf("Snapshot time: %s\n", snapshotTime.Format("2006-01-02 15:04:05 MST")) } + v.printlnStdout() } } @@ -808,14 +873,16 @@ func (v *Vaultik) verifyManifestBlobsExist(manifest *snapshot.Manifest, opts *Ve _, err := v.Storage.Stat(v.ctx, blobPath) if err != nil { if !opts.JSON { - v.printfStdout(" Missing: %s (%s)\n", blob.Hash, humanize.Bytes(uint64(blob.CompressedSize))) + v.stdoutf(" Missing: %s (%s)\n", blob.Hash, humanize.Bytes(uint64(blob.CompressedSize))) } + missing++ missingSize += blob.CompressedSize } else { verified++ } } + return verified, missing, missingSize } @@ -828,23 +895,29 @@ func (v *Vaultik) formatVerifyResult(result *VerifyResult, manifest *snapshot.Ma } else { result.Status = "ok" } + return v.outputVerifyJSON(result) } - v.printfStdout("\nVerification complete:\n") - v.printfStdout(" Verified: %d blobs (%s)\n", result.Verified, + v.stdoutf("\nVerification complete:\n") + v.stdoutf(" Verified: %d blobs (%s)\n", result.Verified, humanize.Bytes(uint64(manifest.TotalCompressedSize-result.MissingSize))) + if result.Missing > 0 { - v.printfStdout(" Missing: %d blobs (%s)\n", result.Missing, humanize.Bytes(uint64(result.MissingSize))) + v.stdoutf(" Missing: %d blobs (%s)\n", result.Missing, humanize.Bytes(uint64(result.MissingSize))) } else { - v.printfStdout(" Missing: 0 blobs\n") + v.stdoutf(" Missing: 0 blobs\n") } - v.printfStdout(" Status: ") + + v.stdoutf(" Status: ") + if result.Missing > 0 { - v.printfStdout("FAILED - %d blobs are missing\n", result.Missing) + v.stdoutf("FAILED - %d blobs are missing\n", result.Missing) + return fmt.Errorf("%d blobs are missing", result.Missing) } - v.printfStdout("OK - All blobs verified\n") + + v.stdoutf("OK - All blobs verified\n") return nil } @@ -853,12 +926,16 @@ func (v *Vaultik) formatVerifyResult(result *VerifyResult, manifest *snapshot.Ma func (v *Vaultik) outputVerifyJSON(result *VerifyResult) error { encoder := json.NewEncoder(v.Stdout) encoder.SetIndent("", " ") - if err := encoder.Encode(result); err != nil { + + err := encoder.Encode(result) + if err != nil { return fmt.Errorf("encoding JSON: %w", err) } + if result.Status == "failed" { return fmt.Errorf("verification failed: %s", result.ErrorMessage) } + return nil } @@ -871,10 +948,12 @@ func (v *Vaultik) CleanupLocalSnapshots() error { if err := v.EnsureStorageBinding(); err != nil { return err } + remoteKeys, err := v.listAllRemoteSnapshotKeys() if err != nil { return err } + remoteSet := make(map[string]bool, len(remoteKeys)) for _, k := range remoteKeys { remoteSet[k] = true @@ -886,14 +965,19 @@ func (v *Vaultik) CleanupLocalSnapshots() error { } var removed int + for _, snap := range localSnapshots { id := snap.ID.String() if !remoteSet[snapshot.RemoteSnapshotKey(id)] { - v.printfStdout("Removing stale local record: %s\n", id) - if err := v.deleteSnapshotFromLocalDB(id); err != nil { + v.stdoutf("Removing stale local record: %s\n", id) + + err := v.deleteSnapshotFromLocalDB(id) + if err != nil { log.Error("Failed to delete local snapshot", "snapshot_id", id, "error", err) + continue } + removed++ } } @@ -901,8 +985,9 @@ func (v *Vaultik) CleanupLocalSnapshots() error { if removed == 0 { v.printlnStdout("No stale local snapshots found.") } else { - v.printfStdout("Removed %d stale local snapshot record(s).\n", removed) + v.stdoutf("Removed %d stale local snapshot record(s).\n", removed) } + return nil } @@ -948,6 +1033,7 @@ func (v *Vaultik) syncWithRemote() error { if strings.HasPrefix(parts[1], ".") { continue } + remoteSnapshots[parts[1]] = true } } @@ -962,11 +1048,14 @@ func (v *Vaultik) syncWithRemote() error { // Remove local snapshots that don't exist remotely removedCount := 0 + for _, snapshot := range localSnapshots { snapshotIDStr := snapshot.ID.String() if !remoteSnapshots[snapshotIDStr] { log.Info("Removing local snapshot not found in remote", "snapshot_id", snapshot.ID) - if err := v.deleteSnapshotFromLocalDB(snapshotIDStr); err != nil { + + err := v.deleteSnapshotFromLocalDB(snapshotIDStr) + if err != nil { log.Error("Failed to delete local snapshot", "snapshot_id", snapshot.ID, "error", err) } else { removedCount++ @@ -1015,59 +1104,74 @@ func (v *Vaultik) RemoveSnapshot(snapshotID string, opts *RemoveOptions) (*Remov SnapshotID: snapshotID, } - if err := v.EnsureStorageBinding(); err != nil { + err := v.EnsureStorageBinding() + if err != nil { return result, err } if opts.DryRun { result.DryRun = true + if !opts.JSON { - v.printfStdout("Would remove snapshot: %s\n", snapshotID) + v.stdoutf("Would remove snapshot: %s\n", snapshotID) + if !opts.LocalOnly { v.printlnStdout("Would also remove snapshot metadata from remote storage (blobs untouched)") } + v.printlnStdout("[Dry run - no changes made]") } + if opts.JSON { return result, v.outputRemoveJSON(result) } + return result, nil } if !opts.Force && !opts.JSON { if opts.LocalOnly { - v.printfStdout("Remove snapshot '%s' from local database (remote untouched)? [y/N] ", snapshotID) + v.stdoutf("Remove snapshot '%s' from local database (remote untouched)? [y/N] ", snapshotID) } else { - v.printfStdout("Remove snapshot '%s' from local database AND its metadata from remote storage? [y/N] ", snapshotID) + v.stdoutf("Remove snapshot '%s' from local database AND its metadata from remote storage? [y/N] ", snapshotID) } + var confirm string if _, err := v.scanStdin(&confirm); err != nil { v.printlnStdout("Cancelled") + return result, nil } + if strings.ToLower(confirm) != "y" { v.printlnStdout("Cancelled") + return result, nil } } log.Info("Removing snapshot from local database", "snapshot_id", snapshotID) - if err := v.deleteSnapshotFromLocalDB(snapshotID); err != nil { + err = v.deleteSnapshotFromLocalDB(snapshotID) + if err != nil { return result, fmt.Errorf("removing from local database: %w", err) } if !opts.LocalOnly { log.Info("Removing snapshot metadata from remote storage", "snapshot_id", snapshotID) + remoteKey := snapshot.RemoteSnapshotKey(snapshotID) - if err := v.deleteRemoteSnapshotByKey(remoteKey); err != nil { + + err := v.deleteRemoteSnapshotByKey(remoteKey) + if err != nil { // Warn-and-proceed: the local-DB removal has already // happened, so let the user know the remote half didn't // finish and they can retry with `vaultik prune` once the // destination store is reachable. log.Warn("Could not remove snapshot metadata from remote storage", "error", err) + if v.UI != nil { - v.UI.Warning("Could not remove snapshot metadata from remote: %v. Run '%s' once the remote is reachable to finish cleanup.", err, pruneCommandHint) + v.UI.Warningf("Could not remove snapshot metadata from remote: %v. Run '%s' once the remote is reachable to finish cleanup.", err, pruneCommandHint) } } else { result.RemoteRemoved = true @@ -1075,7 +1179,8 @@ func (v *Vaultik) RemoveSnapshot(snapshotID string, opts *RemoveOptions) (*Remov } if v.SnapshotManager != nil { - if err := v.SnapshotManager.CleanupOrphanedData(v.ctx); err != nil { + err := v.SnapshotManager.CleanupOrphanedData(v.ctx) + if err != nil { log.Warn("Failed to clean up orphaned local data after removal", "error", err) } } @@ -1084,10 +1189,11 @@ func (v *Vaultik) RemoveSnapshot(snapshotID string, opts *RemoveOptions) (*Remov return result, v.outputRemoveJSON(result) } - v.printfStdout("Removed snapshot '%s' from local database\n", snapshotID) + v.stdoutf("Removed snapshot '%s' from local database\n", snapshotID) + if !opts.LocalOnly && result.RemoteRemoved { v.printlnStdout("Removed snapshot metadata from remote storage") - v.printfStdout("\nNote: The removed snapshot's blobs remain on the remote. Run '%s' to delete any blobs no longer referenced by any remaining remote snapshot.\n", pruneCommandHint) + v.stdoutf("\nNote: The removed snapshot's blobs remain on the remote. Run '%s' to delete any blobs no longer referenced by any remaining remote snapshot.\n", pruneCommandHint) } return result, nil @@ -1102,6 +1208,7 @@ func (v *Vaultik) RemoveAllSnapshots(opts *RemoveOptions) (*RemoveResult, error) if err := v.EnsureStorageBinding(); err != nil { return nil, err } + localSnaps, err := v.localSnapshotIDs() if err != nil { return nil, fmt.Errorf("listing local snapshots: %w", err) @@ -1114,7 +1221,7 @@ func (v *Vaultik) RemoveAllSnapshots(opts *RemoveOptions) (*RemoveResult, error) remoteKeys, remoteErr := v.listAllRemoteSnapshotKeys() if remoteErr != nil { log.Warn("Could not list remote snapshots", "error", remoteErr) - v.UI.Warning("Could not list remote snapshots: %v.", remoteErr) + v.UI.Warningf("Could not list remote snapshots: %v.", remoteErr) } // Anything visible on the remote that doesn't correspond to a @@ -1124,7 +1231,9 @@ func (v *Vaultik) RemoveAllSnapshots(opts *RemoveOptions) (*RemoveResult, error) for _, id := range localSnaps { knownLocalKeys[snapshot.RemoteSnapshotKey(id)] = id } + var orphanRemoteKeys []string + for _, key := range remoteKeys { if _, known := knownLocalKeys[key]; !known { orphanRemoteKeys = append(orphanRemoteKeys, key) @@ -1135,6 +1244,7 @@ func (v *Vaultik) RemoveAllSnapshots(opts *RemoveOptions) (*RemoveResult, error) if !opts.JSON { v.printlnStdout("No snapshots found") } + return &RemoveResult{}, nil } @@ -1152,15 +1262,19 @@ func (v *Vaultik) localSnapshotIDs() ([]string, error) { if v.Repositories == nil { return nil, nil } + snaps, err := v.Repositories.Snapshots.ListRecent(v.ctx, 100000) if err != nil { return nil, err } + ids := make([]string, 0, len(snaps)) for _, s := range snaps { ids = append(ids, s.ID.String()) } + sort.Strings(ids) + return ids, nil } @@ -1171,10 +1285,13 @@ func (v *Vaultik) localSnapshotIDs() ([]string, error) { // fatal. func (v *Vaultik) listAllRemoteSnapshotKeys() ([]string, error) { log.Info("Listing all remote snapshots") + objectCh := v.Storage.ListStream(v.ctx, "metadata/") seen := make(map[string]bool) + var keys []string + for object := range objectCh { if object.Err != nil { return nil, fmt.Errorf("listing remote snapshots: %w", object.Err) @@ -1186,6 +1303,7 @@ func (v *Vaultik) listAllRemoteSnapshotKeys() ([]string, error) { if strings.HasPrefix(parts[1], ".") { continue } + if strings.HasSuffix(object.Key, "/") || strings.Contains(object.Key, "/manifest.json.zst") { key := parts[1] if !seen[key] { @@ -1202,30 +1320,38 @@ func (v *Vaultik) listAllRemoteSnapshotKeys() ([]string, error) { // handleRemoveAllDryRun handles the dry-run mode for removing all snapshots func (v *Vaultik) handleRemoveAllDryRun(localSnaps, orphanRemoteKeys []string, opts *RemoveOptions) (*RemoveResult, error) { result := &RemoveResult{DryRun: true} + result.SnapshotsRemoved = append(result.SnapshotsRemoved, localSnaps...) if !opts.LocalOnly { result.SnapshotsRemoved = append(result.SnapshotsRemoved, orphanRemoteKeys...) } + if !opts.JSON { - v.printfStdout("Would remove %d local snapshot(s):\n", len(localSnaps)) + v.stdoutf("Would remove %d local snapshot(s):\n", len(localSnaps)) + for _, id := range localSnaps { - v.printfStdout(" %s\n", id) + v.stdoutf(" %s\n", id) } + if !opts.LocalOnly { if len(orphanRemoteKeys) > 0 { - v.printfStdout("Would also remove %d orphan remote snapshot key(s):\n", len(orphanRemoteKeys)) + v.stdoutf("Would also remove %d orphan remote snapshot key(s):\n", len(orphanRemoteKeys)) + for _, key := range orphanRemoteKeys { - v.printfStdout(" %s\n", key) + v.stdoutf(" %s\n", key) } } else { v.printlnStdout("Would also remove snapshot metadata from remote storage (blobs untouched)") } } + v.printlnStdout("[Dry run - no changes made]") } + if opts.JSON { return result, v.outputRemoveJSON(result) } + return result, nil } @@ -1235,24 +1361,29 @@ func (v *Vaultik) handleRemoveAllDryRun(localSnaps, orphanRemoteKeys []string, o // hint is the next step. func (v *Vaultik) executeRemoveAll(localSnaps, orphanRemoteKeys []string, opts *RemoveOptions) (*RemoveResult, error) { if !opts.Force { - return nil, fmt.Errorf("--all requires --force") + return nil, errors.New("--all requires --force") } log.Info("Removing all snapshots", "local_count", len(localSnaps), "orphan_remote_count", len(orphanRemoteKeys)) result := &RemoveResult{} remoteErrors := 0 + for _, snapshotID := range localSnaps { log.Info("Removing snapshot", "snapshot_id", snapshotID) - if err := v.deleteSnapshotFromLocalDB(snapshotID); err != nil { + err := v.deleteSnapshotFromLocalDB(snapshotID) + if err != nil { log.Error("Failed to remove from local database", "snapshot_id", snapshotID, "error", err) + continue } if !opts.LocalOnly { - if err := v.deleteRemoteSnapshotByKey(snapshot.RemoteSnapshotKey(snapshotID)); err != nil { + err := v.deleteRemoteSnapshotByKey(snapshot.RemoteSnapshotKey(snapshotID)) + if err != nil { log.Warn("Failed to remove snapshot metadata from remote", "snapshot_id", snapshotID, "error", err) + remoteErrors++ } } @@ -1263,23 +1394,29 @@ func (v *Vaultik) executeRemoveAll(localSnaps, orphanRemoteKeys []string, opts * if !opts.LocalOnly { for _, key := range orphanRemoteKeys { log.Info("Removing orphan remote snapshot", "remote_key", key) - if err := v.deleteRemoteSnapshotByKey(key); err != nil { + + err := v.deleteRemoteSnapshotByKey(key) + if err != nil { log.Warn("Failed to remove orphan from remote", "remote_key", key, "error", err) + remoteErrors++ + continue } + result.SnapshotsRemoved = append(result.SnapshotsRemoved, key) } if remoteErrors == 0 { result.RemoteRemoved = true } else if v.UI != nil { - v.UI.Warning("Some remote metadata deletions failed. Run '%s' once the remote is healthy to clean up unreferenced blobs.", pruneCommandHint) + v.UI.Warningf("Some remote metadata deletions failed. Run '%s' once the remote is healthy to clean up unreferenced blobs.", pruneCommandHint) } } if v.SnapshotManager != nil { - if err := v.SnapshotManager.CleanupOrphanedData(v.ctx); err != nil { + err := v.SnapshotManager.CleanupOrphanedData(v.ctx) + if err != nil { log.Warn("Failed to clean up orphaned local data after bulk removal", "error", err) } } @@ -1288,10 +1425,11 @@ func (v *Vaultik) executeRemoveAll(localSnaps, orphanRemoteKeys []string, opts * return result, v.outputRemoveJSON(result) } - v.printfStdout("Removed %d snapshot(s)\n", len(result.SnapshotsRemoved)) + v.stdoutf("Removed %d snapshot(s)\n", len(result.SnapshotsRemoved)) + if !opts.LocalOnly && result.RemoteRemoved { v.printlnStdout("Removed snapshot metadata from remote storage") - v.printfStdout("\nNote: Removed snapshots' blobs remain on the remote. Run '%s' to delete any blobs no longer referenced by any remaining remote snapshot.\n", pruneCommandHint) + v.stdoutf("\nNote: Removed snapshots' blobs remain on the remote. Run '%s' to delete any blobs no longer referenced by any remaining remote snapshot.\n", pruneCommandHint) } return result, nil @@ -1304,16 +1442,23 @@ func (v *Vaultik) deleteSnapshotFromLocalDB(snapshotID string) error { } // Delete related records first to avoid foreign key constraints - if err := v.Repositories.Snapshots.DeleteSnapshotFiles(v.ctx, snapshotID); err != nil { + err := v.Repositories.Snapshots.DeleteSnapshotFiles(v.ctx, snapshotID) + if err != nil { return fmt.Errorf("deleting snapshot files for %s: %w", snapshotID, err) } - if err := v.Repositories.Snapshots.DeleteSnapshotBlobs(v.ctx, snapshotID); err != nil { + + err = v.Repositories.Snapshots.DeleteSnapshotBlobs(v.ctx, snapshotID) + if err != nil { return fmt.Errorf("deleting snapshot blobs for %s: %w", snapshotID, err) } - if err := v.Repositories.Snapshots.DeleteSnapshotUploads(v.ctx, snapshotID); err != nil { + + err = v.Repositories.Snapshots.DeleteSnapshotUploads(v.ctx, snapshotID) + if err != nil { return fmt.Errorf("deleting snapshot uploads for %s: %w", snapshotID, err) } - if err := v.Repositories.Snapshots.Delete(v.ctx, snapshotID); err != nil { + + err = v.Repositories.Snapshots.Delete(v.ctx, snapshotID) + if err != nil { return fmt.Errorf("deleting snapshot record %s: %w", snapshotID, err) } @@ -1330,17 +1475,21 @@ func (v *Vaultik) deleteRemoteSnapshotByKey(remoteKey string) error { objectCh := v.Storage.ListStream(v.ctx, prefix) var objectsToDelete []string + for object := range objectCh { if object.Err != nil { return fmt.Errorf("listing objects: %w", object.Err) } + objectsToDelete = append(objectsToDelete, object.Key) } for _, key := range objectsToDelete { - if err := v.Storage.Delete(v.ctx, key); err != nil { + err := v.Storage.Delete(v.ctx, key) + if err != nil { return fmt.Errorf("removing %s: %w", key, err) } + log.Debug("Deleted remote object", "key", key) } @@ -1351,6 +1500,7 @@ func (v *Vaultik) deleteRemoteSnapshotByKey(remoteKey string) error { func (v *Vaultik) outputRemoveJSON(result *RemoveResult) error { encoder := json.NewEncoder(v.Stdout) encoder.SetIndent("", " ") + return encoder.Encode(result) } @@ -1367,7 +1517,7 @@ type PruneResult struct { // before starting a new backup or on-demand via the prune command. func (v *Vaultik) PruneDatabase() (*PruneResult, error) { log.Info("Pruning local database: removing incomplete snapshots and orphaned data") - v.UI.Begin("Pruning local index database (removing incomplete snapshots and orphaned data).") + v.UI.Beginf("Pruning local index database (removing incomplete snapshots and orphaned data).") result := &PruneResult{} @@ -1384,16 +1534,23 @@ func (v *Vaultik) PruneDatabase() (*PruneResult, error) { snapshotIDStr := snapshot.ID.String() log.Info("Deleting incomplete snapshot", "snapshot_id", snapshot.ID) // Delete related records first - if err := v.Repositories.Snapshots.DeleteSnapshotFiles(v.ctx, snapshotIDStr); err != nil { + err := v.Repositories.Snapshots.DeleteSnapshotFiles(v.ctx, snapshotIDStr) + if err != nil { log.Error("Failed to delete snapshot files", "snapshot_id", snapshot.ID, "error", err) } - if err := v.Repositories.Snapshots.DeleteSnapshotBlobs(v.ctx, snapshotIDStr); err != nil { + + err = v.Repositories.Snapshots.DeleteSnapshotBlobs(v.ctx, snapshotIDStr) + if err != nil { log.Error("Failed to delete snapshot blobs", "snapshot_id", snapshot.ID, "error", err) } - if err := v.Repositories.Snapshots.DeleteSnapshotUploads(v.ctx, snapshotIDStr); err != nil { + + err = v.Repositories.Snapshots.DeleteSnapshotUploads(v.ctx, snapshotIDStr) + if err != nil { log.Error("Failed to delete snapshot uploads", "snapshot_id", snapshot.ID, "error", err) } - if err := v.Repositories.Snapshots.Delete(v.ctx, snapshotIDStr); err != nil { + + err = v.Repositories.Snapshots.Delete(v.ctx, snapshotIDStr) + if err != nil { log.Error("Failed to delete snapshot", "snapshot_id", snapshot.ID, "error", err) } else { result.SnapshotsDeleted++ @@ -1427,11 +1584,12 @@ func (v *Vaultik) PruneDatabase() (*PruneResult, error) { ) snapshotCountAfter := snapshotCountBefore - result.SnapshotsDeleted - v.UI.Complete("Pruned local index database.") - v.UI.Detail("Incomplete snapshots: %d removed (%d remain).", result.SnapshotsDeleted, snapshotCountAfter) - v.UI.Detail("Orphaned files: %d removed (%d remain).", result.FilesDeleted, fileCountAfter) - v.UI.Detail("Orphaned chunks: %d removed (%d remain).", result.ChunksDeleted, chunkCountAfter) - v.UI.Detail("Orphaned blobs: %d removed (%d remain).", result.BlobsDeleted, blobCountAfter) + + v.UI.Completef("Pruned local index database.") + v.UI.Detailf("Incomplete snapshots: %d removed (%d remain).", result.SnapshotsDeleted, snapshotCountAfter) + v.UI.Detailf("Orphaned files: %d removed (%d remain).", result.FilesDeleted, fileCountAfter) + v.UI.Detailf("Orphaned chunks: %d removed (%d remain).", result.ChunksDeleted, chunkCountAfter) + v.UI.Detailf("Orphaned blobs: %d removed (%d remain).", result.BlobsDeleted, blobCountAfter) return result, nil } @@ -1451,10 +1609,13 @@ func (v *Vaultik) getTableCount(tableName string) (int64, error) { } var count int64 - query := fmt.Sprintf("SELECT COUNT(*) FROM %s", tableName) + + query := "SELECT COUNT(*) FROM " + tableName + err := v.DB.Conn().QueryRowContext(v.ctx, query).Scan(&count) if err != nil { return 0, err } + return count, nil } diff --git a/internal/vaultik/storage_bind.go b/internal/vaultik/storage_bind.go index 8c1083d..27f79c2 100644 --- a/internal/vaultik/storage_bind.go +++ b/internal/vaultik/storage_bind.go @@ -54,10 +54,13 @@ func (v *Vaultik) EnsureStorageBinding() error { } if stored == "" { - if err := v.Repositories.LocalMeta.Set(v.ctx, database.LocalMetaKeyStorageURL, configured); err != nil { + err := v.Repositories.LocalMeta.Set(v.ctx, database.LocalMetaKeyStorageURL, configured) + if err != nil { return fmt.Errorf("recording local storage binding: %w", err) } + log.Info("Bound local index to storage destination", "storage_url", configured) + return nil } diff --git a/internal/vaultik/vaultik.go b/internal/vaultik/vaultik.go index 91c93c1..7679229 100644 --- a/internal/vaultik/vaultik.go +++ b/internal/vaultik/vaultik.go @@ -3,6 +3,7 @@ package vaultik import ( "bytes" "context" + "errors" "fmt" "io" "os" @@ -125,8 +126,9 @@ func (v *Vaultik) CanDecrypt() bool { // Returns an error if no recipients are configured func (v *Vaultik) GetEncryptor() (*crypto.Encryptor, error) { if len(v.Config.AgeRecipients) == 0 { - return nil, fmt.Errorf("no age recipients configured") + return nil, errors.New("no age recipients configured") } + return crypto.NewEncryptor(v.Config.AgeRecipients) } @@ -134,8 +136,9 @@ func (v *Vaultik) GetEncryptor() (*crypto.Encryptor, error) { // Returns an error if no secret key is configured func (v *Vaultik) GetDecryptor() (*crypto.Decryptor, error) { if v.Config.AgeSecretKey == "" { - return nil, fmt.Errorf("no age secret key configured") + return nil, errors.New("no age secret key configured") } + return crypto.NewDecryptor(v.Config.AgeSecretKey) } @@ -144,8 +147,8 @@ func (v *Vaultik) GetFilesystem() afero.Fs { return v.Fs } -// printfStdout writes formatted output to stdout. -func (v *Vaultik) printfStdout(format string, args ...any) { +// stdoutf writes formatted output to stdout. +func (v *Vaultik) stdoutf(format string, args ...any) { _, _ = fmt.Fprintf(v.Stdout, format, args...) } @@ -162,6 +165,7 @@ func (v *Vaultik) scanStdin(a ...any) (int, error) { // TestVaultik wraps a Vaultik with captured stdout/stderr for testing type TestVaultik struct { *Vaultik + Stdout *bytes.Buffer Stderr *bytes.Buffer Stdin *bytes.Buffer @@ -175,6 +179,7 @@ func NewForTesting(storage storage.Storer) *TestVaultik { stdout := &bytes.Buffer{} stderr := &bytes.Buffer{} stdin := &bytes.Buffer{} + return &TestVaultik{ Vaultik: &Vaultik{ Storage: storage, diff --git a/internal/vaultik/verify.go b/internal/vaultik/verify.go index 9ad2f42..61d1089 100644 --- a/internal/vaultik/verify.go +++ b/internal/vaultik/verify.go @@ -39,13 +39,16 @@ type VerifyResult struct { // deepVerifyFailure records a failure in the result and returns it appropriately func (v *Vaultik) deepVerifyFailure(result *VerifyResult, opts *VerifyOptions, msg string, err error) error { result.Status = "failed" + result.ErrorMessage = msg if opts.JSON { return v.outputVerifyJSON(result) } + if err != nil { return err } + return fmt.Errorf("%s", msg) } @@ -58,12 +61,14 @@ func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error { if !v.CanDecrypt() { msg := "VAULTIK_AGE_SECRET_KEY not set; required for deep verification" + return v.deepVerifyFailure(result, opts, msg, fmt.Errorf("%s", msg)) } log.Info("Starting snapshot verification", "snapshot_id", snapshotID, "mode", "deep") + if !opts.JSON { - v.printfStdout("Deep verification of snapshot: %s\n\n", snapshotID) + v.stdoutf("Deep verification of snapshot: %s\n\n", snapshotID) } manifest, tempDB, dbBlobs, err := v.loadVerificationData(snapshotID, opts, result) @@ -77,10 +82,12 @@ func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error { }() result.BlobCount = len(dbBlobs) + var totalSize int64 for _, blob := range dbBlobs { totalSize += blob.CompressedSize } + result.TotalSize = totalSize if err := v.runVerificationSteps(manifest, dbBlobs, tempDB, opts, result, totalSize); err != nil { @@ -96,10 +103,10 @@ func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error { log.Info("✓ Verification completed successfully", "snapshot_id", snapshotID, "mode", "deep", "blobs_verified", len(dbBlobs)) - v.printfStdout("\n✓ Verification completed successfully\n") - v.printfStdout(" Snapshot: %s\n", snapshotID) - v.printfStdout(" Blobs verified: %d\n", len(dbBlobs)) - v.printfStdout(" Total size: %s\n", humanize.Bytes(uint64(totalSize))) + v.stdoutf("\n✓ Verification completed successfully\n") + v.stdoutf(" Snapshot: %s\n", snapshotID) + v.stdoutf(" Blobs verified: %d\n", len(dbBlobs)) + v.stdoutf(" Total size: %s\n", humanize.Bytes(uint64(totalSize))) return nil } @@ -112,15 +119,18 @@ func (v *Vaultik) loadVerificationData(snapshotID string, opts *VerifyOptions, r // Download manifest manifestPath := fmt.Sprintf("metadata/%s/manifest.json.zst", remoteKey) log.Info("Downloading manifest", "path", manifestPath) + if !opts.JSON { - v.printfStdout("Downloading manifest...\n") + v.stdoutf("Downloading manifest...\n") } + manifestReader, err := v.Storage.Get(v.ctx, manifestPath) if err != nil { return nil, nil, nil, v.deepVerifyFailure(result, opts, fmt.Sprintf("failed to download manifest: %v", err), fmt.Errorf("failed to download manifest: %w", err)) } + defer func() { _ = manifestReader.Close() }() manifest, err := snapshot.DecodeManifest(manifestReader) @@ -133,20 +143,23 @@ func (v *Vaultik) loadVerificationData(snapshotID string, opts *VerifyOptions, r log.Info("Manifest loaded", "manifest_blob_count", manifest.BlobCount, "manifest_total_size", humanize.Bytes(uint64(manifest.TotalCompressedSize))) + if !opts.JSON { - v.printfStdout("Manifest loaded: %d blobs (%s)\n", manifest.BlobCount, humanize.Bytes(uint64(manifest.TotalCompressedSize))) - v.printfStdout("Downloading and decrypting database...\n") + v.stdoutf("Manifest loaded: %d blobs (%s)\n", manifest.BlobCount, humanize.Bytes(uint64(manifest.TotalCompressedSize))) + v.stdoutf("Downloading and decrypting database...\n") } // Download and decrypt database dbPath := fmt.Sprintf("metadata/%s/db.zst.age", remoteKey) log.Info("Downloading encrypted database", "path", dbPath) + dbReader, err := v.Storage.Get(v.ctx, dbPath) if err != nil { return nil, nil, nil, v.deepVerifyFailure(result, opts, fmt.Sprintf("failed to download database: %v", err), fmt.Errorf("failed to download database: %w", err)) } + defer func() { _ = dbReader.Close() }() tdb, err := v.decryptAndLoadDatabase(dbReader, v.Config.AgeSecretKey) @@ -159,6 +172,7 @@ func (v *Vaultik) loadVerificationData(snapshotID string, opts *VerifyOptions, r dbBlobs, err := v.getBlobsFromDatabase(snapshotID, tdb.DB) if err != nil { _ = tdb.Close() + return nil, nil, nil, v.deepVerifyFailure(result, opts, fmt.Sprintf("failed to get blobs from database: %v", err), fmt.Errorf("failed to get blobs from database: %w", err)) @@ -172,8 +186,9 @@ func (v *Vaultik) loadVerificationData(snapshotID string, opts *VerifyOptions, r log.Info("Database loaded", "db_blob_count", len(dbBlobs), "db_total_size", humanize.Bytes(uint64(dbTotalSize))) + if !opts.JSON { - v.printfStdout("Database loaded: %d blobs (%s)\n", len(dbBlobs), humanize.Bytes(uint64(dbTotalSize))) + v.stdoutf("Database loaded: %d blobs (%s)\n", len(dbBlobs), humanize.Bytes(uint64(dbTotalSize))) } return manifest, tdb, dbBlobs, nil @@ -182,25 +197,31 @@ func (v *Vaultik) loadVerificationData(snapshotID string, opts *VerifyOptions, r // runVerificationSteps executes manifest verification, blob existence check, and deep content verification func (v *Vaultik) runVerificationSteps(manifest *snapshot.Manifest, dbBlobs []snapshot.BlobInfo, tdb *tempDB, opts *VerifyOptions, result *VerifyResult, totalSize int64) error { if !opts.JSON { - v.printfStdout("Verifying manifest against database...\n") + v.stdoutf("Verifying manifest against database...\n") } - if err := v.verifyManifestAgainstDatabase(manifest, dbBlobs); err != nil { + + err := v.verifyManifestAgainstDatabase(manifest, dbBlobs) + if err != nil { return v.deepVerifyFailure(result, opts, err.Error(), err) } if !opts.JSON { - v.printfStdout("Manifest verified.\n") - v.printfStdout("Checking blob existence in remote storage...\n") + v.stdoutf("Manifest verified.\n") + v.stdoutf("Checking blob existence in remote storage...\n") } - if err := v.verifyBlobExistenceFromDB(dbBlobs); err != nil { + + err = v.verifyBlobExistenceFromDB(dbBlobs) + if err != nil { return v.deepVerifyFailure(result, opts, err.Error(), err) } if !opts.JSON { - v.printfStdout("All blobs exist.\n") - v.printfStdout("Downloading and verifying blob contents (%d blobs, %s)...\n", len(dbBlobs), humanize.Bytes(uint64(totalSize))) + v.stdoutf("All blobs exist.\n") + v.stdoutf("Downloading and verifying blob contents (%d blobs, %s)...\n", len(dbBlobs), humanize.Bytes(uint64(totalSize))) } - if err := v.performDeepVerificationFromDB(dbBlobs, tdb.DB, opts); err != nil { + + err = v.performDeepVerificationFromDB(dbBlobs, tdb.DB, opts) + if err != nil { return v.deepVerifyFailure(result, opts, err.Error(), err) } @@ -210,12 +231,14 @@ func (v *Vaultik) runVerificationSteps(manifest *snapshot.Manifest, dbBlobs []sn // tempDB wraps sql.DB with cleanup type tempDB struct { *sql.DB + tempPath string } func (t *tempDB) Close() error { err := t.DB.Close() _ = os.Remove(t.tempPath) + return err } @@ -245,16 +268,20 @@ func (v *Vaultik) decryptAndLoadDatabase(reader io.ReadCloser, secretKey string) if err != nil { return nil, fmt.Errorf("failed to create temp file: %w", err) } + tempPath := tempFile.Name() // Stream decompress directly to file log.Info("Decompressing database...") + written, err := io.Copy(tempFile, decompressor) if err != nil { _ = tempFile.Close() _ = os.Remove(tempPath) + return nil, fmt.Errorf("failed to decompress database: %w", err) } + _ = tempFile.Close() log.Info("Database decompressed", "size", humanize.Bytes(uint64(written))) @@ -263,6 +290,7 @@ func (v *Vaultik) decryptAndLoadDatabase(reader io.ReadCloser, secretKey string) db, err := sql.Open("sqlite", tempPath) if err != nil { _ = os.Remove(tempPath) + return nil, fmt.Errorf("failed to open database: %w", err) } @@ -332,21 +360,28 @@ func (v *Vaultik) verifyBlobChunks(db *sql.DB, blobHash string, decompressor io. WHERE b.blob_hash = ? ORDER BY bc.offset ` + rows, err := db.QueryContext(v.ctx, query, blobHash) if err != nil { return 0, fmt.Errorf("failed to query blob chunks: %w", err) } + defer func() { _ = rows.Close() }() var lastOffset int64 = -1 + chunkCount := 0 totalRead := int64(0) // Verify each chunk in the blob for rows.Next() { - var chunkHash string - var offset, length int64 - if err := rows.Scan(&chunkHash, &offset, &length); err != nil { + var ( + chunkHash string + offset, length int64 + ) + + err := rows.Scan(&chunkHash, &offset, &length) + if err != nil { return 0, fmt.Errorf("failed to scan chunk row: %w", err) } @@ -354,6 +389,7 @@ func (v *Vaultik) verifyBlobChunks(db *sql.DB, blobHash string, decompressor io. if offset <= lastOffset { return 0, fmt.Errorf("chunks out of order: offset %d after %d", offset, lastOffset) } + lastOffset = offset // Read chunk data from decompressed stream @@ -363,6 +399,7 @@ func (v *Vaultik) verifyBlobChunks(db *sql.DB, blobHash string, decompressor io. if _, err := io.CopyN(io.Discard, decompressor, skipBytes); err != nil { return 0, fmt.Errorf("failed to skip to offset %d: %w", offset, err) } + totalRead = offset } @@ -371,6 +408,7 @@ func (v *Vaultik) verifyBlobChunks(db *sql.DB, blobHash string, decompressor io. if _, err := io.ReadFull(decompressor, chunkData); err != nil { return 0, fmt.Errorf("failed to read chunk at offset %d: %w", offset, err) } + totalRead += length // Verify chunk hash @@ -401,6 +439,7 @@ func (v *Vaultik) verifyBlobFinalIntegrity(decompressor io.Reader, blobHasher ha if err != nil { return fmt.Errorf("failed to check for remaining blob data: %w", err) } + if remaining > 0 { return fmt.Errorf("blob has %d unexpected trailing bytes not covered by chunk list", remaining) } @@ -424,19 +463,27 @@ func (v *Vaultik) getBlobsFromDatabase(snapshotID string, db *sql.DB) ([]snapsho WHERE sb.snapshot_id = ? ORDER BY b.blob_hash ` + rows, err := db.QueryContext(v.ctx, query, snapshotID) if err != nil { return nil, fmt.Errorf("failed to query snapshot blobs: %w", err) } + defer func() { _ = rows.Close() }() var blobs []snapshot.BlobInfo + for rows.Next() { - var hash string - var size int64 - if err := rows.Scan(&hash, &size); err != nil { + var ( + hash string + size int64 + ) + + err := rows.Scan(&hash, &size) + if err != nil { return nil, fmt.Errorf("failed to scan blob row: %w", err) } + blobs = append(blobs, snapshot.BlobInfo{ Hash: hash, CompressedSize: size, @@ -481,6 +528,7 @@ func (v *Vaultik) verifyManifestAgainstDatabase(manifest *snapshot.Manifest, dbB if !exists { return fmt.Errorf("manifest contains blob %s not in database", hash) } + if dbSize != manifestSize { return fmt.Errorf("blob %s size mismatch: database has %d bytes, manifest has %d bytes", hash, dbSize, manifestSize) @@ -491,6 +539,7 @@ func (v *Vaultik) verifyManifestAgainstDatabase(manifest *snapshot.Manifest, dbB "manifest_blobs", len(manifestBlobMap), "database_blobs", len(dbBlobMap), ) + return nil } @@ -525,6 +574,7 @@ func (v *Vaultik) verifyBlobExistenceFromDB(blobs []snapshot.BlobInfo) error { } log.Info("✓ All blobs exist in storage") + return nil } @@ -546,7 +596,8 @@ func (v *Vaultik) performDeepVerificationFromDB(blobs []snapshot.BlobInfo, db *s for i, blobInfo := range blobs { // Verify individual blob - if err := v.verifyBlob(blobInfo, db); err != nil { + err := v.verifyBlob(blobInfo, db) + if err != nil { return fmt.Errorf("blob %s verification failed: %w", blobInfo.Hash, err) } @@ -556,8 +607,10 @@ func (v *Vaultik) performDeepVerificationFromDB(blobs []snapshot.BlobInfo, db *s // Calculate ETA based on bytes processed var eta time.Duration + if bytesProcessed > 0 { bytesPerSec := float64(bytesProcessed) / elapsed.Seconds() + bytesRemaining := totalBytesExpected - bytesProcessed if bytesPerSec > 0 { eta = time.Duration(float64(bytesRemaining)/bytesPerSec) * time.Second @@ -577,7 +630,7 @@ func (v *Vaultik) performDeepVerificationFromDB(blobs []snapshot.BlobInfo, db *s ) if !opts.JSON { - v.printfStdout(" Verified %d/%d blobs (%d remaining) - %s/%s - elapsed %s, eta %s\n", + v.stdoutf(" Verified %d/%d blobs (%d remaining) - %s/%s - elapsed %s, eta %s\n", i+1, len(blobs), remaining, humanize.Bytes(uint64(bytesProcessed)), humanize.Bytes(uint64(totalBytesExpected)), diff --git a/internal/vaultik/verify_test.go b/internal/vaultik/verify_test.go index 6ff6adc..c1fc1c3 100644 --- a/internal/vaultik/verify_test.go +++ b/internal/vaultik/verify_test.go @@ -25,6 +25,7 @@ func TestTeeReaderWithDecryption(t *testing.T) { // Compress the data var compressedBuf bytes.Buffer + compressor, err := zstd.NewWriter(&compressedBuf, zstd.WithEncoderLevel(zstd.SpeedDefault)) require.NoError(t, err) _, err = compressor.Write(testData) @@ -40,6 +41,7 @@ func TestTeeReaderWithDecryption(t *testing.T) { require.NoError(t, err) var encryptedBuf bytes.Buffer + err = encryptor.EncryptStream(&encryptedBuf, bytes.NewReader(compressedBuf.Bytes())) require.NoError(t, err) @@ -68,6 +70,7 @@ func TestTeeReaderWithDecryption(t *testing.T) { // Decompress decompressor, err := zstd.NewReader(decryptedReader) require.NoError(t, err) + defer decompressor.Close() // Read all decompressed data (simulating chunk verification)