Remediate remaining lint findings under the canonical golangci-lint config (closes #61, closes #59) #63

Closed
clawbot wants to merge 3 commits from golangci-v2.12.2 into main
110 changed files with 2566 additions and 722 deletions
Showing only changes of commit 6cf9211407 - Show all commits

View File

@@ -1,5 +1,9 @@
version: "2" version: "2"
# Config schema uses the golangci-lint v2 layout (settings live under
# linters.settings, not top-level linters-settings) so that the
# thresholds below are actually applied by golangci-lint >= v2.
run: run:
timeout: 5m timeout: 5m
modules-download-mode: readonly modules-download-mode: readonly
@@ -14,8 +18,7 @@ linters:
- wsl # Deprecated, replaced by wsl_v5 - wsl # Deprecated, replaced by wsl_v5
- wrapcheck # Too verbose for internal packages - wrapcheck # Too verbose for internal packages
- varnamelen # Short names like db, id are idiomatic Go - varnamelen # Short names like db, id are idiomatic Go
settings:
linters-settings:
lll: lll:
line-length: 88 line-length: 88
funlen: funlen:
@@ -27,6 +30,5 @@ linters-settings:
threshold: 100 threshold: 100
issues: issues:
exclude-use-default: false
max-issues-per-linter: 0 max-issues-per-linter: 0
max-same-issues: 0 max-same-issues: 0

View File

@@ -1,6 +1,6 @@
# Lint stage # Lint stage
# golangci/golangci-lint:v2.11.3-alpine, 2026-03-17 # golangci/golangci-lint:v2.12.2-alpine, 2026-08-07
FROM golangci/golangci-lint:v2.11.3-alpine@sha256:b1c3de5862ad0a95b4e45a993b0f00415835d687e4f12c845c7493b86c13414e AS lint FROM golangci/golangci-lint:v2.12.2-alpine@sha256:91b27804074a0bacea298707f016911e60cf0cdbc6c7bf5ccacb5f0606d18d60 AS lint
RUN apk add --no-cache make build-base RUN apk add --no-cache make build-base

View File

@@ -55,7 +55,7 @@ clean:
# Install dependencies. # Install dependencies.
deps: deps:
go mod download go mod download
go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.12.2
# Run tests with coverage. # Run tests with coverage.
test-coverage: test-coverage:

View File

@@ -16,9 +16,11 @@ func main() {
panic("could not create CPU profile: " + err.Error()) panic("could not create CPU profile: " + err.Error())
} }
defer func() { _ = f.Close() }() defer func() { _ = f.Close() }()
if err := pprof.StartCPUProfile(f); err != nil { if err := pprof.StartCPUProfile(f); err != nil {
panic("could not start CPU profile: " + err.Error()) panic("could not start CPU profile: " + err.Error())
} }
defer pprof.StopCPUProfile() defer pprof.StopCPUProfile()
} }
@@ -30,7 +32,9 @@ func main() {
panic("could not create memory profile: " + err.Error()) panic("could not create memory profile: " + err.Error())
} }
defer func() { _ = f.Close() }() defer func() { _ = f.Close() }()
runtime.GC() // get up-to-date statistics runtime.GC() // get up-to-date statistics
if err := pprof.WriteHeapProfile(f); err != nil { if err := pprof.WriteHeapProfile(f); err != nil {
panic("could not write memory profile: " + err.Error()) panic("could not write memory profile: " + err.Error())
} }

View File

@@ -18,6 +18,7 @@ import (
"context" "context"
"database/sql" "database/sql"
"encoding/hex" "encoding/hex"
"errors"
"fmt" "fmt"
"io" "io"
"sync" "sync"
@@ -124,6 +125,7 @@ type BlobChunkRef struct {
// BlobWithReader wraps a FinishedBlob with its data reader // BlobWithReader wraps a FinishedBlob with its data reader
type BlobWithReader struct { type BlobWithReader struct {
*FinishedBlob *FinishedBlob
Reader io.ReadSeeker Reader io.ReadSeeker
TempFile afero.File // Optional, only set for disk-based blobs TempFile afero.File // Optional, only set for disk-based blobs
InsertedChunkHashes []string // Chunk hashes that were inserted to DB with this blob InsertedChunkHashes []string // Chunk hashes that were inserted to DB with this blob
@@ -134,14 +136,17 @@ type BlobWithReader struct {
// Returns an error if required configuration fields are missing or invalid. // Returns an error if required configuration fields are missing or invalid.
func NewPacker(cfg PackerConfig) (*Packer, error) { func NewPacker(cfg PackerConfig) (*Packer, error) {
if len(cfg.Recipients) == 0 { if len(cfg.Recipients) == 0 {
return nil, fmt.Errorf("recipients are required - blobs must be encrypted") return nil, errors.New("recipients are required - blobs must be encrypted")
} }
if cfg.MaxBlobSize <= 0 { if cfg.MaxBlobSize <= 0 {
return nil, fmt.Errorf("max blob size must be positive") return nil, errors.New("max blob size must be positive")
} }
if cfg.Fs == nil { if cfg.Fs == nil {
return nil, fmt.Errorf("filesystem is required") return nil, errors.New("filesystem is required")
} }
return &Packer{ return &Packer{
maxBlobSize: cfg.MaxBlobSize, maxBlobSize: cfg.MaxBlobSize,
compressionLevel: cfg.CompressionLevel, compressionLevel: cfg.CompressionLevel,
@@ -160,6 +165,7 @@ func NewPacker(cfg PackerConfig) (*Packer, error) {
func (p *Packer) SetBlobHandler(handler BlobHandler) { func (p *Packer) SetBlobHandler(handler BlobHandler) {
p.mu.Lock() p.mu.Lock()
defer p.mu.Unlock() defer p.mu.Unlock()
p.blobHandler = handler p.blobHandler = handler
} }
@@ -169,6 +175,7 @@ func (p *Packer) SetBlobHandler(handler BlobHandler) {
func (p *Packer) AddPendingChunk(hash string, size int64) { func (p *Packer) AddPendingChunk(hash string, size int64) {
p.mu.Lock() p.mu.Lock()
defer p.mu.Unlock() defer p.mu.Unlock()
p.pendingChunks = append(p.pendingChunks, PendingChunk{Hash: hash, Size: size}) p.pendingChunks = append(p.pendingChunks, PendingChunk{Hash: hash, Size: size})
} }
@@ -183,7 +190,8 @@ func (p *Packer) AddChunk(chunk *ChunkRef) error {
// Initialize new blob if needed // Initialize new blob if needed
if p.currentBlob == nil { if p.currentBlob == nil {
if err := p.startNewBlob(); err != nil { err := p.startNewBlob()
if err != nil {
return fmt.Errorf("starting new blob: %w", err) return fmt.Errorf("starting new blob: %w", err)
} }
} }
@@ -202,7 +210,8 @@ func (p *Packer) AddChunk(chunk *ChunkRef) error {
} }
// Add chunk to current blob // Add chunk to current blob
if err := p.addChunkToCurrentBlob(chunk); err != nil { err := p.addChunkToCurrentBlob(chunk)
if err != nil {
return err return err
} }
@@ -218,7 +227,8 @@ func (p *Packer) Flush() error {
defer p.mu.Unlock() defer p.mu.Unlock()
if p.currentBlob != nil && len(p.currentBlob.chunks) > 0 { if p.currentBlob != nil && len(p.currentBlob.chunks) > 0 {
if err := p.finalizeCurrentBlob(); err != nil { err := p.finalizeCurrentBlob()
if err != nil {
return fmt.Errorf("finalizing blob: %w", err) return fmt.Errorf("finalizing blob: %w", err)
} }
} }
@@ -253,6 +263,7 @@ func (p *Packer) GetFinishedBlobs() []*FinishedBlob {
blobs := p.finishedBlobs blobs := p.finishedBlobs
p.finishedBlobs = make([]*FinishedBlob, 0) p.finishedBlobs = make([]*FinishedBlob, 0)
return blobs return blobs
} }
@@ -267,6 +278,7 @@ func (p *Packer) startNewBlob() error {
if err != nil { if err != nil {
return fmt.Errorf("parsing blob ID: %w", err) return fmt.Errorf("parsing blob ID: %w", err)
} }
blob := &database.Blob{ blob := &database.Blob{
ID: blobIDTyped, ID: blobIDTyped,
Hash: types.BlobHash("temp-placeholder-" + blobID), // Temporary placeholder until finalized Hash: types.BlobHash("temp-placeholder-" + blobID), // Temporary placeholder until finalized
@@ -276,6 +288,7 @@ func (p *Packer) startNewBlob() error {
CompressedSize: 0, CompressedSize: 0,
UploadedTS: nil, UploadedTS: nil,
} }
if err := p.repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error { if err := p.repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error {
return p.repos.Blobs.Create(ctx, tx, blob) return p.repos.Blobs.Create(ctx, tx, blob)
}); err != nil { }); err != nil {
@@ -294,6 +307,7 @@ func (p *Packer) startNewBlob() error {
if err != nil { if err != nil {
_ = tempFile.Close() _ = tempFile.Close()
_ = p.fs.Remove(tempFile.Name()) _ = p.fs.Remove(tempFile.Name())
return fmt.Errorf("creating blobgen writer: %w", err) return fmt.Errorf("creating blobgen writer: %w", err)
} }
@@ -308,6 +322,7 @@ func (p *Packer) startNewBlob() error {
} }
log.Debug("Created new blob container", "blob_id", blobID, "temp_file", tempFile.Name()) log.Debug("Created new blob container", "blob_id", blobID, "temp_file", tempFile.Name())
return nil return nil
} }
@@ -316,6 +331,7 @@ func (p *Packer) addChunkToCurrentBlob(chunk *ChunkRef) error {
// Skip if chunk already in current blob // Skip if chunk already in current blob
if p.currentBlob.chunkSet[chunk.Hash] { if p.currentBlob.chunkSet[chunk.Hash] {
log.Debug("Skipping duplicate chunk already in current blob", "chunk_hash", chunk.Hash) log.Debug("Skipping duplicate chunk already in current blob", "chunk_hash", chunk.Hash)
return nil return nil
} }
@@ -403,24 +419,31 @@ func (p *Packer) finalizeCurrentBlob() error {
func (p *Packer) closeBlobWriter() (string, int64, error) { func (p *Packer) closeBlobWriter() (string, int64, error) {
if err := p.currentBlob.writer.Close(); err != nil { if err := p.currentBlob.writer.Close(); err != nil {
p.cleanupTempFile() p.cleanupTempFile()
return "", 0, fmt.Errorf("closing blobgen writer: %w", err) return "", 0, fmt.Errorf("closing blobgen writer: %w", err)
} }
if err := p.currentBlob.tempFile.Sync(); err != nil { if err := p.currentBlob.tempFile.Sync(); err != nil {
p.cleanupTempFile() p.cleanupTempFile()
return "", 0, fmt.Errorf("syncing temp file: %w", err) return "", 0, fmt.Errorf("syncing temp file: %w", err)
} }
finalSize, err := p.currentBlob.tempFile.Seek(0, io.SeekCurrent) finalSize, err := p.currentBlob.tempFile.Seek(0, io.SeekCurrent)
if err != nil { if err != nil {
p.cleanupTempFile() p.cleanupTempFile()
return "", 0, fmt.Errorf("getting file size: %w", err) return "", 0, fmt.Errorf("getting file size: %w", err)
} }
if _, err := p.currentBlob.tempFile.Seek(0, io.SeekStart); err != nil { if _, err := p.currentBlob.tempFile.Seek(0, io.SeekStart); err != nil {
p.cleanupTempFile() p.cleanupTempFile()
return "", 0, fmt.Errorf("seeking to start: %w", err) return "", 0, fmt.Errorf("seeking to start: %w", err)
} }
finalHash := p.currentBlob.writer.Sum256() finalHash := p.currentBlob.writer.Sum256()
return hex.EncodeToString(finalHash), finalSize, nil return hex.EncodeToString(finalHash), finalSize, nil
} }
@@ -432,6 +455,7 @@ func (p *Packer) buildChunkRefs() []*BlobChunkRef {
ChunkHash: chunk.Hash, Offset: chunk.Offset, Length: chunk.Size, ChunkHash: chunk.Hash, Offset: chunk.Offset, Length: chunk.Size,
}) })
} }
return refs return refs
} }
@@ -444,13 +468,16 @@ func (p *Packer) commitBlobToDatabase(blobHash string, finalSize int64, chunksTo
blobIDTyped, parseErr := types.ParseBlobID(p.currentBlob.id) blobIDTyped, parseErr := types.ParseBlobID(p.currentBlob.id)
if parseErr != nil { if parseErr != nil {
p.cleanupTempFile() p.cleanupTempFile()
return fmt.Errorf("parsing blob ID: %w", parseErr) return fmt.Errorf("parsing blob ID: %w", parseErr)
} }
err := p.repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error { err := p.repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error {
for _, chunk := range chunksToInsert { for _, chunk := range chunksToInsert {
dbChunk := &database.Chunk{ChunkHash: types.ChunkHash(chunk.Hash), Size: chunk.Size} dbChunk := &database.Chunk{ChunkHash: types.ChunkHash(chunk.Hash), Size: chunk.Size}
if err := p.repos.Chunks.Create(ctx, tx, dbChunk); err != nil {
err := p.repos.Chunks.Create(ctx, tx, dbChunk)
if err != nil {
return fmt.Errorf("creating chunk: %w", err) return fmt.Errorf("creating chunk: %w", err)
} }
} }
@@ -460,7 +487,9 @@ func (p *Packer) commitBlobToDatabase(blobHash string, finalSize int64, chunksTo
BlobID: blobIDTyped, ChunkHash: types.ChunkHash(chunk.Hash), BlobID: blobIDTyped, ChunkHash: types.ChunkHash(chunk.Hash),
Offset: chunk.Offset, Length: chunk.Size, Offset: chunk.Offset, Length: chunk.Size,
} }
if err := p.repos.BlobChunks.Create(ctx, tx, blobChunk); err != nil {
err := p.repos.BlobChunks.Create(ctx, tx, blobChunk)
if err != nil {
return fmt.Errorf("creating blob_chunk: %w", err) return fmt.Errorf("creating blob_chunk: %w", err)
} }
} }
@@ -469,11 +498,13 @@ func (p *Packer) commitBlobToDatabase(blobHash string, finalSize int64, chunksTo
}) })
if err != nil { if err != nil {
p.cleanupTempFile() p.cleanupTempFile()
return fmt.Errorf("finalizing blob transaction: %w", err) return fmt.Errorf("finalizing blob transaction: %w", err)
} }
log.Debug("Committed blob transaction", log.Debug("Committed blob transaction",
"chunks_inserted", len(chunksToInsert), "blob_chunks_inserted", len(p.currentBlob.chunks)) "chunks_inserted", len(chunksToInsert), "blob_chunks_inserted", len(p.currentBlob.chunks))
return nil return nil
} }
@@ -482,6 +513,7 @@ func (p *Packer) deliverFinishedBlob(finished *FinishedBlob, insertedChunkHashes
if p.blobHandler != nil { if p.blobHandler != nil {
if _, err := p.currentBlob.tempFile.Seek(0, io.SeekStart); err != nil { if _, err := p.currentBlob.tempFile.Seek(0, io.SeekStart); err != nil {
p.cleanupTempFile() p.cleanupTempFile()
return fmt.Errorf("seeking for handler: %w", err) return fmt.Errorf("seeking for handler: %w", err)
} }
@@ -492,30 +524,39 @@ func (p *Packer) deliverFinishedBlob(finished *FinishedBlob, insertedChunkHashes
InsertedChunkHashes: insertedChunkHashes, InsertedChunkHashes: insertedChunkHashes,
} }
if err := p.blobHandler(blobWithReader); err != nil { err := p.blobHandler(blobWithReader)
if err != nil {
p.cleanupTempFile() p.cleanupTempFile()
return fmt.Errorf("blob handler failed: %w", err) return fmt.Errorf("blob handler failed: %w", err)
} }
p.currentBlob = nil p.currentBlob = nil
return nil return nil
} }
// No handler - read data for legacy behavior // No handler - read data for legacy behavior
log.Debug("No blob handler callback configured", "blob_hash", finished.Hash[:8]+"...") log.Debug("No blob handler callback configured", "blob_hash", finished.Hash[:8]+"...")
if _, err := p.currentBlob.tempFile.Seek(0, io.SeekStart); err != nil { if _, err := p.currentBlob.tempFile.Seek(0, io.SeekStart); err != nil {
p.cleanupTempFile() p.cleanupTempFile()
return fmt.Errorf("seeking to read data: %w", err) return fmt.Errorf("seeking to read data: %w", err)
} }
data, err := io.ReadAll(p.currentBlob.tempFile) data, err := io.ReadAll(p.currentBlob.tempFile)
if err != nil { if err != nil {
p.cleanupTempFile() p.cleanupTempFile()
return fmt.Errorf("reading blob data: %w", err) return fmt.Errorf("reading blob data: %w", err)
} }
finished.Data = data finished.Data = data
p.finishedBlobs = append(p.finishedBlobs, finished) p.finishedBlobs = append(p.finishedBlobs, finished)
p.cleanupTempFile() p.cleanupTempFile()
p.currentBlob = nil p.currentBlob = nil
return nil return nil
} }
@@ -532,13 +573,15 @@ func (p *Packer) cleanupTempFile() {
func (p *Packer) PackChunks(chunks []*ChunkRef) error { func (p *Packer) PackChunks(chunks []*ChunkRef) error {
for _, chunk := range chunks { for _, chunk := range chunks {
err := p.AddChunk(chunk) err := p.AddChunk(chunk)
if err == ErrBlobSizeLimitExceeded { if errors.Is(err, ErrBlobSizeLimitExceeded) {
// Finalize current blob and retry // Finalize current blob and retry
if err := p.FinalizeBlob(); err != nil { err := p.FinalizeBlob()
if err != nil {
return fmt.Errorf("finalizing blob before retry: %w", err) return fmt.Errorf("finalizing blob before retry: %w", err)
} }
// Retry the chunk // Retry the chunk
if err := p.AddChunk(chunk); err != nil { err = p.AddChunk(chunk)
if err != nil {
return fmt.Errorf("adding chunk %s after finalize: %w", chunk.Hash, err) return fmt.Errorf("adding chunk %s after finalize: %w", chunk.Hash, err)
} }
} else if err != nil { } else if err != nil {

View File

@@ -6,6 +6,7 @@ import (
"crypto/sha256" "crypto/sha256"
"database/sql" "database/sql"
"encoding/hex" "encoding/hex"
"errors"
"io" "io"
"testing" "testing"
@@ -40,6 +41,7 @@ func TestPacker(t *testing.T) {
t.Fatalf("failed to create test db: %v", err) t.Fatalf("failed to create test db: %v", err)
} }
defer func() { _ = db.Close() }() defer func() { _ = db.Close() }()
repos := database.NewRepositories(db) repos := database.NewRepositories(db)
cfg := PackerConfig{ cfg := PackerConfig{
@@ -49,6 +51,7 @@ func TestPacker(t *testing.T) {
Repositories: repos, Repositories: repos,
Fs: afero.NewMemMapFs(), Fs: afero.NewMemMapFs(),
} }
packer, err := NewPacker(cfg) packer, err := NewPacker(cfg)
if err != nil { if err != nil {
t.Fatalf("failed to create packer: %v", err) t.Fatalf("failed to create packer: %v", err)
@@ -64,6 +67,7 @@ func TestPacker(t *testing.T) {
ChunkHash: types.ChunkHash(hashStr), ChunkHash: types.ChunkHash(hashStr),
Size: int64(len(data)), Size: int64(len(data)),
} }
err = repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error { err = repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error {
return repos.Chunks.Create(ctx, tx, dbChunk) return repos.Chunks.Create(ctx, tx, dbChunk)
}) })
@@ -130,6 +134,7 @@ func TestPacker(t *testing.T) {
t.Fatalf("failed to create test db: %v", err) t.Fatalf("failed to create test db: %v", err)
} }
defer func() { _ = db.Close() }() defer func() { _ = db.Close() }()
repos := database.NewRepositories(db) repos := database.NewRepositories(db)
cfg := PackerConfig{ cfg := PackerConfig{
@@ -139,6 +144,7 @@ func TestPacker(t *testing.T) {
Repositories: repos, Repositories: repos,
Fs: afero.NewMemMapFs(), Fs: afero.NewMemMapFs(),
} }
packer, err := NewPacker(cfg) packer, err := NewPacker(cfg)
if err != nil { if err != nil {
t.Fatalf("failed to create packer: %v", err) t.Fatalf("failed to create packer: %v", err)
@@ -146,7 +152,8 @@ func TestPacker(t *testing.T) {
// Create multiple small chunks // Create multiple small chunks
chunks := make([]*ChunkRef, 10) chunks := make([]*ChunkRef, 10)
for i := 0; i < 10; i++ {
for i := range 10 {
data := bytes.Repeat([]byte{byte(i)}, 1000) data := bytes.Repeat([]byte{byte(i)}, 1000)
hash := sha256.Sum256(data) hash := sha256.Sum256(data)
hashStr := hex.EncodeToString(hash[:]) hashStr := hex.EncodeToString(hash[:])
@@ -156,6 +163,7 @@ func TestPacker(t *testing.T) {
ChunkHash: types.ChunkHash(hashStr), ChunkHash: types.ChunkHash(hashStr),
Size: int64(len(data)), Size: int64(len(data)),
} }
err = repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error { err = repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error {
return repos.Chunks.Create(ctx, tx, dbChunk) return repos.Chunks.Create(ctx, tx, dbChunk)
}) })
@@ -198,9 +206,11 @@ func TestPacker(t *testing.T) {
if chunkRef.Offset != expectedOffset { if chunkRef.Offset != expectedOffset {
t.Errorf("chunk %d: expected offset %d, got %d", i, expectedOffset, chunkRef.Offset) t.Errorf("chunk %d: expected offset %d, got %d", i, expectedOffset, chunkRef.Offset)
} }
if chunkRef.Length != 1000 { if chunkRef.Length != 1000 {
t.Errorf("chunk %d: expected length 1000, got %d", i, chunkRef.Length) t.Errorf("chunk %d: expected length 1000, got %d", i, chunkRef.Length)
} }
expectedOffset += chunkRef.Length expectedOffset += chunkRef.Length
} }
}) })
@@ -212,6 +222,7 @@ func TestPacker(t *testing.T) {
t.Fatalf("failed to create test db: %v", err) t.Fatalf("failed to create test db: %v", err)
} }
defer func() { _ = db.Close() }() defer func() { _ = db.Close() }()
repos := database.NewRepositories(db) repos := database.NewRepositories(db)
// Small blob size limit to force multiple blobs // Small blob size limit to force multiple blobs
@@ -222,6 +233,7 @@ func TestPacker(t *testing.T) {
Repositories: repos, Repositories: repos,
Fs: afero.NewMemMapFs(), Fs: afero.NewMemMapFs(),
} }
packer, err := NewPacker(cfg) packer, err := NewPacker(cfg)
if err != nil { if err != nil {
t.Fatalf("failed to create packer: %v", err) t.Fatalf("failed to create packer: %v", err)
@@ -229,7 +241,8 @@ func TestPacker(t *testing.T) {
// Create chunks that will exceed the limit // Create chunks that will exceed the limit
chunks := make([]*ChunkRef, 10) chunks := make([]*ChunkRef, 10)
for i := 0; i < 10; i++ {
for i := range 10 {
data := bytes.Repeat([]byte{byte(i)}, 1000) // 1KB each data := bytes.Repeat([]byte{byte(i)}, 1000) // 1KB each
hash := sha256.Sum256(data) hash := sha256.Sum256(data)
hashStr := hex.EncodeToString(hash[:]) hashStr := hex.EncodeToString(hash[:])
@@ -239,6 +252,7 @@ func TestPacker(t *testing.T) {
ChunkHash: types.ChunkHash(hashStr), ChunkHash: types.ChunkHash(hashStr),
Size: int64(len(data)), Size: int64(len(data)),
} }
err = repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error { err = repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error {
return repos.Chunks.Create(ctx, tx, dbChunk) return repos.Chunks.Create(ctx, tx, dbChunk)
}) })
@@ -257,14 +271,17 @@ func TestPacker(t *testing.T) {
// Add chunks and handle size limit errors // Add chunks and handle size limit errors
for _, chunk := range chunks { for _, chunk := range chunks {
err := packer.AddChunk(chunk) err := packer.AddChunk(chunk)
if err == ErrBlobSizeLimitExceeded { if errors.Is(err, ErrBlobSizeLimitExceeded) {
// Finalize current blob // Finalize current blob
if err := packer.FinalizeBlob(); err != nil { err := packer.FinalizeBlob()
if err != nil {
t.Fatalf("failed to finalize blob: %v", err) t.Fatalf("failed to finalize blob: %v", err)
} }
blobCount++ blobCount++
// Retry adding the chunk // Retry adding the chunk
if err := packer.AddChunk(chunk); err != nil { err = packer.AddChunk(chunk)
if err != nil {
t.Fatalf("failed to add chunk after finalize: %v", err) t.Fatalf("failed to add chunk after finalize: %v", err)
} }
} else if err != nil { } else if err != nil {
@@ -301,6 +318,7 @@ func TestPacker(t *testing.T) {
t.Fatalf("failed to create test db: %v", err) t.Fatalf("failed to create test db: %v", err)
} }
defer func() { _ = db.Close() }() defer func() { _ = db.Close() }()
repos := database.NewRepositories(db) repos := database.NewRepositories(db)
// Generate test identity (using the one from parent test) // Generate test identity (using the one from parent test)
@@ -311,6 +329,7 @@ func TestPacker(t *testing.T) {
Repositories: repos, Repositories: repos,
Fs: afero.NewMemMapFs(), Fs: afero.NewMemMapFs(),
} }
packer, err := NewPacker(cfg) packer, err := NewPacker(cfg)
if err != nil { if err != nil {
t.Fatalf("failed to create packer: %v", err) t.Fatalf("failed to create packer: %v", err)
@@ -326,6 +345,7 @@ func TestPacker(t *testing.T) {
ChunkHash: types.ChunkHash(hashStr), ChunkHash: types.ChunkHash(hashStr),
Size: int64(len(data)), Size: int64(len(data)),
} }
err = repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error { err = repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error {
return repos.Chunks.Create(ctx, tx, dbChunk) return repos.Chunks.Create(ctx, tx, dbChunk)
}) })
@@ -342,6 +362,7 @@ func TestPacker(t *testing.T) {
if err := packer.AddChunk(chunk); err != nil { if err := packer.AddChunk(chunk); err != nil {
t.Fatalf("failed to add chunk: %v", err) t.Fatalf("failed to add chunk: %v", err)
} }
if err := packer.Flush(); err != nil { if err := packer.Flush(); err != nil {
t.Fatalf("failed to flush: %v", err) t.Fatalf("failed to flush: %v", err)
} }

View File

@@ -28,6 +28,7 @@ func CompressData(data []byte, compressionLevel int, recipients []string) (*Comp
// Write data // Write data
if _, err := w.Write(data); err != nil { if _, err := w.Write(data); err != nil {
_ = w.Close() _ = w.Close()
return nil, fmt.Errorf("writing data: %w", err) return nil, fmt.Errorf("writing data: %w", err)
} }
@@ -68,6 +69,7 @@ func CompressStream(dst io.Writer, src io.Reader, compressionLevel int, recipien
if err := w.Close(); err != nil { if err := w.Close(); err != nil {
return 0, "", fmt.Errorf("closing writer: %w", err) return 0, "", fmt.Errorf("closing writer: %w", err)
} }
closed = true closed = true
return w.BytesWritten(), hex.EncodeToString(w.Sum256()), nil return w.BytesWritten(), hex.EncodeToString(w.Sum256()), nil

View File

@@ -20,13 +20,14 @@ const testRecipient = "age1cplgrwj77ta54dnmydvvmzn64ltk83ankxl5sww04mrtmu62kv3s8
// cause a double close. // cause a double close.
func TestCompressStreamNoDoubleClose(t *testing.T) { func TestCompressStreamNoDoubleClose(t *testing.T) {
input := []byte("regression test data for issue #28 double-close fix") input := []byte("regression test data for issue #28 double-close fix")
var buf bytes.Buffer var buf bytes.Buffer
written, hash, err := CompressStream(&buf, bytes.NewReader(input), 3, []string{testRecipient}) written, hash, err := CompressStream(&buf, bytes.NewReader(input), 3, []string{testRecipient})
require.NoError(t, err, "CompressStream should not return an error") require.NoError(t, err, "CompressStream should not return an error")
assert.True(t, written > 0, "expected bytes written > 0") assert.Positive(t, written, "expected bytes written > 0")
assert.NotEmpty(t, hash, "expected non-empty hash") assert.NotEmpty(t, hash, "expected non-empty hash")
assert.True(t, buf.Len() > 0, "expected non-empty output") assert.Positive(t, buf.Len(), "expected non-empty output")
} }
// TestCompressStreamLargeInput exercises CompressStream with a larger payload // TestCompressStreamLargeInput exercises CompressStream with a larger payload
@@ -37,9 +38,10 @@ func TestCompressStreamLargeInput(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
var buf bytes.Buffer var buf bytes.Buffer
written, hash, err := CompressStream(&buf, bytes.NewReader(data), 3, []string{testRecipient}) written, hash, err := CompressStream(&buf, bytes.NewReader(data), 3, []string{testRecipient})
require.NoError(t, err) require.NoError(t, err)
assert.True(t, written > 0) assert.Positive(t, written)
assert.NotEmpty(t, hash) assert.NotEmpty(t, hash)
} }
@@ -47,6 +49,7 @@ func TestCompressStreamLargeInput(t *testing.T) {
// without double-close issues. // without double-close issues.
func TestCompressStreamEmptyInput(t *testing.T) { func TestCompressStreamEmptyInput(t *testing.T) {
var buf bytes.Buffer var buf bytes.Buffer
_, hash, err := CompressStream(&buf, strings.NewReader(""), 3, []string{testRecipient}) _, hash, err := CompressStream(&buf, strings.NewReader(""), 3, []string{testRecipient})
require.NoError(t, err) require.NoError(t, err)
assert.NotEmpty(t, hash) assert.NotEmpty(t, hash)
@@ -58,7 +61,7 @@ func TestCompressDataNoDoubleClose(t *testing.T) {
input := []byte("CompressData regression test for double-close") input := []byte("CompressData regression test for double-close")
result, err := CompressData(input, 3, []string{testRecipient}) result, err := CompressData(input, 3, []string{testRecipient})
require.NoError(t, err) require.NoError(t, err)
assert.True(t, result.CompressedSize > 0) assert.Positive(t, result.CompressedSize)
assert.True(t, result.UncompressedSize == int64(len(input))) assert.Equal(t, result.UncompressedSize, int64(len(input)))
assert.NotEmpty(t, result.SHA256) assert.NotEmpty(t, result.SHA256)
} }

View File

@@ -53,12 +53,14 @@ func NewReader(r io.Reader, identity age.Identity) (*Reader, error) {
func (r *Reader) Read(p []byte) (n int, err error) { func (r *Reader) Read(p []byte) (n int, err error) {
n, err = r.teeReader.Read(p) n, err = r.teeReader.Read(p)
r.bytesRead += int64(n) r.bytesRead += int64(n)
return n, err return n, err
} }
// Close closes the decompressor // Close closes the decompressor
func (r *Reader) Close() error { func (r *Reader) Close() error {
r.decompressor.Close() r.decompressor.Close()
return nil return nil
} }

View File

@@ -36,11 +36,13 @@ func NewWriter(w io.Writer, compressionLevel int, recipients []string) (*Writer,
// Parse recipients // Parse recipients
var ageRecipients []age.Recipient var ageRecipients []age.Recipient
for _, recipient := range recipients { for _, recipient := range recipients {
r, err := age.ParseX25519Recipient(recipient) r, err := age.ParseX25519Recipient(recipient)
if err != nil { if err != nil {
return nil, fmt.Errorf("parsing recipient %s: %w", recipient, err) return nil, fmt.Errorf("parsing recipient %s: %w", recipient, err)
} }
ageRecipients = append(ageRecipients, r) ageRecipients = append(ageRecipients, r)
} }
@@ -51,10 +53,7 @@ func NewWriter(w io.Writer, compressionLevel int, recipients []string) (*Writer,
} }
// Calculate compression concurrency: CPUs - 2, minimum 1 // Calculate compression concurrency: CPUs - 2, minimum 1
concurrency := runtime.NumCPU() - 2 concurrency := max(runtime.NumCPU()-2, 1)
if concurrency < 1 {
concurrency = 1
}
// Create compression writer with encryption as destination // Create compression writer with encryption as destination
compressor, err := zstd.NewWriter(encWriter, compressor, err := zstd.NewWriter(encWriter,
@@ -63,6 +62,7 @@ func NewWriter(w io.Writer, compressionLevel int, recipients []string) (*Writer,
) )
if err != nil { if err != nil {
_ = encWriter.Close() _ = encWriter.Close()
return nil, fmt.Errorf("creating compression writer: %w", err) return nil, fmt.Errorf("creating compression writer: %w", err)
} }
@@ -82,18 +82,21 @@ func NewWriter(w io.Writer, compressionLevel int, recipients []string) (*Writer,
func (w *Writer) Write(p []byte) (n int, err error) { func (w *Writer) Write(p []byte) (n int, err error) {
n, err = w.teeWriter.Write(p) n, err = w.teeWriter.Write(p)
w.bytesWritten += int64(n) w.bytesWritten += int64(n)
return n, err return n, err
} }
// Close closes all layers and returns any errors // Close closes all layers and returns any errors
func (w *Writer) Close() error { func (w *Writer) Close() error {
// Close compressor first // Close compressor first
if err := w.compressor.Close(); err != nil { err := w.compressor.Close()
if err != nil {
return fmt.Errorf("closing compressor: %w", err) return fmt.Errorf("closing compressor: %w", err)
} }
// Then close encryptor // Then close encryptor
if err := w.encryptor.Close(); err != nil { err = w.encryptor.Close()
if err != nil {
return fmt.Errorf("closing encryptor: %w", err) return fmt.Errorf("closing encryptor: %w", err)
} }
@@ -109,6 +112,7 @@ func (w *Writer) Sum256() []byte {
firstHash := w.hasher.Sum(nil) firstHash := w.hasher.Sum(nil)
// Second hash: SHA256(firstHash) - this is the blob ID // Second hash: SHA256(firstHash) - this is the blob ID
secondHash := sha256.Sum256(firstHash) secondHash := sha256.Sum256(firstHash)
return secondHash[:] return secondHash[:]
} }
@@ -123,5 +127,6 @@ func validateCompressionLevel(level int) error {
if level < 1 || level > 19 { if level < 1 || level > 19 {
return fmt.Errorf("invalid compression level %d: must be between 1 and 19", level) return fmt.Errorf("invalid compression level %d: must be between 1 and 19", level)
} }
return nil return nil
} }

View File

@@ -3,6 +3,7 @@ package chunker
import ( import (
"crypto/sha256" "crypto/sha256"
"encoding/hex" "encoding/hex"
"errors"
"fmt" "fmt"
"io" "io"
"os" "os"
@@ -50,13 +51,15 @@ func (c *Chunker) ChunkReader(r io.Reader) ([]Chunk, error) {
defer chunker.Release() defer chunker.Release()
var chunks []Chunk var chunks []Chunk
offset := int64(0) offset := int64(0)
for { for {
chunk, err := chunker.Next() chunk, err := chunker.Next()
if err == io.EOF { if errors.Is(err, io.EOF) {
break break
} }
if err != nil { if err != nil {
return nil, fmt.Errorf("reading chunk: %w", err) return nil, fmt.Errorf("reading chunk: %w", err)
} }
@@ -104,9 +107,10 @@ func (c *Chunker) ChunkReaderStreaming(r io.Reader, callback ChunkCallback) (str
for { for {
chunk, err := chunker.Next() chunk, err := chunker.Next()
if err == io.EOF { if errors.Is(err, io.EOF) {
break break
} }
if err != nil { if err != nil {
return "", fmt.Errorf("reading chunk: %w", err) return "", fmt.Errorf("reading chunk: %w", err)
} }
@@ -143,7 +147,8 @@ func (c *Chunker) ChunkFile(path string) ([]Chunk, error) {
return nil, fmt.Errorf("opening file: %w", err) return nil, fmt.Errorf("opening file: %w", err)
} }
defer func() { defer func() {
if err := file.Close(); err != nil && err.Error() != "invalid argument" { err := file.Close()
if err != nil && err.Error() != "invalid argument" {
// Log error or handle as needed // Log error or handle as needed
_ = err _ = err
} }

View File

@@ -42,7 +42,7 @@ func TestChunkerExpectedChunkCount(t *testing.T) {
// Create data with some variation to trigger chunk boundaries // Create data with some variation to trigger chunk boundaries
data := make([]byte, tt.fileSize) data := make([]byte, tt.fileSize)
for i := 0; i < len(data); i++ { for i := range data {
// Use a pattern that should create boundaries // Use a pattern that should create boundaries
data[i] = byte((i * 17) ^ (i >> 5)) data[i] = byte((i * 17) ^ (i >> 5))
} }
@@ -59,6 +59,7 @@ func TestChunkerExpectedChunkCount(t *testing.T) {
t.Errorf("too few chunks: got %d, expected at least %d", t.Errorf("too few chunks: got %d, expected at least %d",
len(chunks), tt.minExpected) len(chunks), tt.minExpected)
} }
if len(chunks) > tt.maxExpected { if len(chunks) > tt.maxExpected {
t.Errorf("too many chunks: got %d, expected at most %d", t.Errorf("too many chunks: got %d, expected at most %d",
len(chunks), tt.maxExpected) len(chunks), tt.maxExpected)
@@ -69,6 +70,7 @@ func TestChunkerExpectedChunkCount(t *testing.T) {
for _, chunk := range chunks { for _, chunk := range chunks {
reconstructed = append(reconstructed, chunk.Data...) reconstructed = append(reconstructed, chunk.Data...)
} }
if !bytes.Equal(data, reconstructed) { if !bytes.Equal(data, reconstructed) {
t.Error("reconstructed data doesn't match original") t.Error("reconstructed data doesn't match original")
} }

View File

@@ -60,6 +60,7 @@ func TestChunker(t *testing.T) {
if chunk.Offset != expectedOffset { if chunk.Offset != expectedOffset {
t.Errorf("chunk %d: expected offset %d, got %d", i, expectedOffset, chunk.Offset) t.Errorf("chunk %d: expected offset %d, got %d", i, expectedOffset, chunk.Offset)
} }
expectedOffset += chunk.Size expectedOffset += chunk.Size
} }
}) })
@@ -90,6 +91,7 @@ func TestChunker(t *testing.T) {
if chunks1[i].Hash != chunks2[i].Hash { if chunks1[i].Hash != chunks2[i].Hash {
t.Errorf("chunk %d: different hashes", i) t.Errorf("chunk %d: different hashes", i)
} }
if chunks1[i].Size != chunks2[i].Size { if chunks1[i].Size != chunks2[i].Size {
t.Errorf("chunk %d: different sizes", i) t.Errorf("chunk %d: different sizes", i)
} }
@@ -121,6 +123,7 @@ func TestChunkBoundaries(t *testing.T) {
if i < len(chunks)-1 && chunk.Size < minSize { if i < len(chunks)-1 && chunk.Size < minSize {
t.Errorf("chunk %d size %d is below minimum %d", i, chunk.Size, minSize) t.Errorf("chunk %d size %d is below minimum %d", i, chunk.Size, minSize)
} }
if chunk.Size > maxSize { if chunk.Size > maxSize {
t.Errorf("chunk %d size %d exceeds maximum %d", i, chunk.Size, maxSize) t.Errorf("chunk %d size %d exceeds maximum %d", i, chunk.Size, maxSize)
} }

View File

@@ -28,7 +28,7 @@ type ReusableChunker struct {
// reusableChunkerPool pools ReusableChunker instances to avoid allocations. // reusableChunkerPool pools ReusableChunker instances to avoid allocations.
var reusableChunkerPool = sync.Pool{ var reusableChunkerPool = sync.Pool{
New: func() interface{} { New: func() any {
return &ReusableChunker{} return &ReusableChunker{}
}, },
} }
@@ -39,17 +39,20 @@ var bufferPools = sync.Map{}
func getBuffer(size int) []byte { func getBuffer(size int) []byte {
poolI, _ := bufferPools.LoadOrStore(size, &sync.Pool{ poolI, _ := bufferPools.LoadOrStore(size, &sync.Pool{
New: func() interface{} { New: func() any {
buf := make([]byte, size) buf := make([]byte, size)
return &buf return &buf
}, },
}) })
pool := poolI.(*sync.Pool) pool := poolI.(*sync.Pool)
return *pool.Get().(*[]byte) return *pool.Get().(*[]byte)
} }
func putBuffer(buf []byte) { func putBuffer(buf []byte) {
size := cap(buf) size := cap(buf)
poolI, ok := bufferPools.Load(size) poolI, ok := bufferPools.Load(size)
if ok { if ok {
pool := poolI.(*sync.Pool) pool := poolI.(*sync.Pool)
@@ -77,6 +80,7 @@ func AcquireReusableChunker(rd io.Reader, minSize, avgSize, maxSize int) *Reusab
if c.buf != nil { if c.buf != nil {
putBuffer(c.buf) putBuffer(c.buf)
} }
c.buf = getBuffer(bufSize) c.buf = getBuffer(bufSize)
} else { } else {
// Restore buffer to full capacity (may have been truncated by previous EOF) // Restore buffer to full capacity (may have been truncated by previous EOF)
@@ -120,6 +124,7 @@ func (c *ReusableChunker) fillBuffer() error {
if c.eof { if c.eof {
c.buf = c.buf[:n] c.buf = c.buf[:n]
return nil return nil
} }
@@ -134,15 +139,18 @@ func (c *ReusableChunker) fillBuffer() error {
} else if err != nil { } else if err != nil {
return err return err
} }
return nil return nil
} }
// Next returns the next chunk or io.EOF when done. // Next returns the next chunk or io.EOF when done.
// The returned Data slice is only valid until the next call to Next. // The returned Data slice is only valid until the next call to Next.
func (c *ReusableChunker) Next() (FastCDCChunk, error) { func (c *ReusableChunker) Next() (FastCDCChunk, error) {
if err := c.fillBuffer(); err != nil { err := c.fillBuffer()
if err != nil {
return FastCDCChunk{}, err return FastCDCChunk{}, err
} }
if len(c.buf) == 0 { if len(c.buf) == 0 {
return FastCDCChunk{}, io.EOF return FastCDCChunk{}, io.EOF
} }
@@ -189,13 +197,6 @@ func (c *ReusableChunker) nextChunk(data []byte) (int, uint64) {
return i, fp return i, fp
} }
func min(a, b int) int {
if a < b {
return a
}
return b
}
// 256 random uint64s for the rolling hash function (from FastCDC paper) // 256 random uint64s for the rolling hash function (from FastCDC paper)
var table = [256]uint64{ var table = [256]uint64{
0xe80e8d55032474b3, 0x11b25b61f5924e15, 0x03aa5bd82a9eb669, 0xc45a153ef107a38c, 0xe80e8d55032474b3, 0x11b25b61f5924e15, 0x03aa5bd82a9eb669, 0xc45a153ef107a38c,

View File

@@ -44,9 +44,11 @@ func setupGlobals(lc fx.Lifecycle, g *globals.Globals, v *vaultik.Vaultik, opts
lc.Append(fx.Hook{ lc.Append(fx.Hook{
OnStart: func(ctx context.Context) error { OnStart: func(ctx context.Context) error {
g.StartTime = time.Now().UTC() g.StartTime = time.Now().UTC()
if opts.Cron || opts.Quiet { if opts.Cron || opts.Quiet {
v.UI.SetQuiet(true) v.UI.SetQuiet(true)
} }
return nil return nil
}, },
}) })
@@ -56,12 +58,12 @@ func setupGlobals(lc fx.Lifecycle, g *globals.Globals, v *vaultik.Vaultik, opts
// blank line. Used both from the fx hook (for subcommand invocations) and // blank line. Used both from the fx hook (for subcommand invocations) and
// from the root cobra Run handler (for `vaultik` with no subcommand). // from the root cobra Run handler (for `vaultik` with no subcommand).
func writeStartupBanner(w *ui.Writer, startTime time.Time, shortCommit string) { func writeStartupBanner(w *ui.Writer, startTime time.Time, shortCommit string) {
w.Banner("%s %s by %s (commit %s, built on %s) starting up at %s.", w.Bannerf("%s %s by %s (commit %s, built on %s) starting up at %s.",
globals.Appname, globals.Version, globals.Author, globals.Appname, globals.Version, globals.Author,
shortCommit, globals.CommitDate, shortCommit, globals.CommitDate,
startTime.Format(time.RFC3339)) startTime.Format(time.RFC3339))
w.Banner("%s", globals.Homepage) w.Bannerf("%s", globals.Homepage)
w.Banner("") w.Bannerf("")
} }
// NewApp creates a new fx application with common modules. // NewApp creates a new fx application with common modules.
@@ -105,6 +107,7 @@ func cleanStartupError(err error) error {
if idx := strings.LastIndex(msg, "): "); idx >= 0 { if idx := strings.LastIndex(msg, "): "); idx >= 0 {
msg = msg[idx+3:] msg = msg[idx+3:]
} }
return errors.New(msg) return errors.New(msg)
} }
@@ -122,7 +125,8 @@ func RunApp(ctx context.Context, app *fx.App) error {
defer cancel() defer cancel()
// Start the app // Start the app
if err := app.Start(ctx); err != nil { err := app.Start(ctx)
if err != nil {
return cleanStartupError(err) return cleanStartupError(err)
} }
@@ -130,6 +134,7 @@ func RunApp(ctx context.Context, app *fx.App) error {
shutdownComplete := make(chan struct{}) shutdownComplete := make(chan struct{})
go func() { go func() {
defer close(shutdownComplete) defer close(shutdownComplete)
<-sigChan <-sigChan
log.Notice("Received interrupt signal, shutting down gracefully...") log.Notice("Received interrupt signal, shutting down gracefully...")
@@ -137,7 +142,8 @@ func RunApp(ctx context.Context, app *fx.App) error {
shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 30*time.Second) shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 30*time.Second)
defer shutdownCancel() defer shutdownCancel()
if err := app.Stop(shutdownCtx); err != nil { err := app.Stop(shutdownCtx)
if err != nil {
log.Error("Error during shutdown", "error", err) log.Error("Error during shutdown", "error", err)
} }
}() }()
@@ -149,9 +155,11 @@ func RunApp(ctx context.Context, app *fx.App) error {
return nil return nil
case <-ctx.Done(): case <-ctx.Done():
// Context cancelled (shouldn't happen in normal operation) // Context cancelled (shouldn't happen in normal operation)
if err := app.Stop(context.Background()); err != nil { err := app.Stop(context.Background())
if err != nil {
log.Error("Error stopping app", "error", err) log.Error("Error stopping app", "error", err)
} }
return ctx.Err() return ctx.Err()
case <-app.Done(): case <-app.Done():
// App finished running (e.g., backup completed) // App finished running (e.g., backup completed)
@@ -166,19 +174,24 @@ func RunApp(ctx context.Context, app *fx.App) error {
func RunWithApp(ctx context.Context, opts AppOptions) error { func RunWithApp(ctx context.Context, opts AppOptions) error {
// Acquire PID lock to prevent concurrent instances // Acquire PID lock to prevent concurrent instances
lockDir := filepath.Join(xdg.DataHome, "vaultik") lockDir := filepath.Join(xdg.DataHome, "vaultik")
lock, err := pidlock.Acquire(lockDir) lock, err := pidlock.Acquire(lockDir)
if err != nil { if err != nil {
if errors.Is(err, pidlock.ErrAlreadyRunning) { if errors.Is(err, pidlock.ErrAlreadyRunning) {
return fmt.Errorf("cannot start: %w", err) return fmt.Errorf("cannot start: %w", err)
} }
return fmt.Errorf("failed to acquire lock: %w", err) return fmt.Errorf("failed to acquire lock: %w", err)
} }
defer func() { defer func() {
if err := lock.Release(); err != nil { err := lock.Release()
if err != nil {
log.Warn("Failed to release PID lock", "error", err) log.Warn("Failed to release PID lock", "error", err)
} }
}() }()
app := NewApp(opts) app := NewApp(opts)
return RunApp(ctx, app) return RunApp(ctx, app)
} }

View File

@@ -1,6 +1,7 @@
package cli package cli
import ( import (
"errors"
"fmt" "fmt"
"os" "os"
"os/exec" "os/exec"
@@ -240,16 +241,20 @@ on macOS, ~/.config/ on Linux, /etc/vaultik/ as root).`,
} }
dir := filepath.Dir(path) dir := filepath.Dir(path)
if err := os.MkdirAll(dir, 0o755); err != nil {
err := os.MkdirAll(dir, 0o755)
if err != nil {
return fmt.Errorf("creating config directory %s: %w", dir, err) return fmt.Errorf("creating config directory %s: %w", dir, err)
} }
if err := os.WriteFile(path, []byte(defaultConfigTemplate), 0o600); err != nil { err = os.WriteFile(path, []byte(defaultConfigTemplate), 0o600)
if err != nil {
return fmt.Errorf("writing config file: %w", err) return fmt.Errorf("writing config file: %w", err)
} }
fmt.Printf("Config written to %s\n", path) fmt.Printf("Config written to %s\n", path)
fmt.Println("Edit it to set your age_recipients, snapshots, and storage_url.") fmt.Println("Edit it to set your age_recipients, snapshots, and storage_url.")
return nil return nil
}, },
} }
@@ -276,6 +281,7 @@ func newConfigEditCommand() *cobra.Command {
ed.Stdin = os.Stdin ed.Stdin = os.Stdin
ed.Stdout = os.Stdout ed.Stdout = os.Stdout
ed.Stderr = os.Stderr ed.Stderr = os.Stderr
return ed.Run() return ed.Run()
}, },
} }
@@ -305,6 +311,7 @@ func newConfigGetCommand() *cobra.Command {
if node.Kind == yaml.ScalarNode { if node.Kind == yaml.ScalarNode {
fmt.Println(node.Value) fmt.Println(node.Value)
return nil return nil
} }
@@ -312,7 +319,9 @@ func newConfigGetCommand() *cobra.Command {
if err != nil { if err != nil {
return fmt.Errorf("marshaling value: %w", err) return fmt.Errorf("marshaling value: %w", err)
} }
fmt.Print(string(out)) fmt.Print(string(out))
return nil return nil
}, },
} }
@@ -363,6 +372,7 @@ Examples:
} }
fmt.Printf("%s = %s\n", args[0], args[1]) fmt.Printf("%s = %s\n", args[0], args[1])
return nil return nil
}, },
} }
@@ -399,8 +409,9 @@ func yamlPathGet(root *yaml.Node, keys []string) (*yaml.Node, error) {
node := root node := root
if node.Kind == yaml.DocumentNode { if node.Kind == yaml.DocumentNode {
if len(node.Content) == 0 { if len(node.Content) == 0 {
return nil, fmt.Errorf("empty config file") return nil, errors.New("empty config file")
} }
node = node.Content[0] node = node.Content[0]
} }
@@ -408,13 +419,16 @@ func yamlPathGet(root *yaml.Node, keys []string) (*yaml.Node, error) {
switch node.Kind { switch node.Kind {
case yaml.MappingNode: case yaml.MappingNode:
found := false found := false
for j := 0; j+1 < len(node.Content); j += 2 { for j := 0; j+1 < len(node.Content); j += 2 {
if node.Content[j].Value == key { if node.Content[j].Value == key {
node = node.Content[j+1] node = node.Content[j+1]
found = true found = true
break break
} }
} }
if !found { if !found {
return nil, fmt.Errorf("key not found: %s", strings.Join(keys[:i+1], ".")) return nil, fmt.Errorf("key not found: %s", strings.Join(keys[:i+1], "."))
} }
@@ -423,9 +437,11 @@ func yamlPathGet(root *yaml.Node, keys []string) (*yaml.Node, error) {
if err != nil { if err != nil {
return nil, fmt.Errorf("key %q is a list; use a numeric index", strings.Join(keys[:i], ".")) return nil, fmt.Errorf("key %q is a list; use a numeric index", strings.Join(keys[:i], "."))
} }
if idx < 0 || idx >= len(node.Content) { if idx < 0 || idx >= len(node.Content) {
return nil, fmt.Errorf("index %d out of range for %s (len %d)", idx, strings.Join(keys[:i], "."), len(node.Content)) return nil, fmt.Errorf("index %d out of range for %s (len %d)", idx, strings.Join(keys[:i], "."), len(node.Content))
} }
node = node.Content[idx] node = node.Content[idx]
default: default:
return nil, fmt.Errorf("key %q is not a map or list", strings.Join(keys[:i], ".")) return nil, fmt.Errorf("key %q is not a map or list", strings.Join(keys[:i], "."))
@@ -445,6 +461,7 @@ func yamlPathSet(root *yaml.Node, keys []string, value string) error {
if len(node.Content) == 0 { if len(node.Content) == 0 {
node.Content = []*yaml.Node{{Kind: yaml.MappingNode}} node.Content = []*yaml.Node{{Kind: yaml.MappingNode}}
} }
node = node.Content[0] node = node.Content[0]
} }
@@ -454,19 +471,23 @@ func yamlPathSet(root *yaml.Node, keys []string, value string) error {
switch node.Kind { switch node.Kind {
case yaml.MappingNode: case yaml.MappingNode:
var valueNode *yaml.Node var valueNode *yaml.Node
for j := 0; j+1 < len(node.Content); j += 2 { for j := 0; j+1 < len(node.Content); j += 2 {
if node.Content[j].Value == key { if node.Content[j].Value == key {
valueNode = node.Content[j+1] valueNode = node.Content[j+1]
break break
} }
} }
if valueNode == nil { if valueNode == nil {
keyNode := &yaml.Node{Kind: yaml.ScalarNode, Value: key} keyNode := &yaml.Node{Kind: yaml.ScalarNode, Value: key}
valueNode = &yaml.Node{Kind: yaml.MappingNode} valueNode = &yaml.Node{Kind: yaml.MappingNode}
if last { if last {
valueNode = &yaml.Node{Kind: yaml.ScalarNode, Value: value} valueNode = &yaml.Node{Kind: yaml.ScalarNode, Value: value}
} }
node.Content = append(node.Content, keyNode, valueNode) node.Content = append(node.Content, keyNode, valueNode)
} else if last { } else if last {
setScalar(valueNode, value) setScalar(valueNode, value)
@@ -479,18 +500,22 @@ func yamlPathSet(root *yaml.Node, keys []string, value string) error {
if err != nil { if err != nil {
return fmt.Errorf("key %q is a list; use a numeric index", strings.Join(keys[:i], ".")) return fmt.Errorf("key %q is a list; use a numeric index", strings.Join(keys[:i], "."))
} }
if idx < 0 || idx > len(node.Content) { if idx < 0 || idx > len(node.Content) {
return fmt.Errorf("index %d out of range for %s (len %d)", idx, strings.Join(keys[:i], "."), len(node.Content)) return fmt.Errorf("index %d out of range for %s (len %d)", idx, strings.Join(keys[:i], "."), len(node.Content))
} }
if idx == len(node.Content) { if idx == len(node.Content) {
newNode := &yaml.Node{Kind: yaml.MappingNode} newNode := &yaml.Node{Kind: yaml.MappingNode}
if last { if last {
newNode = &yaml.Node{Kind: yaml.ScalarNode, Value: value} newNode = &yaml.Node{Kind: yaml.ScalarNode, Value: value}
} }
node.Content = append(node.Content, newNode) node.Content = append(node.Content, newNode)
} else if last { } else if last {
setScalar(node.Content[idx], value) setScalar(node.Content[idx], value)
} }
node = node.Content[idx] node = node.Content[idx]
default: default:
@@ -516,8 +541,10 @@ func configPathForInit() string {
if rootFlags.ConfigPath != "" { if rootFlags.ConfigPath != "" {
return rootFlags.ConfigPath return rootFlags.ConfigPath
} }
if envPath := os.Getenv("VAULTIK_CONFIG"); envPath != "" { if envPath := os.Getenv("VAULTIK_CONFIG"); envPath != "" {
return envPath return envPath
} }
return DefaultConfigPath() return DefaultConfigPath()
} }

View File

@@ -12,7 +12,9 @@ import (
// that unmarshals into the Config struct with the expected snapshots. // that unmarshals into the Config struct with the expected snapshots.
func TestDefaultConfigTemplateParses(t *testing.T) { func TestDefaultConfigTemplateParses(t *testing.T) {
var cfg config.Config var cfg config.Config
if err := yaml.Unmarshal([]byte(defaultConfigTemplate), &cfg); err != nil {
err := yaml.Unmarshal([]byte(defaultConfigTemplate), &cfg)
if err != nil {
t.Fatalf("default config template is not valid YAML: %v", err) t.Fatalf("default config template is not valid YAML: %v", err)
} }
@@ -24,9 +26,11 @@ func TestDefaultConfigTemplateParses(t *testing.T) {
if !ok { if !ok {
t.Fatal("expected 'home' snapshot in default config") t.Fatal("expected 'home' snapshot in default config")
} }
if len(home.Paths) == 0 { if len(home.Paths) == 0 {
t.Error("home snapshot should have at least one path") t.Error("home snapshot should have at least one path")
} }
if len(home.Exclude) == 0 { if len(home.Exclude) == 0 {
t.Error("home snapshot should have exclude patterns") t.Error("home snapshot should have exclude patterns")
} }
@@ -35,9 +39,11 @@ func TestDefaultConfigTemplateParses(t *testing.T) {
if !ok { if !ok {
t.Fatal("expected 'apps' snapshot in default config") t.Fatal("expected 'apps' snapshot in default config")
} }
if len(apps.Paths) != 1 || apps.Paths[0] != "/Applications" { if len(apps.Paths) != 1 || apps.Paths[0] != "/Applications" {
t.Errorf("apps snapshot should back up /Applications, got %v", apps.Paths) t.Errorf("apps snapshot should back up /Applications, got %v", apps.Paths)
} }
if len(apps.Exclude) == 0 { if len(apps.Exclude) == 0 {
t.Error("apps snapshot should have exclude patterns") t.Error("apps snapshot should have exclude patterns")
} }
@@ -58,10 +64,14 @@ snapshots:
func parseTestYAML(t *testing.T) *yaml.Node { func parseTestYAML(t *testing.T) *yaml.Node {
t.Helper() t.Helper()
var root yaml.Node var root yaml.Node
if err := yaml.Unmarshal([]byte(testYAML), &root); err != nil {
err := yaml.Unmarshal([]byte(testYAML), &root)
if err != nil {
t.Fatalf("parsing test yaml: %v", err) t.Fatalf("parsing test yaml: %v", err)
} }
return &root return &root
} }
@@ -91,11 +101,14 @@ func TestYAMLPathGet(t *testing.T) {
if err == nil { if err == nil {
t.Fatalf("expected error for %q", tt.path) t.Fatalf("expected error for %q", tt.path)
} }
return return
} }
if err != nil { if err != nil {
t.Fatalf("unexpected error: %v", err) t.Fatalf("unexpected error: %v", err)
} }
if node.Value != tt.want { if node.Value != tt.want {
t.Errorf("get %q = %q, want %q", tt.path, node.Value, tt.want) t.Errorf("get %q = %q, want %q", tt.path, node.Value, tt.want)
} }
@@ -115,6 +128,7 @@ func TestYAMLPathSet(t *testing.T) {
if err := yamlPathSet(root, splitPath("s3.endpoint"), "s3.example.com"); err != nil { if err := yamlPathSet(root, splitPath("s3.endpoint"), "s3.example.com"); err != nil {
t.Fatalf("set s3.endpoint: %v", err) t.Fatalf("set s3.endpoint: %v", err)
} }
if err := yamlPathSet(root, splitPath("newmap.newkey"), "val"); err != nil { if err := yamlPathSet(root, splitPath("newmap.newkey"), "val"); err != nil {
t.Fatalf("set newmap.newkey: %v", err) t.Fatalf("set newmap.newkey: %v", err)
} }
@@ -123,9 +137,11 @@ func TestYAMLPathSet(t *testing.T) {
if err := yamlPathSet(root, splitPath("age_recipients.0"), "age1bbb"); err != nil { if err := yamlPathSet(root, splitPath("age_recipients.0"), "age1bbb"); err != nil {
t.Fatalf("set age_recipients.0: %v", err) t.Fatalf("set age_recipients.0: %v", err)
} }
if err := yamlPathSet(root, splitPath("age_recipients.1"), "age1ccc"); err != nil { if err := yamlPathSet(root, splitPath("age_recipients.1"), "age1ccc"); err != nil {
t.Fatalf("append age_recipients.1: %v", err) t.Fatalf("append age_recipients.1: %v", err)
} }
if err := yamlPathSet(root, splitPath("age_recipients.5"), "age1ddd"); err == nil { if err := yamlPathSet(root, splitPath("age_recipients.5"), "age1ddd"); err == nil {
t.Error("expected out-of-range append to fail") t.Error("expected out-of-range append to fail")
} }
@@ -135,6 +151,7 @@ func TestYAMLPathSet(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("marshal: %v", err) t.Fatalf("marshal: %v", err)
} }
text := string(out) text := string(out)
for _, want := range []string{"newbucket", "s3.example.com", "newkey: val", "# top comment", "# inline comment", "age1bbb", "age1ccc"} { for _, want := range []string{"newbucket", "s3.example.com", "newkey: val", "# top comment", "# inline comment", "age1bbb", "age1ccc"} {
@@ -147,6 +164,7 @@ func TestYAMLPathSet(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("get after set: %v", err) t.Fatalf("get after set: %v", err)
} }
if got.Value != "newbucket" { if got.Value != "newbucket" {
t.Errorf("s3.bucket = %q after set, want newbucket", got.Value) t.Errorf("s3.bucket = %q after set, want newbucket", got.Value)
} }

View File

@@ -66,6 +66,7 @@ Use --force to skip the confirmation prompt.`,
// Check if database exists // Check if database exists
if _, err := os.Stat(dbPath); os.IsNotExist(err) { if _, err := os.Stat(dbPath); os.IsNotExist(err) {
fmt.Printf("Database does not exist: %s\n", dbPath) fmt.Printf("Database does not exist: %s\n", dbPath)
return nil return nil
} }
@@ -73,9 +74,11 @@ Use --force to skip the confirmation prompt.`,
if !force { if !force {
fmt.Printf("This will delete the local state database at:\n %s\n\n", dbPath) fmt.Printf("This will delete the local state database at:\n %s\n\n", dbPath)
fmt.Print("Are you sure? Type 'yes' to confirm: ") fmt.Print("Are you sure? Type 'yes' to confirm: ")
var confirm string var confirm string
if _, err := fmt.Scanln(&confirm); err != nil || confirm != "yes" { if _, err := fmt.Scanln(&confirm); err != nil || confirm != "yes" {
fmt.Println("Aborted.") fmt.Println("Aborted.")
return nil return nil
} }
} }
@@ -97,6 +100,7 @@ Use --force to skip the confirmation prompt.`,
} }
log.Info("Local state database deleted", "path", dbPath) log.Info("Local state database deleted", "path", dbPath)
return nil return nil
}, },
} }

View File

@@ -1,6 +1,7 @@
package cli package cli
import ( import (
"errors"
"fmt" "fmt"
"regexp" "regexp"
"strconv" "strconv"
@@ -25,7 +26,7 @@ func parseDuration(s string) (time.Duration, error) {
// Extended duration parsing // Extended duration parsing
// Check for negative values // Check for negative values
if strings.HasPrefix(strings.TrimSpace(s), "-") { if strings.HasPrefix(strings.TrimSpace(s), "-") {
return 0, fmt.Errorf("negative durations are not supported") return 0, errors.New("negative durations are not supported")
} }
// Pattern matches: number + unit, repeated // Pattern matches: number + unit, repeated
@@ -48,6 +49,7 @@ func parseDuration(s string) (time.Duration, error) {
} }
var d time.Duration var d time.Duration
switch unit { switch unit {
// Standard time units // Standard time units
case "ns", "nanosecond", "nanoseconds": case "ns", "nanosecond", "nanoseconds":
@@ -75,7 +77,7 @@ func parseDuration(s string) (time.Duration, error) {
d = time.Duration(value * float64(365*24*time.Hour)) d = time.Duration(value * float64(365*24*time.Hour))
default: default:
// Try parsing as standard Go duration unit // Try parsing as standard Go duration unit
testStr := fmt.Sprintf("1%s", unit) testStr := "1" + unit
if _, err := time.ParseDuration(testStr); err == nil { if _, err := time.ParseDuration(testStr); err == nil {
// It's a valid Go duration unit, parse the full value // It's a valid Go duration unit, parse the full value
fullStr := fmt.Sprintf("%g%s", value, unit) fullStr := fmt.Sprintf("%g%s", value, unit)

View File

@@ -185,6 +185,7 @@ func TestParseDuration(t *testing.T) {
if tt.wantErr { if tt.wantErr {
assert.Error(t, err, "expected error for input %q", tt.input) assert.Error(t, err, "expected error for input %q", tt.input)
return return
} }

View File

@@ -19,24 +19,26 @@ func CLIEntry() {
if len(short) > 12 { if len(short) > 12 {
short = short[:12] short = short[:12]
} }
writeStartupBanner(ui.New(os.Stdout), time.Now().UTC(), short) writeStartupBanner(ui.New(os.Stdout), time.Now().UTC(), short)
} }
rootCmd := NewRootCommand() rootCmd := NewRootCommand()
rootCmd.SilenceErrors = true rootCmd.SilenceErrors = true
if err := rootCmd.Execute(); err != nil { err := rootCmd.Execute()
ReportError("%s", err.Error()) if err != nil {
ReportErrorf("%s", err.Error())
os.Exit(1) os.Exit(1)
} }
} }
// ReportError emits a user-facing error to stderr in the standard // ReportErrorf emits a user-facing error to stderr in the standard
// 🛑 ERROR: format. Use it from goroutine error paths (where returning // 🛑 ERROR: format. Use it from goroutine error paths (where returning
// an error to cobra isn't an option) and anywhere else a CLI command // an error to cobra isn't an option) and anywhere else a CLI command
// must surface a failure outside the normal RunE return path. // must surface a failure outside the normal RunE return path.
func ReportError(format string, args ...any) { func ReportErrorf(format string, args ...any) {
ui.New(os.Stderr).Error(format, args...) ui.New(os.Stderr).Errorf(format, args...)
} }
// bannerSuppressedInArgs reports whether any of args is a flag that // bannerSuppressedInArgs reports whether any of args is a flag that
@@ -48,10 +50,12 @@ func bannerSuppressedInArgs(args []string) bool {
if a == "--" { if a == "--" {
return false return false
} }
switch a { switch a {
case "--quiet", "-q", "--cron": case "--quiet", "-q", "--cron":
return true return true
} }
if strings.HasPrefix(a, "--quiet=") || strings.HasPrefix(a, "--cron=") { if strings.HasPrefix(a, "--quiet=") || strings.HasPrefix(a, "--cron=") {
return true return true
} }
@@ -64,5 +68,6 @@ func bannerSuppressedInArgs(args []string) bool {
} }
} }
} }
return false return false
} }

View File

@@ -21,12 +21,15 @@ func TestCLIEntry(t *testing.T) {
expectedCommands := []string{"config", "snapshot", "prune", "info", "version", "remote", "database"} expectedCommands := []string{"config", "snapshot", "prune", "info", "version", "remote", "database"}
for _, expected := range expectedCommands { for _, expected := range expectedCommands {
found := false found := false
for _, cmd := range cmd.Commands() { for _, cmd := range cmd.Commands() {
if cmd.Use == expected || cmd.Name() == expected { if cmd.Use == expected || cmd.Name() == expected {
found = true found = true
break break
} }
} }
if !found { if !found {
t.Errorf("Expected command '%s' not found", expected) t.Errorf("Expected command '%s' not found", expected)
} }
@@ -41,12 +44,15 @@ func TestCLIEntry(t *testing.T) {
expectedSubCommands := []string{"create", "list", "purge", "verify", "remove", "restore"} expectedSubCommands := []string{"create", "list", "purge", "verify", "remove", "restore"}
for _, expected := range expectedSubCommands { for _, expected := range expectedSubCommands {
found := false found := false
for _, subcmd := range snapshotCmd.Commands() { for _, subcmd := range snapshotCmd.Commands() {
if subcmd.Use == expected || subcmd.Name() == expected { if subcmd.Use == expected || subcmd.Name() == expected {
found = true found = true
break break
} }
} }
if !found { if !found {
t.Errorf("Expected snapshot subcommand '%s' not found", expected) t.Errorf("Expected snapshot subcommand '%s' not found", expected)
} }

View File

@@ -2,6 +2,7 @@ package cli
import ( import (
"context" "context"
"errors"
"os" "os"
"github.com/spf13/cobra" "github.com/spf13/cobra"
@@ -31,6 +32,7 @@ func NewInfoCommand() *cobra.Command {
// Use the app framework // Use the app framework
rootFlags := GetRootFlags() rootFlags := GetRootFlags()
return RunWithApp(cmd.Context(), AppOptions{ return RunWithApp(cmd.Context(), AppOptions{
ConfigPath: configPath, ConfigPath: configPath,
LogOptions: log.LogOptions{ LogOptions: log.LogOptions{
@@ -44,21 +46,26 @@ func NewInfoCommand() *cobra.Command {
lc.Append(fx.Hook{ lc.Append(fx.Hook{
OnStart: func(ctx context.Context) error { OnStart: func(ctx context.Context) error {
go func() { go func() {
if err := v.ShowInfo(); err != nil { err := v.ShowInfo()
if err != context.Canceled { if err != nil {
if !errors.Is(err, context.Canceled) {
log.Error("Failed to show info", "error", err) log.Error("Failed to show info", "error", err)
ReportError("Failed to show info: %v", err) ReportErrorf("Failed to show info: %v", err)
os.Exit(1) os.Exit(1)
} }
} }
if err := v.Shutdowner.Shutdown(); err != nil {
err = v.Shutdowner.Shutdown()
if err != nil {
log.Error("Failed to shutdown", "error", err) log.Error("Failed to shutdown", "error", err)
} }
}() }()
return nil return nil
}, },
OnStop: func(ctx context.Context) error { OnStop: func(ctx context.Context) error {
v.Cancel() v.Cancel()
return nil return nil
}, },
}) })

View File

@@ -2,6 +2,7 @@ package cli
import ( import (
"context" "context"
"errors"
"os" "os"
"github.com/spf13/cobra" "github.com/spf13/cobra"
@@ -39,6 +40,7 @@ work (e.g. after a crashed backup or to reclaim storage).`,
// Use the app framework like other commands // Use the app framework like other commands
rootFlags := GetRootFlags() rootFlags := GetRootFlags()
return RunWithApp(cmd.Context(), AppOptions{ return RunWithApp(cmd.Context(), AppOptions{
ConfigPath: configPath, ConfigPath: configPath,
LogOptions: log.LogOptions{ LogOptions: log.LogOptions{
@@ -54,26 +56,31 @@ work (e.g. after a crashed backup or to reclaim storage).`,
// Start the prune operation in a goroutine // Start the prune operation in a goroutine
go func() { go func() {
// Run the prune operation // Run the prune operation
if err := v.Prune(opts); err != nil { err := v.Prune(opts)
if err != context.Canceled { if err != nil {
if !errors.Is(err, context.Canceled) {
if !opts.JSON { if !opts.JSON {
log.Error("Prune operation failed", "error", err) log.Error("Prune operation failed", "error", err)
ReportError("Prune failed: %v", err) ReportErrorf("Prune failed: %v", err)
} }
os.Exit(1) os.Exit(1)
} }
} }
// Shutdown the app when prune completes // Shutdown the app when prune completes
if err := v.Shutdowner.Shutdown(); err != nil { err = v.Shutdowner.Shutdown()
if err != nil {
log.Error("Failed to shutdown", "error", err) log.Error("Failed to shutdown", "error", err)
} }
}() }()
return nil return nil
}, },
OnStop: func(ctx context.Context) error { OnStop: func(ctx context.Context) error {
log.Debug("Stopping prune operation") log.Debug("Stopping prune operation")
v.Cancel() v.Cancel()
return nil return nil
}, },
}) })

View File

@@ -2,7 +2,7 @@ package cli
import ( import (
"context" "context"
"fmt" "errors"
"os" "os"
"github.com/spf13/cobra" "github.com/spf13/cobra"
@@ -41,7 +41,7 @@ This is destructive and irreversible. Requires --force.`,
Args: cobra.NoArgs, Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, args []string) error { RunE: func(cmd *cobra.Command, args []string) error {
if !force { if !force {
return fmt.Errorf("remote nuke requires --force (this deletes ALL remote snapshots and blobs)") return errors.New("remote nuke requires --force (this deletes ALL remote snapshots and blobs)")
} }
configPath, err := ResolveConfigPath() configPath, err := ResolveConfigPath()
@@ -50,6 +50,7 @@ This is destructive and irreversible. Requires --force.`,
} }
rootFlags := GetRootFlags() rootFlags := GetRootFlags()
return RunWithApp(cmd.Context(), AppOptions{ return RunWithApp(cmd.Context(), AppOptions{
ConfigPath: configPath, ConfigPath: configPath,
LogOptions: log.LogOptions{ LogOptions: log.LogOptions{
@@ -63,21 +64,26 @@ This is destructive and irreversible. Requires --force.`,
lc.Append(fx.Hook{ lc.Append(fx.Hook{
OnStart: func(ctx context.Context) error { OnStart: func(ctx context.Context) error {
go func() { go func() {
if err := v.NukeRemote(true); err != nil { err := v.NukeRemote(true)
if err != context.Canceled { if err != nil {
if !errors.Is(err, context.Canceled) {
log.Error("Remote nuke failed", "error", err) log.Error("Remote nuke failed", "error", err)
ReportError("Remote nuke failed: %v", err) ReportErrorf("Remote nuke failed: %v", err)
os.Exit(1) os.Exit(1)
} }
} }
if err := v.Shutdowner.Shutdown(); err != nil {
err = v.Shutdowner.Shutdown()
if err != nil {
log.Error("Failed to shutdown", "error", err) log.Error("Failed to shutdown", "error", err)
} }
}() }()
return nil return nil
}, },
OnStop: func(ctx context.Context) error { OnStop: func(ctx context.Context) error {
v.Cancel() v.Cancel()
return nil return nil
}, },
}) })
@@ -113,6 +119,7 @@ func newRemoteInfoCommand() *cobra.Command {
} }
rootFlags := GetRootFlags() rootFlags := GetRootFlags()
return RunWithApp(cmd.Context(), AppOptions{ return RunWithApp(cmd.Context(), AppOptions{
ConfigPath: configPath, ConfigPath: configPath,
LogOptions: log.LogOptions{ LogOptions: log.LogOptions{
@@ -126,23 +133,29 @@ func newRemoteInfoCommand() *cobra.Command {
lc.Append(fx.Hook{ lc.Append(fx.Hook{
OnStart: func(ctx context.Context) error { OnStart: func(ctx context.Context) error {
go func() { go func() {
if err := v.RemoteInfo(jsonOutput); err != nil { err := v.RemoteInfo(jsonOutput)
if err != context.Canceled { if err != nil {
if !errors.Is(err, context.Canceled) {
if !jsonOutput { if !jsonOutput {
log.Error("Failed to get remote info", "error", err) log.Error("Failed to get remote info", "error", err)
ReportError("Failed to get remote info: %v", err) ReportErrorf("Failed to get remote info: %v", err)
} }
os.Exit(1) os.Exit(1)
} }
} }
if err := v.Shutdowner.Shutdown(); err != nil {
err = v.Shutdowner.Shutdown()
if err != nil {
log.Error("Failed to shutdown", "error", err) log.Error("Failed to shutdown", "error", err)
} }
}() }()
return nil return nil
}, },
OnStop: func(ctx context.Context) error { OnStop: func(ctx context.Context) error {
v.Cancel() v.Cancel()
return nil return nil
}, },
}) })

View File

@@ -78,6 +78,7 @@ func ResolveConfigPath() (string, error) {
if _, err := os.Stat(path); err != nil { if _, err := os.Stat(path); err != nil {
return "", fmt.Errorf("config file from --config not found: %s (run 'vaultik config init --config %s' to create it)", path, path) return "", fmt.Errorf("config file from --config not found: %s (run 'vaultik config init --config %s' to create it)", path, path)
} }
return path, nil return path, nil
} }
@@ -85,6 +86,7 @@ func ResolveConfigPath() (string, error) {
if _, err := os.Stat(path); err != nil { if _, err := os.Stat(path); err != nil {
return "", fmt.Errorf("config file from $VAULTIK_CONFIG not found: %s (unset VAULTIK_CONFIG, point it at an existing file, or run 'vaultik config init')", path) return "", fmt.Errorf("config file from $VAULTIK_CONFIG not found: %s (unset VAULTIK_CONFIG, point it at an existing file, or run 'vaultik config init')", path)
} }
return path, nil return path, nil
} }
@@ -114,5 +116,6 @@ func DefaultConfigPath() string {
if os.Getuid() == 0 { if os.Getuid() == 0 {
return "/etc/vaultik/config.yml" return "/etc/vaultik/config.yml"
} }
return filepath.Join(xdg.ConfigHome, "vaultik", "config.yml") return filepath.Join(xdg.ConfigHome, "vaultik", "config.yml")
} }

View File

@@ -2,6 +2,7 @@ package cli
import ( import (
"context" "context"
"errors"
"fmt" "fmt"
"os" "os"
@@ -58,6 +59,7 @@ specifying a path using --config or by setting VAULTIK_CONFIG to a path.`,
// Use the backup functionality from cli package // Use the backup functionality from cli package
rootFlags := GetRootFlags() rootFlags := GetRootFlags()
return RunWithApp(cmd.Context(), AppOptions{ return RunWithApp(cmd.Context(), AppOptions{
ConfigPath: configPath, ConfigPath: configPath,
LogOptions: log.LogOptions{ LogOptions: log.LogOptions{
@@ -74,25 +76,29 @@ specifying a path using --config or by setting VAULTIK_CONFIG to a path.`,
// Start the snapshot creation in a goroutine // Start the snapshot creation in a goroutine
go func() { go func() {
// --cron suppression is wired through v.UI by setupGlobals. // --cron suppression is wired through v.UI by setupGlobals.
if err := v.CreateSnapshot(opts); err != nil { err := v.CreateSnapshot(opts)
if err != context.Canceled { if err != nil {
if !errors.Is(err, context.Canceled) {
log.Error("Snapshot creation failed", "error", err) log.Error("Snapshot creation failed", "error", err)
ReportError("Snapshot creation failed: %v", err) ReportErrorf("Snapshot creation failed: %v", err)
os.Exit(1) os.Exit(1)
} }
} }
// Shutdown the app when snapshot completes // Shutdown the app when snapshot completes
if err := v.Shutdowner.Shutdown(); err != nil { err = v.Shutdowner.Shutdown()
if err != nil {
log.Error("Failed to shutdown", "error", err) log.Error("Failed to shutdown", "error", err)
} }
}() }()
return nil return nil
}, },
OnStop: func(ctx context.Context) error { OnStop: func(ctx context.Context) error {
log.Debug("Stopping snapshot creation") log.Debug("Stopping snapshot creation")
// Cancel the Vaultik context // Cancel the Vaultik context
v.Cancel() v.Cancel()
return nil return nil
}, },
}) })
@@ -127,6 +133,7 @@ func newSnapshotListCommand() *cobra.Command {
} }
rootFlags := GetRootFlags() rootFlags := GetRootFlags()
return RunWithApp(cmd.Context(), AppOptions{ return RunWithApp(cmd.Context(), AppOptions{
ConfigPath: configPath, ConfigPath: configPath,
LogOptions: log.LogOptions{ LogOptions: log.LogOptions{
@@ -140,21 +147,26 @@ func newSnapshotListCommand() *cobra.Command {
lc.Append(fx.Hook{ lc.Append(fx.Hook{
OnStart: func(ctx context.Context) error { OnStart: func(ctx context.Context) error {
go func() { go func() {
if err := v.ListSnapshots(jsonOutput); err != nil { err := v.ListSnapshots(jsonOutput)
if err != context.Canceled { if err != nil {
if !errors.Is(err, context.Canceled) {
log.Error("Failed to list snapshots", "error", err) log.Error("Failed to list snapshots", "error", err)
ReportError("Failed to list snapshots: %v", err) ReportErrorf("Failed to list snapshots: %v", err)
os.Exit(1) os.Exit(1)
} }
} }
if err := v.Shutdowner.Shutdown(); err != nil {
err = v.Shutdowner.Shutdown()
if err != nil {
log.Error("Failed to shutdown", "error", err) log.Error("Failed to shutdown", "error", err)
} }
}() }()
return nil return nil
}, },
OnStop: func(ctx context.Context) error { OnStop: func(ctx context.Context) error {
v.Cancel() v.Cancel()
return nil return nil
}, },
}) })
@@ -185,10 +197,11 @@ restrict the operation to specific snapshot names.`,
RunE: func(cmd *cobra.Command, args []string) error { RunE: func(cmd *cobra.Command, args []string) error {
// Validate flags // Validate flags
if !opts.KeepLatest && opts.OlderThan == "" { if !opts.KeepLatest && opts.OlderThan == "" {
return fmt.Errorf("must specify either --keep-latest or --older-than") return errors.New("must specify either --keep-latest or --older-than")
} }
if opts.KeepLatest && opts.OlderThan != "" { if opts.KeepLatest && opts.OlderThan != "" {
return fmt.Errorf("cannot specify both --keep-latest and --older-than") return errors.New("cannot specify both --keep-latest and --older-than")
} }
// Use unified config resolution // Use unified config resolution
@@ -198,6 +211,7 @@ restrict the operation to specific snapshot names.`,
} }
rootFlags := GetRootFlags() rootFlags := GetRootFlags()
return RunWithApp(cmd.Context(), AppOptions{ return RunWithApp(cmd.Context(), AppOptions{
ConfigPath: configPath, ConfigPath: configPath,
LogOptions: log.LogOptions{ LogOptions: log.LogOptions{
@@ -211,21 +225,26 @@ restrict the operation to specific snapshot names.`,
lc.Append(fx.Hook{ lc.Append(fx.Hook{
OnStart: func(ctx context.Context) error { OnStart: func(ctx context.Context) error {
go func() { go func() {
if err := v.PurgeSnapshotsWithOptions(opts); err != nil { err := v.PurgeSnapshotsWithOptions(opts)
if err != context.Canceled { if err != nil {
if !errors.Is(err, context.Canceled) {
log.Error("Failed to purge snapshots", "error", err) log.Error("Failed to purge snapshots", "error", err)
ReportError("Failed to purge snapshots: %v", err) ReportErrorf("Failed to purge snapshots: %v", err)
os.Exit(1) os.Exit(1)
} }
} }
if err := v.Shutdowner.Shutdown(); err != nil {
err = v.Shutdowner.Shutdown()
if err != nil {
log.Error("Failed to shutdown", "error", err) log.Error("Failed to shutdown", "error", err)
} }
}() }()
return nil return nil
}, },
OnStop: func(ctx context.Context) error { OnStop: func(ctx context.Context) error {
v.Cancel() v.Cancel()
return nil return nil
}, },
}) })
@@ -254,11 +273,14 @@ func newSnapshotVerifyCommand() *cobra.Command {
Args: func(cmd *cobra.Command, args []string) error { Args: func(cmd *cobra.Command, args []string) error {
if len(args) != 1 { if len(args) != 1 {
_ = cmd.Help() _ = cmd.Help()
if len(args) == 0 { if len(args) == 0 {
return fmt.Errorf("snapshot ID required") return errors.New("snapshot ID required")
} }
return fmt.Errorf("expected 1 argument, got %d", len(args)) return fmt.Errorf("expected 1 argument, got %d", len(args))
} }
return nil return nil
}, },
RunE: func(cmd *cobra.Command, args []string) error { RunE: func(cmd *cobra.Command, args []string) error {
@@ -271,6 +293,7 @@ func newSnapshotVerifyCommand() *cobra.Command {
} }
rootFlags := GetRootFlags() rootFlags := GetRootFlags()
return RunWithApp(cmd.Context(), AppOptions{ return RunWithApp(cmd.Context(), AppOptions{
ConfigPath: configPath, ConfigPath: configPath,
LogOptions: log.LogOptions{ LogOptions: log.LogOptions{
@@ -284,23 +307,29 @@ func newSnapshotVerifyCommand() *cobra.Command {
lc.Append(fx.Hook{ lc.Append(fx.Hook{
OnStart: func(ctx context.Context) error { OnStart: func(ctx context.Context) error {
go func() { go func() {
if err := v.VerifySnapshotWithOptions(snapshotID, opts); err != nil { err := v.VerifySnapshotWithOptions(snapshotID, opts)
if err != context.Canceled { if err != nil {
if !errors.Is(err, context.Canceled) {
if !opts.JSON { if !opts.JSON {
log.Error("Verification failed", "error", err) log.Error("Verification failed", "error", err)
ReportError("Verification failed: %v", err) ReportErrorf("Verification failed: %v", err)
} }
os.Exit(1) os.Exit(1)
} }
} }
if err := v.Shutdowner.Shutdown(); err != nil {
err = v.Shutdowner.Shutdown()
if err != nil {
log.Error("Failed to shutdown", "error", err) log.Error("Failed to shutdown", "error", err)
} }
}() }()
return nil return nil
}, },
OnStop: func(ctx context.Context) error { OnStop: func(ctx context.Context) error {
v.Cancel() v.Cancel()
return nil return nil
}, },
}) })
@@ -345,11 +374,14 @@ nuke --force' — it is the single supported entry point for that.`,
Args: func(cmd *cobra.Command, args []string) error { Args: func(cmd *cobra.Command, args []string) error {
if len(args) != 1 { if len(args) != 1 {
_ = cmd.Help() _ = cmd.Help()
if len(args) == 0 { if len(args) == 0 {
return fmt.Errorf("snapshot ID required") return errors.New("snapshot ID required")
} }
return fmt.Errorf("expected 1 argument, got %d", len(args)) return fmt.Errorf("expected 1 argument, got %d", len(args))
} }
return nil return nil
}, },
RunE: func(cmd *cobra.Command, args []string) error { RunE: func(cmd *cobra.Command, args []string) error {
@@ -360,6 +392,7 @@ nuke --force' — it is the single supported entry point for that.`,
} }
rootFlags := GetRootFlags() rootFlags := GetRootFlags()
return RunWithApp(cmd.Context(), AppOptions{ return RunWithApp(cmd.Context(), AppOptions{
ConfigPath: configPath, ConfigPath: configPath,
LogOptions: log.LogOptions{ LogOptions: log.LogOptions{
@@ -375,22 +408,26 @@ nuke --force' — it is the single supported entry point for that.`,
go func() { go func() {
_, err := v.RemoveSnapshot(args[0], opts) _, err := v.RemoveSnapshot(args[0], opts)
if err != nil { if err != nil {
if err != context.Canceled { if !errors.Is(err, context.Canceled) {
if !opts.JSON { if !opts.JSON {
log.Error("Failed to remove snapshot", "error", err) log.Error("Failed to remove snapshot", "error", err)
ReportError("Failed to remove snapshot: %v", err) ReportErrorf("Failed to remove snapshot: %v", err)
} }
os.Exit(1) os.Exit(1)
} }
} }
if err := v.Shutdowner.Shutdown(); err != nil { if err := v.Shutdowner.Shutdown(); err != nil {
log.Error("Failed to shutdown", "error", err) log.Error("Failed to shutdown", "error", err)
} }
}() }()
return nil return nil
}, },
OnStop: func(ctx context.Context) error { OnStop: func(ctx context.Context) error {
v.Cancel() v.Cancel()
return nil return nil
}, },
}) })

View File

@@ -2,6 +2,7 @@ package cli
import ( import (
"context" "context"
"errors"
"os" "os"
"github.com/spf13/cobra" "github.com/spf13/cobra"
@@ -70,6 +71,7 @@ Examples:
// runRestore parses arguments and runs the restore operation through the app framework // runRestore parses arguments and runs the restore operation through the app framework
func runRestore(cmd *cobra.Command, args []string, opts *RestoreOptions) error { func runRestore(cmd *cobra.Command, args []string, opts *RestoreOptions) error {
snapshotID := args[0] snapshotID := args[0]
opts.TargetDir = args[1] opts.TargetDir = args[1]
if len(args) > 2 { if len(args) > 2 {
opts.Paths = args[2:] opts.Paths = args[2:]
@@ -83,6 +85,7 @@ func runRestore(cmd *cobra.Command, args []string, opts *RestoreOptions) error {
// Use the app framework like other commands // Use the app framework like other commands
rootFlags := GetRootFlags() rootFlags := GetRootFlags()
return RunWithApp(cmd.Context(), AppOptions{ return RunWithApp(cmd.Context(), AppOptions{
ConfigPath: configPath, ConfigPath: configPath,
LogOptions: log.LogOptions{ LogOptions: log.LogOptions{
@@ -129,24 +132,29 @@ func buildRestoreInvokes(snapshotID string, opts *RestoreOptions) []fx.Option {
Verify: opts.Verify, Verify: opts.Verify,
SkipErrors: GetRootFlags().SkipErrors, SkipErrors: GetRootFlags().SkipErrors,
} }
if err := app.Vaultik.Restore(restoreOpts); err != nil {
if err != context.Canceled { err := app.Vaultik.Restore(restoreOpts)
if err != nil {
if !errors.Is(err, context.Canceled) {
log.Error("Restore operation failed", "error", err) log.Error("Restore operation failed", "error", err)
ReportError("Restore failed: %v", err) ReportErrorf("Restore failed: %v", err)
os.Exit(1) os.Exit(1)
} }
} }
// Shutdown the app when restore completes // Shutdown the app when restore completes
if err := app.Shutdowner.Shutdown(); err != nil { err = app.Shutdowner.Shutdown()
if err != nil {
log.Error("Failed to shutdown", "error", err) log.Error("Failed to shutdown", "error", err)
} }
}() }()
return nil return nil
}, },
OnStop: func(ctx context.Context) error { OnStop: func(ctx context.Context) error {
log.Debug("Stopping restore operation") log.Debug("Stopping restore operation")
app.Vaultik.Cancel() app.Vaultik.Cancel()
return nil return nil
}, },
}) })

View File

@@ -24,6 +24,7 @@ func NewVersionCommand() *cobra.Command {
fmt.Printf(" author: %s\n", globals.Author) fmt.Printf(" author: %s\n", globals.Author)
fmt.Printf(" homepage: %s\n", globals.Homepage) fmt.Printf(" homepage: %s\n", globals.Homepage)
fmt.Printf(" license: %s\n", globals.License) fmt.Printf(" license: %s\n", globals.License)
if globals.Version == "dev" { if globals.Version == "dev" {
fmt.Println() fmt.Println()
fmt.Println("This is a development build (no version information embedded).") fmt.Println("This is a development build (no version information embedded).")

View File

@@ -1,6 +1,7 @@
package config package config
import ( import (
"errors"
"fmt" "fmt"
"os" "os"
"path/filepath" "path/filepath"
@@ -21,12 +22,16 @@ const appName = "vaultik"
func expandTilde(path string) string { func expandTilde(path string) string {
if path == "~" { if path == "~" {
home, _ := os.UserHomeDir() home, _ := os.UserHomeDir()
return home return home
} }
if strings.HasPrefix(path, "~/") { if strings.HasPrefix(path, "~/") {
home, _ := os.UserHomeDir() home, _ := os.UserHomeDir()
return filepath.Join(home, path[2:]) return filepath.Join(home, path[2:])
} }
return path return path
} }
@@ -34,8 +39,10 @@ func expandTilde(path string) string {
func expandTildeInURL(url string) string { func expandTildeInURL(url string) string {
if strings.HasPrefix(url, "file://~/") { if strings.HasPrefix(url, "file://~/") {
home, _ := os.UserHomeDir() home, _ := os.UserHomeDir()
return "file://" + filepath.Join(home, url[9:]) return "file://" + filepath.Join(home, url[9:])
} }
return url return url
} }
@@ -63,6 +70,7 @@ func (c *Config) GetExcludes(snapshotName string) []string {
combined := make([]string, 0, len(c.Exclude)+len(snap.Exclude)) combined := make([]string, 0, len(c.Exclude)+len(snap.Exclude))
combined = append(combined, c.Exclude...) combined = append(combined, c.Exclude...)
combined = append(combined, snap.Exclude...) combined = append(combined, snap.Exclude...)
return combined return combined
} }
@@ -74,6 +82,7 @@ func (c *Config) SnapshotNames() []string {
} }
// Sort for deterministic order // Sort for deterministic order
sort.Strings(names) sort.Strings(names)
return names return names
} }
@@ -126,7 +135,7 @@ type ConfigPath string
// Returns an error if the path is empty or if loading fails. // Returns an error if the path is empty or if loading fails.
func New(path ConfigPath) (*Config, error) { func New(path ConfigPath) (*Config, error) {
if path == "" { if path == "" {
return nil, fmt.Errorf("config path not provided") return nil, errors.New("config path not provided")
} }
cfg, err := Load(string(path)) cfg, err := Load(string(path))
@@ -159,6 +168,7 @@ func Load(path string) (*Config, error) {
// Convert smartconfig data to YAML then unmarshal // Convert smartconfig data to YAML then unmarshal
configData := sc.Data() configData := sc.Data()
yamlBytes, err := yaml.Marshal(configData) yamlBytes, err := yaml.Marshal(configData)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to marshal config data: %w", err) return nil, fmt.Errorf("failed to marshal config data: %w", err)
@@ -177,6 +187,7 @@ func Load(path string) (*Config, error) {
for i, path := range snap.Paths { for i, path := range snap.Paths {
snap.Paths[i] = expandTilde(path) snap.Paths[i] = expandTilde(path)
} }
cfg.Snapshots[name] = snap cfg.Snapshots[name] = snap
} }
@@ -196,6 +207,7 @@ func Load(path string) (*Config, error) {
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to get hostname: %w", err) return nil, fmt.Errorf("failed to get hostname: %w", err)
} }
cfg.Hostname = hostname cfg.Hostname = hostname
} }
@@ -203,6 +215,7 @@ func Load(path string) (*Config, error) {
if cfg.S3.Region == "" { if cfg.S3.Region == "" {
cfg.S3.Region = "us-east-1" cfg.S3.Region = "us-east-1"
} }
if cfg.S3.PartSize == 0 { if cfg.S3.PartSize == 0 {
cfg.S3.PartSize = Size(5 * 1024 * 1024) // 5MB cfg.S3.PartSize = Size(5 * 1024 * 1024) // 5MB
} }
@@ -236,11 +249,11 @@ func Load(path string) (*Config, error) {
// Returns an error describing the first validation failure encountered. // Returns an error describing the first validation failure encountered.
func (c *Config) Validate() error { func (c *Config) Validate() error {
if len(c.AgeRecipients) == 0 { if len(c.AgeRecipients) == 0 {
return fmt.Errorf("at least one age_recipient is required (generate with: age-keygen)") return errors.New("at least one age_recipient is required (generate with: age-keygen)")
} }
if len(c.Snapshots) == 0 { if len(c.Snapshots) == 0 {
return fmt.Errorf("at least one snapshot must be configured (see config.example.yml)") return errors.New("at least one snapshot must be configured (see config.example.yml)")
} }
for name, snap := range c.Snapshots { for name, snap := range c.Snapshots {
@@ -250,20 +263,21 @@ func (c *Config) Validate() error {
} }
// Validate storage configuration // Validate storage configuration
if err := c.validateStorage(); err != nil { err := c.validateStorage()
if err != nil {
return err return err
} }
if c.ChunkSize.Int64() < 1024*1024 { // 1MB minimum if c.ChunkSize.Int64() < 1024*1024 { // 1MB minimum
return fmt.Errorf("chunk_size must be at least 1MB") return errors.New("chunk_size must be at least 1MB")
} }
if c.BlobSizeLimit.Int64() < c.ChunkSize.Int64() { if c.BlobSizeLimit.Int64() < c.ChunkSize.Int64() {
return fmt.Errorf("blob_size_limit must be at least chunk_size") return errors.New("blob_size_limit must be at least chunk_size")
} }
if c.CompressionLevel < 1 || c.CompressionLevel > 19 { if c.CompressionLevel < 1 || c.CompressionLevel > 19 {
return fmt.Errorf("compression_level must be between 1 and 19") return errors.New("compression_level must be between 1 and 19")
} }
return nil return nil
@@ -280,38 +294,43 @@ func (c *Config) validateStorage() error {
// File storage doesn't need S3 credentials // File storage doesn't need S3 credentials
return nil return nil
} }
if strings.HasPrefix(c.StorageURL, "s3://") { if strings.HasPrefix(c.StorageURL, "s3://") {
// S3 storage needs credentials // S3 storage needs credentials
if c.S3.AccessKeyID == "" { if c.S3.AccessKeyID == "" {
return fmt.Errorf("s3.access_key_id is required for s3:// URLs") return errors.New("s3.access_key_id is required for s3:// URLs")
} }
if c.S3.SecretAccessKey == "" { if c.S3.SecretAccessKey == "" {
return fmt.Errorf("s3.secret_access_key is required for s3:// URLs") return errors.New("s3.secret_access_key is required for s3:// URLs")
} }
return nil return nil
} }
if strings.HasPrefix(c.StorageURL, "rclone://") { if strings.HasPrefix(c.StorageURL, "rclone://") {
// Rclone storage uses rclone's own config // Rclone storage uses rclone's own config
return nil return nil
} }
return fmt.Errorf("storage_url must start with s3://, file://, or rclone://")
return errors.New("storage_url must start with s3://, file://, or rclone://")
} }
// Legacy S3 configuration // Legacy S3 configuration
if c.S3.Endpoint == "" { if c.S3.Endpoint == "" {
return fmt.Errorf("storage not configured; set storage_url or provide s3.endpoint + s3.bucket + credentials") return errors.New("storage not configured; set storage_url or provide s3.endpoint + s3.bucket + credentials")
} }
if c.S3.Bucket == "" { if c.S3.Bucket == "" {
return fmt.Errorf("s3.bucket is required (or set storage_url)") return errors.New("s3.bucket is required (or set storage_url)")
} }
if c.S3.AccessKeyID == "" { if c.S3.AccessKeyID == "" {
return fmt.Errorf("s3.access_key_id is required") return errors.New("s3.access_key_id is required")
} }
if c.S3.SecretAccessKey == "" { if c.S3.SecretAccessKey == "" {
return fmt.Errorf("s3.secret_access_key is required") return errors.New("s3.secret_access_key is required")
} }
return nil return nil
@@ -329,6 +348,7 @@ func extractAgeSecretKey(input string) string {
if id, ok := identities[0].(*age.X25519Identity); ok { if id, ok := identities[0].(*age.X25519Identity); ok {
return id.String() return id.String()
} }
return strings.TrimSpace(input) return strings.TrimSpace(input)
} }

View File

@@ -41,6 +41,7 @@ func TestConfigLoad(t *testing.T) {
if len(cfg.AgeRecipients) != 2 { if len(cfg.AgeRecipients) != 2 {
t.Errorf("Expected 2 age recipients, got %d", len(cfg.AgeRecipients)) t.Errorf("Expected 2 age recipients, got %d", len(cfg.AgeRecipients))
} }
if cfg.AgeRecipients[0] != TEST_SNEAK_AGE_PUBLIC_KEY { if cfg.AgeRecipients[0] != TEST_SNEAK_AGE_PUBLIC_KEY {
t.Errorf("Expected first age recipient to be %s, got '%s'", TEST_SNEAK_AGE_PUBLIC_KEY, cfg.AgeRecipients[0]) t.Errorf("Expected first age recipient to be %s, got '%s'", TEST_SNEAK_AGE_PUBLIC_KEY, cfg.AgeRecipients[0])
} }

View File

@@ -1,6 +1,7 @@
package config package config
import ( import (
"errors"
"fmt" "fmt"
"github.com/dustin/go-humanize" "github.com/dustin/go-humanize"
@@ -14,18 +15,19 @@ type Size int64
// UnmarshalYAML implements yaml.Unmarshaler for Size, allowing it to be // UnmarshalYAML implements yaml.Unmarshaler for Size, allowing it to be
// parsed from YAML configuration files. It accepts both numeric values // parsed from YAML configuration files. It accepts both numeric values
// (interpreted as bytes) and string values with units (e.g., "10MB"). // (interpreted as bytes) and string values with units (e.g., "10MB").
func (s *Size) UnmarshalYAML(unmarshal func(interface{}) error) error { func (s *Size) UnmarshalYAML(unmarshal func(any) error) error {
// Try to unmarshal as int64 first // Try to unmarshal as int64 first
var intVal int64 var intVal int64
if err := unmarshal(&intVal); err == nil { if err := unmarshal(&intVal); err == nil {
*s = Size(intVal) *s = Size(intVal)
return nil return nil
} }
// Try to unmarshal as string // Try to unmarshal as string
var strVal string var strVal string
if err := unmarshal(&strVal); err != nil { if err := unmarshal(&strVal); err != nil {
return fmt.Errorf("size must be a number or string") return errors.New("size must be a number or string")
} }
// Parse the string using go-humanize // Parse the string using go-humanize
@@ -35,6 +37,7 @@ func (s *Size) UnmarshalYAML(unmarshal func(interface{}) error) error {
} }
*s = Size(bytes) *s = Size(bytes)
return nil return nil
} }
@@ -58,5 +61,6 @@ func ParseSize(s string) (Size, error) {
if err != nil { if err != nil {
return 0, fmt.Errorf("invalid size format: %w", err) return 0, fmt.Errorf("invalid size format: %w", err)
} }
return Size(bytes), nil return Size(bytes), nil
} }

View File

@@ -2,6 +2,7 @@ package crypto
import ( import (
"bytes" "bytes"
"errors"
"fmt" "fmt"
"io" "io"
"sync" "sync"
@@ -25,7 +26,7 @@ type Encryptor struct {
// public keys are invalid or if no recipients are specified. // public keys are invalid or if no recipients are specified.
func NewEncryptor(publicKeys []string) (*Encryptor, error) { func NewEncryptor(publicKeys []string) (*Encryptor, error) {
if len(publicKeys) == 0 { if len(publicKeys) == 0 {
return nil, fmt.Errorf("at least one recipient is required") return nil, errors.New("at least one recipient is required")
} }
recipients := make([]age.Recipient, 0, len(publicKeys)) recipients := make([]age.Recipient, 0, len(publicKeys))
@@ -34,6 +35,7 @@ func NewEncryptor(publicKeys []string) (*Encryptor, error) {
if err != nil { if err != nil {
return nil, fmt.Errorf("parsing age recipient %s: %w", key, err) return nil, fmt.Errorf("parsing age recipient %s: %w", key, err)
} }
recipients = append(recipients, recipient) recipients = append(recipients, recipient)
} }
@@ -126,7 +128,7 @@ func (e *Encryptor) EncryptWriter(dst io.Writer) (io.WriteCloser, error) {
// of the public keys are invalid or if no recipients are specified. // of the public keys are invalid or if no recipients are specified.
func (e *Encryptor) UpdateRecipients(publicKeys []string) error { func (e *Encryptor) UpdateRecipients(publicKeys []string) error {
if len(publicKeys) == 0 { if len(publicKeys) == 0 {
return fmt.Errorf("at least one recipient is required") return errors.New("at least one recipient is required")
} }
recipients := make([]age.Recipient, 0, len(publicKeys)) recipients := make([]age.Recipient, 0, len(publicKeys))
@@ -135,6 +137,7 @@ func (e *Encryptor) UpdateRecipients(publicKeys []string) error {
if err != nil { if err != nil {
return fmt.Errorf("parsing age recipient %s: %w", key, err) return fmt.Errorf("parsing age recipient %s: %w", key, err)
} }
recipients = append(recipients, recipient) recipients = append(recipients, recipient)
} }

View File

@@ -58,10 +58,12 @@ func TestEncryptorMultipleRecipients(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("failed to generate identity1: %v", err) t.Fatalf("failed to generate identity1: %v", err)
} }
identity2, err := age.GenerateX25519Identity() identity2, err := age.GenerateX25519Identity()
if err != nil { if err != nil {
t.Fatalf("failed to generate identity2: %v", err) t.Fatalf("failed to generate identity2: %v", err)
} }
identity3, err := age.GenerateX25519Identity() identity3, err := age.GenerateX25519Identity()
if err != nil { if err != nil {
t.Fatalf("failed to generate identity3: %v", err) t.Fatalf("failed to generate identity3: %v", err)
@@ -123,6 +125,7 @@ func TestEncryptorUpdateRecipients(t *testing.T) {
// Encrypt with first key // Encrypt with first key
plaintext := []byte("test data") plaintext := []byte("test data")
ciphertext1, err := enc.Encrypt(plaintext) ciphertext1, err := enc.Encrypt(plaintext)
if err != nil { if err != nil {
t.Fatalf("failed to encrypt: %v", err) t.Fatalf("failed to encrypt: %v", err)
@@ -143,6 +146,7 @@ func TestEncryptorUpdateRecipients(t *testing.T) {
if _, err := age.Decrypt(bytes.NewReader(ciphertext1), identity1); err != nil { if _, err := age.Decrypt(bytes.NewReader(ciphertext1), identity1); err != nil {
t.Error("failed to decrypt with identity1") t.Error("failed to decrypt with identity1")
} }
if _, err := age.Decrypt(bytes.NewReader(ciphertext1), identity2); err == nil { if _, err := age.Decrypt(bytes.NewReader(ciphertext1), identity2); err == nil {
t.Error("should not decrypt with identity2") t.Error("should not decrypt with identity2")
} }
@@ -151,6 +155,7 @@ func TestEncryptorUpdateRecipients(t *testing.T) {
if _, err := age.Decrypt(bytes.NewReader(ciphertext2), identity2); err != nil { if _, err := age.Decrypt(bytes.NewReader(ciphertext2), identity2); err != nil {
t.Error("failed to decrypt with identity2") t.Error("failed to decrypt with identity2")
} }
if _, err := age.Decrypt(bytes.NewReader(ciphertext2), identity1); err == nil { if _, err := age.Decrypt(bytes.NewReader(ciphertext2), identity1); err == nil {
t.Error("should not decrypt with identity1") t.Error("should not decrypt with identity1")
} }

View File

@@ -49,12 +49,15 @@ func (r *BlobChunkRepository) GetByBlobID(ctx context.Context, blobID string) ([
defer CloseRows(rows) defer CloseRows(rows)
var blobChunks []*BlobChunk var blobChunks []*BlobChunk
for rows.Next() { for rows.Next() {
var bc BlobChunk var bc BlobChunk
err := rows.Scan(&bc.BlobID, &bc.ChunkHash, &bc.Offset, &bc.Length) err := rows.Scan(&bc.BlobID, &bc.ChunkHash, &bc.Offset, &bc.Length)
if err != nil { if err != nil {
return nil, fmt.Errorf("scanning blob chunk: %w", err) return nil, fmt.Errorf("scanning blob chunk: %w", err)
} }
blobChunks = append(blobChunks, &bc) blobChunks = append(blobChunks, &bc)
} }
@@ -70,7 +73,9 @@ func (r *BlobChunkRepository) GetByChunkHash(ctx context.Context, chunkHash stri
` `
LogSQL("GetByChunkHash", query, chunkHash) LogSQL("GetByChunkHash", query, chunkHash)
var bc BlobChunk var bc BlobChunk
err := r.db.conn.QueryRowContext(ctx, query, chunkHash).Scan( err := r.db.conn.QueryRowContext(ctx, query, chunkHash).Scan(
&bc.BlobID, &bc.BlobID,
&bc.ChunkHash, &bc.ChunkHash,
@@ -80,14 +85,18 @@ func (r *BlobChunkRepository) GetByChunkHash(ctx context.Context, chunkHash stri
if err == sql.ErrNoRows { if err == sql.ErrNoRows {
LogSQL("GetByChunkHash", "No rows found", chunkHash) LogSQL("GetByChunkHash", "No rows found", chunkHash)
return nil, nil return nil, nil
} }
if err != nil { if err != nil {
LogSQL("GetByChunkHash", "Error", chunkHash, err) LogSQL("GetByChunkHash", "Error", chunkHash, err)
return nil, fmt.Errorf("querying blob chunk: %w", err) return nil, fmt.Errorf("querying blob chunk: %w", err)
} }
LogSQL("GetByChunkHash", "Found blob", chunkHash, "blob", bc.BlobID) LogSQL("GetByChunkHash", "Found blob", chunkHash, "blob", bc.BlobID)
return &bc, nil return &bc, nil
} }
@@ -101,7 +110,9 @@ func (r *BlobChunkRepository) GetByChunkHashTx(ctx context.Context, tx *sql.Tx,
` `
LogSQL("GetByChunkHashTx", query, chunkHash) LogSQL("GetByChunkHashTx", query, chunkHash)
var bc BlobChunk var bc BlobChunk
err := tx.QueryRowContext(ctx, query, chunkHash).Scan( err := tx.QueryRowContext(ctx, query, chunkHash).Scan(
&bc.BlobID, &bc.BlobID,
&bc.ChunkHash, &bc.ChunkHash,
@@ -111,14 +122,18 @@ func (r *BlobChunkRepository) GetByChunkHashTx(ctx context.Context, tx *sql.Tx,
if err == sql.ErrNoRows { if err == sql.ErrNoRows {
LogSQL("GetByChunkHashTx", "No rows found", chunkHash) LogSQL("GetByChunkHashTx", "No rows found", chunkHash)
return nil, nil return nil, nil
} }
if err != nil { if err != nil {
LogSQL("GetByChunkHashTx", "Error", chunkHash, err) LogSQL("GetByChunkHashTx", "Error", chunkHash, err)
return nil, fmt.Errorf("querying blob chunk: %w", err) return nil, fmt.Errorf("querying blob chunk: %w", err)
} }
LogSQL("GetByChunkHashTx", "Found blob", chunkHash, "blob", bc.BlobID) LogSQL("GetByChunkHashTx", "Found blob", chunkHash, "blob", bc.BlobID)
return &bc, nil return &bc, nil
} }

View File

@@ -22,6 +22,7 @@ func TestBlobChunkRepository(t *testing.T) {
Hash: types.BlobHash("blob1-hash"), Hash: types.BlobHash("blob1-hash"),
CreatedTS: time.Now(), CreatedTS: time.Now(),
} }
err := repos.Blobs.Create(ctx, nil, blob) err := repos.Blobs.Create(ctx, nil, blob)
if err != nil { if err != nil {
t.Fatalf("failed to create blob: %v", err) t.Fatalf("failed to create blob: %v", err)
@@ -34,6 +35,7 @@ func TestBlobChunkRepository(t *testing.T) {
ChunkHash: chunkHash, ChunkHash: chunkHash,
Size: 1024, Size: 1024,
} }
err = repos.Chunks.Create(ctx, nil, chunk) err = repos.Chunks.Create(ctx, nil, chunk)
if err != nil { if err != nil {
t.Fatalf("failed to create chunk %s: %v", chunkHash, err) t.Fatalf("failed to create chunk %s: %v", chunkHash, err)
@@ -60,6 +62,7 @@ func TestBlobChunkRepository(t *testing.T) {
Offset: 1024, Offset: 1024,
Length: 2048, Length: 2048,
} }
err = repos.BlobChunks.Create(ctx, nil, bc2) err = repos.BlobChunks.Create(ctx, nil, bc2)
if err != nil { if err != nil {
t.Fatalf("failed to create second blob chunk: %v", err) t.Fatalf("failed to create second blob chunk: %v", err)
@@ -71,6 +74,7 @@ func TestBlobChunkRepository(t *testing.T) {
Offset: 3072, Offset: 3072,
Length: 512, Length: 512,
} }
err = repos.BlobChunks.Create(ctx, nil, bc3) err = repos.BlobChunks.Create(ctx, nil, bc3)
if err != nil { if err != nil {
t.Fatalf("failed to create third blob chunk: %v", err) t.Fatalf("failed to create third blob chunk: %v", err)
@@ -81,6 +85,7 @@ func TestBlobChunkRepository(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("failed to get blob chunks: %v", err) t.Fatalf("failed to get blob chunks: %v", err)
} }
if len(blobChunks) != 3 { if len(blobChunks) != 3 {
t.Errorf("expected 3 chunks, got %d", len(blobChunks)) t.Errorf("expected 3 chunks, got %d", len(blobChunks))
} }
@@ -98,12 +103,15 @@ func TestBlobChunkRepository(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("failed to get blob chunk by chunk hash: %v", err) t.Fatalf("failed to get blob chunk by chunk hash: %v", err)
} }
if bc == nil { if bc == nil {
t.Fatal("expected blob chunk, got nil") t.Fatal("expected blob chunk, got nil")
} }
if bc.BlobID != blob.ID { if bc.BlobID != blob.ID {
t.Errorf("wrong blob ID: expected %s, got %s", blob.ID, bc.BlobID) t.Errorf("wrong blob ID: expected %s, got %s", blob.ID, bc.BlobID)
} }
if bc.Offset != 1024 { if bc.Offset != 1024 {
t.Errorf("wrong offset: expected 1024, got %d", bc.Offset) t.Errorf("wrong offset: expected 1024, got %d", bc.Offset)
} }
@@ -113,6 +121,7 @@ func TestBlobChunkRepository(t *testing.T) {
if err == nil { if err == nil {
t.Fatal("duplicate blob_chunk insert should fail due to primary key constraint") t.Fatal("duplicate blob_chunk insert should fail due to primary key constraint")
} }
if !strings.Contains(err.Error(), "UNIQUE") && !strings.Contains(err.Error(), "constraint") { if !strings.Contains(err.Error(), "UNIQUE") && !strings.Contains(err.Error(), "constraint") {
t.Fatalf("expected constraint error, got: %v", err) t.Fatalf("expected constraint error, got: %v", err)
} }
@@ -122,6 +131,7 @@ func TestBlobChunkRepository(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("unexpected error: %v", err) t.Fatalf("unexpected error: %v", err)
} }
if bc != nil { if bc != nil {
t.Error("expected nil for non-existent chunk") t.Error("expected nil for non-existent chunk")
} }
@@ -150,6 +160,7 @@ func TestBlobChunkRepositoryMultipleBlobs(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("failed to create blob1: %v", err) t.Fatalf("failed to create blob1: %v", err)
} }
err = repos.Blobs.Create(ctx, nil, blob2) err = repos.Blobs.Create(ctx, nil, blob2)
if err != nil { if err != nil {
t.Fatalf("failed to create blob2: %v", err) t.Fatalf("failed to create blob2: %v", err)
@@ -162,6 +173,7 @@ func TestBlobChunkRepositoryMultipleBlobs(t *testing.T) {
ChunkHash: chunkHash, ChunkHash: chunkHash,
Size: 1024, Size: 1024,
} }
err = repos.Chunks.Create(ctx, nil, chunk) err = repos.Chunks.Create(ctx, nil, chunk)
if err != nil { if err != nil {
t.Fatalf("failed to create chunk %s: %v", chunkHash, err) t.Fatalf("failed to create chunk %s: %v", chunkHash, err)
@@ -189,6 +201,7 @@ func TestBlobChunkRepositoryMultipleBlobs(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("failed to get blob1 chunks: %v", err) t.Fatalf("failed to get blob1 chunks: %v", err)
} }
if len(chunks) != 2 { if len(chunks) != 2 {
t.Errorf("expected 2 chunks for blob1, got %d", len(chunks)) t.Errorf("expected 2 chunks for blob1, got %d", len(chunks))
} }
@@ -198,6 +211,7 @@ func TestBlobChunkRepositoryMultipleBlobs(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("failed to get blob2 chunks: %v", err) t.Fatalf("failed to get blob2 chunks: %v", err)
} }
if len(chunks) != 2 { if len(chunks) != 2 {
t.Errorf("expected 2 chunks for blob2, got %d", len(chunks)) t.Errorf("expected 2 chunks for blob2, got %d", len(chunks))
} }
@@ -207,6 +221,7 @@ func TestBlobChunkRepositoryMultipleBlobs(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("failed to get shared chunk: %v", err) t.Fatalf("failed to get shared chunk: %v", err)
} }
if bc == nil { if bc == nil {
t.Fatal("expected shared chunk, got nil") t.Fatal("expected shared chunk, got nil")
} }

View File

@@ -24,10 +24,12 @@ func (r *BlobRepository) Create(ctx context.Context, tx *sql.Tx, blob *Blob) err
` `
var finishedTS, uploadedTS *int64 var finishedTS, uploadedTS *int64
if blob.FinishedTS != nil { if blob.FinishedTS != nil {
ts := blob.FinishedTS.Unix() ts := blob.FinishedTS.Unix()
finishedTS = &ts finishedTS = &ts
} }
if blob.UploadedTS != nil { if blob.UploadedTS != nil {
ts := blob.UploadedTS.Unix() ts := blob.UploadedTS.Unix()
uploadedTS = &ts uploadedTS = &ts
@@ -56,9 +58,11 @@ func (r *BlobRepository) GetByHash(ctx context.Context, hash string) (*Blob, err
WHERE blob_hash = ? WHERE blob_hash = ?
` `
var blob Blob var (
var createdTSUnix int64 blob Blob
var finishedTSUnix, uploadedTSUnix sql.NullInt64 createdTSUnix int64
finishedTSUnix, uploadedTSUnix sql.NullInt64
)
err := r.db.conn.QueryRowContext(ctx, query, hash).Scan( err := r.db.conn.QueryRowContext(ctx, query, hash).Scan(
&blob.ID, &blob.ID,
@@ -73,6 +77,7 @@ func (r *BlobRepository) GetByHash(ctx context.Context, hash string) (*Blob, err
if err == sql.ErrNoRows { if err == sql.ErrNoRows {
return nil, nil return nil, nil
} }
if err != nil { if err != nil {
return nil, fmt.Errorf("querying blob: %w", err) return nil, fmt.Errorf("querying blob: %w", err)
} }
@@ -82,10 +87,12 @@ func (r *BlobRepository) GetByHash(ctx context.Context, hash string) (*Blob, err
ts := time.Unix(finishedTSUnix.Int64, 0).UTC() ts := time.Unix(finishedTSUnix.Int64, 0).UTC()
blob.FinishedTS = &ts blob.FinishedTS = &ts
} }
if uploadedTSUnix.Valid { if uploadedTSUnix.Valid {
ts := time.Unix(uploadedTSUnix.Int64, 0).UTC() ts := time.Unix(uploadedTSUnix.Int64, 0).UTC()
blob.UploadedTS = &ts blob.UploadedTS = &ts
} }
return &blob, nil return &blob, nil
} }
@@ -97,9 +104,11 @@ func (r *BlobRepository) GetByID(ctx context.Context, id string) (*Blob, error)
WHERE id = ? WHERE id = ?
` `
var blob Blob var (
var createdTSUnix int64 blob Blob
var finishedTSUnix, uploadedTSUnix sql.NullInt64 createdTSUnix int64
finishedTSUnix, uploadedTSUnix sql.NullInt64
)
err := r.db.conn.QueryRowContext(ctx, query, id).Scan( err := r.db.conn.QueryRowContext(ctx, query, id).Scan(
&blob.ID, &blob.ID,
@@ -114,6 +123,7 @@ func (r *BlobRepository) GetByID(ctx context.Context, id string) (*Blob, error)
if err == sql.ErrNoRows { if err == sql.ErrNoRows {
return nil, nil return nil, nil
} }
if err != nil { if err != nil {
return nil, fmt.Errorf("querying blob: %w", err) return nil, fmt.Errorf("querying blob: %w", err)
} }
@@ -123,10 +133,12 @@ func (r *BlobRepository) GetByID(ctx context.Context, id string) (*Blob, error)
ts := time.Unix(finishedTSUnix.Int64, 0).UTC() ts := time.Unix(finishedTSUnix.Int64, 0).UTC()
blob.FinishedTS = &ts blob.FinishedTS = &ts
} }
if uploadedTSUnix.Valid { if uploadedTSUnix.Valid {
ts := time.Unix(uploadedTSUnix.Int64, 0).UTC() ts := time.Unix(uploadedTSUnix.Int64, 0).UTC()
blob.UploadedTS = &ts blob.UploadedTS = &ts
} }
return &blob, nil return &blob, nil
} }
@@ -146,11 +158,15 @@ func (r *BlobRepository) GetAll(ctx context.Context) (map[string]*Blob, error) {
defer CloseRows(rows) defer CloseRows(rows)
out := make(map[string]*Blob) out := make(map[string]*Blob)
for rows.Next() { for rows.Next() {
var blob Blob var (
var createdTSUnix int64 blob Blob
var finishedTSUnix, uploadedTSUnix sql.NullInt64 createdTSUnix int64
if err := rows.Scan( finishedTSUnix, uploadedTSUnix sql.NullInt64
)
err := rows.Scan(
&blob.ID, &blob.ID,
&blob.Hash, &blob.Hash,
&createdTSUnix, &createdTSUnix,
@@ -158,20 +174,25 @@ func (r *BlobRepository) GetAll(ctx context.Context) (map[string]*Blob, error) {
&blob.UncompressedSize, &blob.UncompressedSize,
&blob.CompressedSize, &blob.CompressedSize,
&uploadedTSUnix, &uploadedTSUnix,
); err != nil { )
if err != nil {
return nil, fmt.Errorf("scanning blob: %w", err) return nil, fmt.Errorf("scanning blob: %w", err)
} }
blob.CreatedTS = time.Unix(createdTSUnix, 0).UTC() blob.CreatedTS = time.Unix(createdTSUnix, 0).UTC()
if finishedTSUnix.Valid { if finishedTSUnix.Valid {
ts := time.Unix(finishedTSUnix.Int64, 0).UTC() ts := time.Unix(finishedTSUnix.Int64, 0).UTC()
blob.FinishedTS = &ts blob.FinishedTS = &ts
} }
if uploadedTSUnix.Valid { if uploadedTSUnix.Valid {
ts := time.Unix(uploadedTSUnix.Int64, 0).UTC() ts := time.Unix(uploadedTSUnix.Int64, 0).UTC()
blob.UploadedTS = &ts blob.UploadedTS = &ts
} }
out[blob.ID.String()] = &blob out[blob.ID.String()] = &blob
} }
return out, rows.Err() return out, rows.Err()
} }
@@ -184,6 +205,7 @@ func (r *BlobRepository) UpdateFinished(ctx context.Context, tx *sql.Tx, id stri
` `
now := time.Now().UTC().Unix() now := time.Now().UTC().Unix()
var err error var err error
if tx != nil { if tx != nil {
_, err = tx.ExecContext(ctx, query, hash, now, uncompressedSize, compressedSize, id) _, err = tx.ExecContext(ctx, query, hash, now, uncompressedSize, compressedSize, id)
@@ -207,6 +229,7 @@ func (r *BlobRepository) UpdateUploaded(ctx context.Context, tx *sql.Tx, id stri
` `
now := time.Now().UTC().Unix() now := time.Now().UTC().Unix()
var err error var err error
if tx != nil { if tx != nil {
_, err = tx.ExecContext(ctx, query, now, id) _, err = tx.ExecContext(ctx, query, now, id)

View File

@@ -32,12 +32,15 @@ func TestBlobRepository(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("failed to get blob: %v", err) t.Fatalf("failed to get blob: %v", err)
} }
if retrieved == nil { if retrieved == nil {
t.Fatal("expected blob, got nil") t.Fatal("expected blob, got nil")
} }
if retrieved.Hash != blob.Hash { if retrieved.Hash != blob.Hash {
t.Errorf("blob hash mismatch: got %s, want %s", retrieved.Hash, blob.Hash) t.Errorf("blob hash mismatch: got %s, want %s", retrieved.Hash, blob.Hash)
} }
if !retrieved.CreatedTS.Equal(blob.CreatedTS) { if !retrieved.CreatedTS.Equal(blob.CreatedTS) {
t.Errorf("created timestamp mismatch: got %v, want %v", retrieved.CreatedTS, blob.CreatedTS) t.Errorf("created timestamp mismatch: got %v, want %v", retrieved.CreatedTS, blob.CreatedTS)
} }
@@ -47,9 +50,11 @@ func TestBlobRepository(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("failed to get blob by ID: %v", err) t.Fatalf("failed to get blob by ID: %v", err)
} }
if retrievedByID == nil { if retrievedByID == nil {
t.Fatal("expected blob, got nil") t.Fatal("expected blob, got nil")
} }
if retrievedByID.ID != blob.ID { if retrievedByID.ID != blob.ID {
t.Errorf("blob ID mismatch: got %s, want %s", retrievedByID.ID, blob.ID) t.Errorf("blob ID mismatch: got %s, want %s", retrievedByID.ID, blob.ID)
} }
@@ -60,6 +65,7 @@ func TestBlobRepository(t *testing.T) {
Hash: types.BlobHash("blobhash456"), Hash: types.BlobHash("blobhash456"),
CreatedTS: time.Now().Truncate(time.Second), CreatedTS: time.Now().Truncate(time.Second),
} }
err = repo.Create(ctx, nil, blob2) err = repo.Create(ctx, nil, blob2)
if err != nil { if err != nil {
t.Fatalf("failed to create second blob: %v", err) t.Fatalf("failed to create second blob: %v", err)
@@ -67,6 +73,7 @@ func TestBlobRepository(t *testing.T) {
// Test UpdateFinished // Test UpdateFinished
now := time.Now() now := time.Now()
err = repo.UpdateFinished(ctx, nil, blob.ID.String(), blob.Hash.String(), 1000, 500) err = repo.UpdateFinished(ctx, nil, blob.ID.String(), blob.Hash.String(), 1000, 500)
if err != nil { if err != nil {
t.Fatalf("failed to update blob as finished: %v", err) t.Fatalf("failed to update blob as finished: %v", err)
@@ -77,12 +84,15 @@ func TestBlobRepository(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("failed to get updated blob: %v", err) t.Fatalf("failed to get updated blob: %v", err)
} }
if updated.FinishedTS == nil { if updated.FinishedTS == nil {
t.Fatal("expected finished timestamp to be set") t.Fatal("expected finished timestamp to be set")
} }
if updated.UncompressedSize != 1000 { if updated.UncompressedSize != 1000 {
t.Errorf("expected uncompressed size 1000, got %d", updated.UncompressedSize) t.Errorf("expected uncompressed size 1000, got %d", updated.UncompressedSize)
} }
if updated.CompressedSize != 500 { if updated.CompressedSize != 500 {
t.Errorf("expected compressed size 500, got %d", updated.CompressedSize) t.Errorf("expected compressed size 500, got %d", updated.CompressedSize)
} }
@@ -98,6 +108,7 @@ func TestBlobRepository(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("failed to get uploaded blob: %v", err) t.Fatalf("failed to get uploaded blob: %v", err)
} }
if uploaded.UploadedTS == nil { if uploaded.UploadedTS == nil {
t.Fatal("expected uploaded timestamp to be set") t.Fatal("expected uploaded timestamp to be set")
} }

View File

@@ -19,10 +19,12 @@ func TestCascadeDeleteDebug(t *testing.T) {
// Check if foreign keys are enabled // Check if foreign keys are enabled
var fkEnabled int var fkEnabled int
err := db.conn.QueryRow("PRAGMA foreign_keys").Scan(&fkEnabled) err := db.conn.QueryRow("PRAGMA foreign_keys").Scan(&fkEnabled)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
t.Logf("Foreign keys enabled: %d", fkEnabled) t.Logf("Foreign keys enabled: %d", fkEnabled)
// Create a file // Create a file
@@ -34,18 +36,21 @@ func TestCascadeDeleteDebug(t *testing.T) {
UID: 1000, UID: 1000,
GID: 1000, GID: 1000,
} }
err = repos.Files.Create(ctx, nil, file) err = repos.Files.Create(ctx, nil, file)
if err != nil { if err != nil {
t.Fatalf("failed to create file: %v", err) t.Fatalf("failed to create file: %v", err)
} }
t.Logf("Created file with ID: %s", file.ID) t.Logf("Created file with ID: %s", file.ID)
// Create chunks and file-chunk mappings // Create chunks and file-chunk mappings
for i := 0; i < 3; i++ { for i := range 3 {
chunk := &Chunk{ chunk := &Chunk{
ChunkHash: types.ChunkHash(fmt.Sprintf("cascade-chunk-%d", i)), ChunkHash: types.ChunkHash(fmt.Sprintf("cascade-chunk-%d", i)),
Size: 1024, Size: 1024,
} }
err = repos.Chunks.Create(ctx, nil, chunk) err = repos.Chunks.Create(ctx, nil, chunk)
if err != nil { if err != nil {
t.Fatalf("failed to create chunk: %v", err) t.Fatalf("failed to create chunk: %v", err)
@@ -56,10 +61,12 @@ func TestCascadeDeleteDebug(t *testing.T) {
Idx: i, Idx: i,
ChunkHash: chunk.ChunkHash, ChunkHash: chunk.ChunkHash,
} }
err = repos.FileChunks.Create(ctx, nil, fc) err = repos.FileChunks.Create(ctx, nil, fc)
if err != nil { if err != nil {
t.Fatalf("failed to create file chunk: %v", err) t.Fatalf("failed to create file chunk: %v", err)
} }
t.Logf("Created file chunk mapping: file_id=%s, idx=%d, chunk=%s", fc.FileID, fc.Idx, fc.ChunkHash) t.Logf("Created file chunk mapping: file_id=%s, idx=%d, chunk=%s", fc.FileID, fc.Idx, fc.ChunkHash)
} }
@@ -68,10 +75,12 @@ func TestCascadeDeleteDebug(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
t.Logf("File chunks before delete: %d", len(fileChunks)) t.Logf("File chunks before delete: %d", len(fileChunks))
// Check the foreign key constraint // Check the foreign key constraint
var fkInfo string var fkInfo string
err = db.conn.QueryRow(` err = db.conn.QueryRow(`
SELECT sql FROM sqlite_master SELECT sql FROM sqlite_master
WHERE type='table' AND name='file_chunks' WHERE type='table' AND name='file_chunks'
@@ -79,10 +88,12 @@ func TestCascadeDeleteDebug(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
t.Logf("file_chunks table definition:\n%s", fkInfo) t.Logf("file_chunks table definition:\n%s", fkInfo)
// Delete the file // Delete the file
t.Log("Deleting file...") t.Log("Deleting file...")
err = repos.Files.DeleteByID(ctx, nil, file.ID) err = repos.Files.DeleteByID(ctx, nil, file.ID)
if err != nil { if err != nil {
t.Fatalf("failed to delete file: %v", err) t.Fatalf("failed to delete file: %v", err)
@@ -93,6 +104,7 @@ func TestCascadeDeleteDebug(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
if deletedFile != nil { if deletedFile != nil {
t.Error("file should have been deleted") t.Error("file should have been deleted")
} else { } else {
@@ -104,14 +116,17 @@ func TestCascadeDeleteDebug(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
t.Logf("File chunks after delete: %d", len(fileChunks)) t.Logf("File chunks after delete: %d", len(fileChunks))
// Manually check the database // Manually check the database
var count int var count int
err = db.conn.QueryRow("SELECT COUNT(*) FROM file_chunks WHERE file_id = ?", file.ID).Scan(&count) err = db.conn.QueryRow("SELECT COUNT(*) FROM file_chunks WHERE file_id = ?", file.ID).Scan(&count)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
t.Logf("Manual count of file_chunks for deleted file: %d", count) t.Logf("Manual count of file_chunks for deleted file: %d", count)
if len(fileChunks) != 0 { if len(fileChunks) != 0 {

View File

@@ -4,6 +4,7 @@ import (
"context" "context"
"database/sql" "database/sql"
"fmt" "fmt"
"strings"
"sneak.berlin/go/vaultik/internal/types" "sneak.berlin/go/vaultik/internal/types"
) )
@@ -90,18 +91,25 @@ func (r *ChunkFileRepository) GetByFileID(ctx context.Context, fileID types.File
// scanChunkFiles is a helper that scans chunk file rows // scanChunkFiles is a helper that scans chunk file rows
func (r *ChunkFileRepository) scanChunkFiles(rows *sql.Rows) ([]*ChunkFile, error) { func (r *ChunkFileRepository) scanChunkFiles(rows *sql.Rows) ([]*ChunkFile, error) {
var chunkFiles []*ChunkFile var chunkFiles []*ChunkFile
for rows.Next() { for rows.Next() {
var cf ChunkFile var (
var chunkHashStr, fileIDStr string cf ChunkFile
chunkHashStr, fileIDStr string
)
err := rows.Scan(&chunkHashStr, &fileIDStr, &cf.FileOffset, &cf.Length) err := rows.Scan(&chunkHashStr, &fileIDStr, &cf.FileOffset, &cf.Length)
if err != nil { if err != nil {
return nil, fmt.Errorf("scanning chunk file: %w", err) return nil, fmt.Errorf("scanning chunk file: %w", err)
} }
cf.ChunkHash = types.ChunkHash(chunkHashStr) cf.ChunkHash = types.ChunkHash(chunkHashStr)
cf.FileID, err = types.ParseFileID(fileIDStr) cf.FileID, err = types.ParseFileID(fileIDStr)
if err != nil { if err != nil {
return nil, fmt.Errorf("parsing file ID: %w", err) return nil, fmt.Errorf("parsing file ID: %w", err)
} }
chunkFiles = append(chunkFiles, &cf) chunkFiles = append(chunkFiles, &cf)
} }
@@ -136,14 +144,13 @@ func (r *ChunkFileRepository) DeleteByFileIDs(ctx context.Context, tx *sql.Tx, f
const batchSize = 500 const batchSize = 500
for i := 0; i < len(fileIDs); i += batchSize { for i := 0; i < len(fileIDs); i += batchSize {
end := i + batchSize end := min(i+batchSize, len(fileIDs))
if end > len(fileIDs) {
end = len(fileIDs)
}
batch := fileIDs[i:end] batch := fileIDs[i:end]
query := "DELETE FROM chunk_files WHERE file_id IN (?" + repeatPlaceholder(len(batch)-1) + ")" query := "DELETE FROM chunk_files WHERE file_id IN (?" + repeatPlaceholder(len(batch)-1) + ")"
args := make([]interface{}, len(batch))
args := make([]any, len(batch))
for j, id := range batch { for j, id := range batch {
args[j] = id.String() args[j] = id.String()
} }
@@ -154,6 +161,7 @@ func (r *ChunkFileRepository) DeleteByFileIDs(ctx context.Context, tx *sql.Tx, f
} else { } else {
_, err = r.db.ExecWithLog(ctx, query, args...) _, err = r.db.ExecWithLog(ctx, query, args...)
} }
if err != nil { if err != nil {
return fmt.Errorf("batch deleting chunk_files: %w", err) return fmt.Errorf("batch deleting chunk_files: %w", err)
} }
@@ -172,21 +180,28 @@ func (r *ChunkFileRepository) CreateBatch(ctx context.Context, tx *sql.Tx, cfs [
const batchSize = 200 const batchSize = 200
for i := 0; i < len(cfs); i += batchSize { for i := 0; i < len(cfs); i += batchSize {
end := i + batchSize end := min(i+batchSize, len(cfs))
if end > len(cfs) {
end = len(cfs)
}
batch := cfs[i:end] batch := cfs[i:end]
query := "INSERT INTO chunk_files (chunk_hash, file_id, file_offset, length) VALUES " query := "INSERT INTO chunk_files (chunk_hash, file_id, file_offset, length) VALUES "
args := make([]interface{}, 0, len(batch)*4)
args := make([]any, 0, len(batch)*4)
var querySb183 strings.Builder
for j, cf := range batch { for j, cf := range batch {
if j > 0 { if j > 0 {
query += ", " querySb183.WriteString(", ")
} }
query += "(?, ?, ?, ?)"
querySb183.WriteString("(?, ?, ?, ?)")
args = append(args, cf.ChunkHash.String(), cf.FileID.String(), cf.FileOffset, cf.Length) args = append(args, cf.ChunkHash.String(), cf.FileID.String(), cf.FileOffset, cf.Length)
} }
query += querySb183.String()
query += " ON CONFLICT(chunk_hash, file_id) DO NOTHING" query += " ON CONFLICT(chunk_hash, file_id) DO NOTHING"
var err error var err error
@@ -195,6 +210,7 @@ func (r *ChunkFileRepository) CreateBatch(ctx context.Context, tx *sql.Tx, cfs [
} else { } else {
_, err = r.db.ExecWithLog(ctx, query, args...) _, err = r.db.ExecWithLog(ctx, query, args...)
} }
if err != nil { if err != nil {
return fmt.Errorf("batch inserting chunk_files: %w", err) return fmt.Errorf("batch inserting chunk_files: %w", err)
} }

View File

@@ -28,6 +28,7 @@ func TestChunkFileRepository(t *testing.T) {
GID: 1000, GID: 1000,
LinkTarget: "", LinkTarget: "",
} }
err := fileRepo.Create(ctx, nil, file1) err := fileRepo.Create(ctx, nil, file1)
if err != nil { if err != nil {
t.Fatalf("failed to create file1: %v", err) t.Fatalf("failed to create file1: %v", err)
@@ -42,6 +43,7 @@ func TestChunkFileRepository(t *testing.T) {
GID: 1000, GID: 1000,
LinkTarget: "", LinkTarget: "",
} }
err = fileRepo.Create(ctx, nil, file2) err = fileRepo.Create(ctx, nil, file2)
if err != nil { if err != nil {
t.Fatalf("failed to create file2: %v", err) t.Fatalf("failed to create file2: %v", err)
@@ -52,6 +54,7 @@ func TestChunkFileRepository(t *testing.T) {
ChunkHash: types.ChunkHash("chunk1"), ChunkHash: types.ChunkHash("chunk1"),
Size: 1024, Size: 1024,
} }
err = chunksRepo.Create(ctx, nil, chunk) err = chunksRepo.Create(ctx, nil, chunk)
if err != nil { if err != nil {
t.Fatalf("failed to create chunk: %v", err) t.Fatalf("failed to create chunk: %v", err)
@@ -77,6 +80,7 @@ func TestChunkFileRepository(t *testing.T) {
FileOffset: 2048, FileOffset: 2048,
Length: 1024, Length: 1024,
} }
err = repo.Create(ctx, nil, cf2) err = repo.Create(ctx, nil, cf2)
if err != nil { if err != nil {
t.Fatalf("failed to create second chunk file: %v", err) t.Fatalf("failed to create second chunk file: %v", err)
@@ -87,6 +91,7 @@ func TestChunkFileRepository(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("failed to get chunk files: %v", err) t.Fatalf("failed to get chunk files: %v", err)
} }
if len(chunkFiles) != 2 { if len(chunkFiles) != 2 {
t.Errorf("expected 2 files for chunk, got %d", len(chunkFiles)) t.Errorf("expected 2 files for chunk, got %d", len(chunkFiles))
} }
@@ -94,14 +99,17 @@ func TestChunkFileRepository(t *testing.T) {
// Verify both files are returned // Verify both files are returned
foundFile1 := false foundFile1 := false
foundFile2 := false foundFile2 := false
for _, cf := range chunkFiles { for _, cf := range chunkFiles {
if cf.FileID == file1.ID && cf.FileOffset == 0 { if cf.FileID == file1.ID && cf.FileOffset == 0 {
foundFile1 = true foundFile1 = true
} }
if cf.FileID == file2.ID && cf.FileOffset == 2048 { if cf.FileID == file2.ID && cf.FileOffset == 2048 {
foundFile2 = true foundFile2 = true
} }
} }
if !foundFile1 || !foundFile2 { if !foundFile1 || !foundFile2 {
t.Error("not all expected files found") t.Error("not all expected files found")
} }
@@ -111,9 +119,11 @@ func TestChunkFileRepository(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("failed to get chunks by file ID: %v", err) t.Fatalf("failed to get chunks by file ID: %v", err)
} }
if len(chunkFiles) != 1 { if len(chunkFiles) != 1 {
t.Errorf("expected 1 chunk for file, got %d", len(chunkFiles)) t.Errorf("expected 1 chunk for file, got %d", len(chunkFiles))
} }
if chunkFiles[0].ChunkHash != types.ChunkHash("chunk1") { if chunkFiles[0].ChunkHash != types.ChunkHash("chunk1") {
t.Errorf("wrong chunk hash: expected chunk1, got %s", chunkFiles[0].ChunkHash) t.Errorf("wrong chunk hash: expected chunk1, got %s", chunkFiles[0].ChunkHash)
} }
@@ -143,9 +153,11 @@ func TestChunkFileRepositoryComplexDeduplication(t *testing.T) {
if err := fileRepo.Create(ctx, nil, file1); err != nil { if err := fileRepo.Create(ctx, nil, file1); err != nil {
t.Fatalf("failed to create file1: %v", err) t.Fatalf("failed to create file1: %v", err)
} }
if err := fileRepo.Create(ctx, nil, file2); err != nil { if err := fileRepo.Create(ctx, nil, file2); err != nil {
t.Fatalf("failed to create file2: %v", err) t.Fatalf("failed to create file2: %v", err)
} }
if err := fileRepo.Create(ctx, nil, file3); err != nil { if err := fileRepo.Create(ctx, nil, file3); err != nil {
t.Fatalf("failed to create file3: %v", err) t.Fatalf("failed to create file3: %v", err)
} }
@@ -157,6 +169,7 @@ func TestChunkFileRepositoryComplexDeduplication(t *testing.T) {
ChunkHash: chunkHash, ChunkHash: chunkHash,
Size: 1024, Size: 1024,
} }
err := chunksRepo.Create(ctx, nil, chunk) err := chunksRepo.Create(ctx, nil, chunk)
if err != nil { if err != nil {
t.Fatalf("failed to create chunk %s: %v", chunkHash, err) t.Fatalf("failed to create chunk %s: %v", chunkHash, err)
@@ -194,6 +207,7 @@ func TestChunkFileRepositoryComplexDeduplication(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("failed to get files for chunk1: %v", err) t.Fatalf("failed to get files for chunk1: %v", err)
} }
if len(files) != 2 { if len(files) != 2 {
t.Errorf("expected 2 files for chunk1, got %d", len(files)) t.Errorf("expected 2 files for chunk1, got %d", len(files))
} }
@@ -203,6 +217,7 @@ func TestChunkFileRepositoryComplexDeduplication(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("failed to get files for chunk2: %v", err) t.Fatalf("failed to get files for chunk2: %v", err)
} }
if len(files) != 2 { if len(files) != 2 {
t.Errorf("expected 2 files for chunk2, got %d", len(files)) t.Errorf("expected 2 files for chunk2, got %d", len(files))
} }
@@ -212,6 +227,7 @@ func TestChunkFileRepositoryComplexDeduplication(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("failed to get chunks for file2: %v", err) t.Fatalf("failed to get chunks for file2: %v", err)
} }
if len(file2Chunks) != 3 { if len(file2Chunks) != 3 {
t.Errorf("expected 3 chunks for file2, got %d", len(file2Chunks)) t.Errorf("expected 3 chunks for file2, got %d", len(file2Chunks))
} }

View File

@@ -4,6 +4,7 @@ import (
"context" "context"
"database/sql" "database/sql"
"fmt" "fmt"
"strings"
"sneak.berlin/go/vaultik/internal/log" "sneak.berlin/go/vaultik/internal/log"
) )
@@ -54,6 +55,7 @@ func (r *ChunkRepository) GetByHash(ctx context.Context, hash string) (*Chunk, e
if err == sql.ErrNoRows { if err == sql.ErrNoRows {
return nil, nil return nil, nil
} }
if err != nil { if err != nil {
return nil, fmt.Errorf("querying chunk: %w", err) return nil, fmt.Errorf("querying chunk: %w", err)
} }
@@ -71,14 +73,22 @@ func (r *ChunkRepository) GetByHashes(ctx context.Context, hashes []string) ([]*
FROM chunks FROM chunks
WHERE chunk_hash IN (` WHERE chunk_hash IN (`
args := make([]interface{}, len(hashes)) args := make([]any, len(hashes))
var querySb75 strings.Builder
for i, hash := range hashes { for i, hash := range hashes {
if i > 0 { if i > 0 {
query += ", " querySb75.WriteString(", ")
} }
query += "?"
querySb75.WriteString("?")
args[i] = hash args[i] = hash
} }
query += querySb75.String()
query += ") ORDER BY chunk_hash" query += ") ORDER BY chunk_hash"
rows, err := r.db.conn.QueryContext(ctx, query, args...) rows, err := r.db.conn.QueryContext(ctx, query, args...)
@@ -88,6 +98,7 @@ func (r *ChunkRepository) GetByHashes(ctx context.Context, hashes []string) ([]*
defer CloseRows(rows) defer CloseRows(rows)
var chunks []*Chunk var chunks []*Chunk
for rows.Next() { for rows.Next() {
var chunk Chunk var chunk Chunk
@@ -122,6 +133,7 @@ func (r *ChunkRepository) ListUnpacked(ctx context.Context, limit int) ([]*Chunk
defer CloseRows(rows) defer CloseRows(rows)
var chunks []*Chunk var chunks []*Chunk
for rows.Next() { for rows.Next() {
var chunk Chunk var chunk Chunk

View File

@@ -19,6 +19,7 @@ func (r *ChunkRepository) List(ctx context.Context) ([]*Chunk, error) {
defer CloseRows(rows) defer CloseRows(rows)
var chunks []*Chunk var chunks []*Chunk
for rows.Next() { for rows.Next() {
var chunk Chunk var chunk Chunk

View File

@@ -30,12 +30,15 @@ func TestChunkRepository(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("failed to get chunk: %v", err) t.Fatalf("failed to get chunk: %v", err)
} }
if retrieved == nil { if retrieved == nil {
t.Fatal("expected chunk, got nil") t.Fatal("expected chunk, got nil")
} }
if retrieved.ChunkHash != chunk.ChunkHash { if retrieved.ChunkHash != chunk.ChunkHash {
t.Errorf("chunk hash mismatch: got %s, want %s", retrieved.ChunkHash, chunk.ChunkHash) t.Errorf("chunk hash mismatch: got %s, want %s", retrieved.ChunkHash, chunk.ChunkHash)
} }
if retrieved.Size != chunk.Size { if retrieved.Size != chunk.Size {
t.Errorf("size mismatch: got %d, want %d", retrieved.Size, chunk.Size) t.Errorf("size mismatch: got %d, want %d", retrieved.Size, chunk.Size)
} }
@@ -51,6 +54,7 @@ func TestChunkRepository(t *testing.T) {
ChunkHash: types.ChunkHash("chunkhash456"), ChunkHash: types.ChunkHash("chunkhash456"),
Size: 8192, Size: 8192,
} }
err = repo.Create(ctx, nil, chunk2) err = repo.Create(ctx, nil, chunk2)
if err != nil { if err != nil {
t.Fatalf("failed to create second chunk: %v", err) t.Fatalf("failed to create second chunk: %v", err)
@@ -60,6 +64,7 @@ func TestChunkRepository(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("failed to get chunks by hashes: %v", err) t.Fatalf("failed to get chunks by hashes: %v", err)
} }
if len(chunks) != 2 { if len(chunks) != 2 {
t.Errorf("expected 2 chunks, got %d", len(chunks)) t.Errorf("expected 2 chunks, got %d", len(chunks))
} }
@@ -69,6 +74,7 @@ func TestChunkRepository(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("failed to list unpacked chunks: %v", err) t.Fatalf("failed to list unpacked chunks: %v", err)
} }
if len(unpacked) != 2 { if len(unpacked) != 2 {
t.Errorf("expected 2 unpacked chunks, got %d", len(unpacked)) t.Errorf("expected 2 unpacked chunks, got %d", len(unpacked))
} }
@@ -86,6 +92,7 @@ func TestChunkRepositoryNotFound(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("unexpected error: %v", err) t.Fatalf("unexpected error: %v", err)
} }
if chunk != nil { if chunk != nil {
t.Error("expected nil for non-existent chunk") t.Error("expected nil for non-existent chunk")
} }
@@ -95,6 +102,7 @@ func TestChunkRepositoryNotFound(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("unexpected error: %v", err) t.Fatalf("unexpected error: %v", err)
} }
if chunks != nil { if chunks != nil {
t.Error("expected nil for empty hash list") t.Error("expected nil for empty hash list")
} }

View File

@@ -57,8 +57,8 @@ func ParseMigrationVersion(filename string) (int, error) {
// Split on underscore to separate version from description. // Split on underscore to separate version from description.
// If there's no underscore, the entire stem is the version. // If there's no underscore, the entire stem is the version.
versionStr := name versionStr := name
if idx := strings.IndexByte(name, '_'); idx >= 0 { if before, _, ok := strings.Cut(name, "_"); ok {
versionStr = name[:idx] versionStr = before
} }
if versionStr == "" { if versionStr == "" {
@@ -98,6 +98,7 @@ func New(ctx context.Context, path string) (*DB, error) {
// First attempt with standard WAL mode // First attempt with standard WAL mode
log.Debug("Attempting to open database with WAL mode", "path", path) log.Debug("Attempting to open database with WAL mode", "path", path)
conn, err := sql.Open( conn, err := sql.Open(
"sqlite", "sqlite",
path+"?_journal_mode=WAL&_synchronous=NORMAL&_busy_timeout=10000&_locking_mode=NORMAL&_foreign_keys=ON", path+"?_journal_mode=WAL&_synchronous=NORMAL&_busy_timeout=10000&_locking_mode=NORMAL&_foreign_keys=ON",
@@ -110,7 +111,8 @@ func New(ctx context.Context, path string) (*DB, error) {
conn.SetMaxOpenConns(1) conn.SetMaxOpenConns(1)
conn.SetMaxIdleConns(1) conn.SetMaxIdleConns(1)
if err := conn.PingContext(ctx); err == nil { err := conn.PingContext(ctx)
if err == nil {
// Success on first try // Success on first try
log.Debug("Database opened successfully with WAL mode", "path", path) log.Debug("Database opened successfully with WAL mode", "path", path)
@@ -120,13 +122,19 @@ func New(ctx context.Context, path string) (*DB, error) {
} }
db := &DB{conn: conn, path: path} db := &DB{conn: conn, path: path}
if err := applyMigrations(ctx, conn); err != nil {
err := applyMigrations(ctx, conn)
if err != nil {
_ = conn.Close() _ = conn.Close()
return nil, fmt.Errorf("applying migrations: %w", err) return nil, fmt.Errorf("applying migrations: %w", err)
} }
return db, nil return db, nil
} }
log.Debug("Failed to ping database, closing connection", "path", path, "error", err) log.Debug("Failed to ping database, closing connection", "path", path, "error", err)
_ = conn.Close() _ = conn.Close()
} }
@@ -135,6 +143,7 @@ func New(ctx context.Context, path string) (*DB, error) {
"Database appears locked, attempting recovery with TRUNCATE mode", "Database appears locked, attempting recovery with TRUNCATE mode",
"path", path, "path", path,
) )
conn, err = sql.Open( conn, err = sql.Open(
"sqlite", "sqlite",
path+"?_journal_mode=TRUNCATE&_synchronous=NORMAL&_busy_timeout=10000&_foreign_keys=ON", path+"?_journal_mode=TRUNCATE&_synchronous=NORMAL&_busy_timeout=10000&_foreign_keys=ON",
@@ -152,7 +161,9 @@ func New(ctx context.Context, path string) (*DB, error) {
if err := conn.PingContext(ctx); err != nil { if err := conn.PingContext(ctx); err != nil {
log.Debug("Failed to ping database in recovery mode, closing", "path", path, "error", err) log.Debug("Failed to ping database in recovery mode, closing", "path", path, "error", err)
_ = conn.Close() _ = conn.Close()
return nil, fmt.Errorf( return nil, fmt.Errorf(
"database still locked after recovery attempt: %w", "database still locked after recovery attempt: %w",
err, err,
@@ -163,6 +174,7 @@ func New(ctx context.Context, path string) (*DB, error) {
// Switch back to WAL mode // Switch back to WAL mode
log.Debug("Switching database back to WAL mode", "path", path) log.Debug("Switching database back to WAL mode", "path", path)
if _, err := conn.ExecContext(ctx, "PRAGMA journal_mode=WAL"); err != nil { if _, err := conn.ExecContext(ctx, "PRAGMA journal_mode=WAL"); err != nil {
log.Warn("Failed to switch back to WAL mode", "path", path, "error", err) log.Warn("Failed to switch back to WAL mode", "path", path, "error", err)
} }
@@ -175,10 +187,12 @@ func New(ctx context.Context, path string) (*DB, error) {
db := &DB{conn: conn, path: path} db := &DB{conn: conn, path: path}
if err := applyMigrations(ctx, conn); err != nil { if err := applyMigrations(ctx, conn); err != nil {
_ = conn.Close() _ = conn.Close()
return nil, fmt.Errorf("applying migrations: %w", err) return nil, fmt.Errorf("applying migrations: %w", err)
} }
log.Debug("Database connection established successfully", "path", path) log.Debug("Database connection established successfully", "path", path)
return db, nil return db, nil
} }
@@ -187,11 +201,16 @@ func New(ctx context.Context, path string) (*DB, error) {
// Returns an error if the database connection cannot be closed properly. // Returns an error if the database connection cannot be closed properly.
func (db *DB) Close() error { func (db *DB) Close() error {
log.Debug("Closing database connection", "path", db.path) log.Debug("Closing database connection", "path", db.path)
if err := db.conn.Close(); err != nil {
err := db.conn.Close()
if err != nil {
log.Error("Failed to close database", "path", db.path, "error", err) log.Error("Failed to close database", "path", db.path, "error", err)
return fmt.Errorf("failed to close database: %w", err) return fmt.Errorf("failed to close database: %w", err)
} }
log.Debug("Database connection closed successfully", "path", db.path) log.Debug("Database connection closed successfully", "path", db.path)
return nil return nil
} }
@@ -227,9 +246,10 @@ func (db *DB) BeginTx(
func (db *DB) ExecWithLog( func (db *DB) ExecWithLog(
ctx context.Context, ctx context.Context,
query string, query string,
args ...interface{}, args ...any,
) (sql.Result, error) { ) (sql.Result, error) {
LogSQL("Execute", query, args...) LogSQL("Execute", query, args...)
return db.conn.ExecContext(ctx, query, args...) return db.conn.ExecContext(ctx, query, args...)
} }
@@ -240,9 +260,10 @@ func (db *DB) ExecWithLog(
func (db *DB) QueryRowWithLog( func (db *DB) QueryRowWithLog(
ctx context.Context, ctx context.Context,
query string, query string,
args ...interface{}, args ...any,
) *sql.Row { ) *sql.Row {
LogSQL("QueryRow", query, args...) LogSQL("QueryRow", query, args...)
return db.conn.QueryRowContext(ctx, query, args...) return db.conn.QueryRowContext(ctx, query, args...)
} }
@@ -375,6 +396,7 @@ func repeatPlaceholder(n int) string {
if n <= 0 { if n <= 0 {
return "" return ""
} }
return strings.Repeat(", ?", n) return strings.Repeat(", ?", n)
} }
@@ -385,7 +407,7 @@ func repeatPlaceholder(n int) string {
// The operation parameter describes the type of SQL operation (e.g., "Execute", "Query"). // The operation parameter describes the type of SQL operation (e.g., "Execute", "Query").
// The query parameter is the SQL statement being executed. // The query parameter is the SQL statement being executed.
// The args parameter contains the query arguments that will be interpolated. // The args parameter contains the query arguments that will be interpolated.
func LogSQL(operation, query string, args ...interface{}) { func LogSQL(operation, query string, args ...any) {
if strings.Contains(os.Getenv("GODEBUG"), "vaultik") { if strings.Contains(os.Getenv("GODEBUG"), "vaultik") {
log.Debug( log.Debug(
"SQL "+operation, "SQL "+operation,

View File

@@ -17,7 +17,8 @@ func TestDatabase(t *testing.T) {
t.Fatalf("failed to create database: %v", err) t.Fatalf("failed to create database: %v", err)
} }
defer func() { defer func() {
if err := db.Close(); err != nil { err := db.Close()
if err != nil {
t.Errorf("failed to close database: %v", err) t.Errorf("failed to close database: %v", err)
} }
}() }()
@@ -37,6 +38,7 @@ func TestDatabase(t *testing.T) {
for _, table := range tables { for _, table := range tables {
var name string var name string
err := db.conn.QueryRow("SELECT name FROM sqlite_master WHERE type='table' AND name=?", table).Scan(&name) err := db.conn.QueryRow("SELECT name FROM sqlite_master WHERE type='table' AND name=?", table).Scan(&name)
if err != nil { if err != nil {
t.Errorf("table %s does not exist: %v", table, err) t.Errorf("table %s does not exist: %v", table, err)
@@ -63,7 +65,8 @@ func TestDatabaseConcurrentAccess(t *testing.T) {
t.Fatalf("failed to create database: %v", err) t.Fatalf("failed to create database: %v", err)
} }
defer func() { defer func() {
if err := db.Close(); err != nil { err := db.Close()
if err != nil {
t.Errorf("failed to close database: %v", err) t.Errorf("failed to close database: %v", err)
} }
}() }()
@@ -73,9 +76,10 @@ func TestDatabaseConcurrentAccess(t *testing.T) {
index int index int
err error err error
} }
results := make(chan result, 10) results := make(chan result, 10)
for i := 0; i < 10; i++ { for i := range 10 {
go func(i int) { go func(i int) {
_, err := db.ExecWithLog(ctx, "INSERT INTO chunks (chunk_hash, size) VALUES (?, ?)", _, err := db.ExecWithLog(ctx, "INSERT INTO chunks (chunk_hash, size) VALUES (?, ?)",
fmt.Sprintf("hash%d", i), i*1024) fmt.Sprintf("hash%d", i), i*1024)
@@ -84,7 +88,7 @@ func TestDatabaseConcurrentAccess(t *testing.T) {
} }
// Wait for all goroutines and check results // Wait for all goroutines and check results
for i := 0; i < 10; i++ { for range 10 {
r := <-results r := <-results
if r.err != nil { if r.err != nil {
t.Fatalf("concurrent insert %d failed: %v", r.index, r.err) t.Fatalf("concurrent insert %d failed: %v", r.index, r.err)
@@ -93,10 +97,12 @@ func TestDatabaseConcurrentAccess(t *testing.T) {
// Verify all inserts succeeded // Verify all inserts succeeded
var count int var count int
err = db.conn.QueryRowContext(ctx, "SELECT COUNT(*) FROM chunks").Scan(&count) err = db.conn.QueryRowContext(ctx, "SELECT COUNT(*) FROM chunks").Scan(&count)
if err != nil { if err != nil {
t.Fatalf("failed to count chunks: %v", err) t.Fatalf("failed to count chunks: %v", err)
} }
if count != 10 { if count != 10 {
t.Errorf("expected 10 chunks, got %d", count) t.Errorf("expected 10 chunks, got %d", count)
} }
@@ -127,12 +133,16 @@ func TestParseMigrationVersion(t *testing.T) {
if err == nil { if err == nil {
t.Errorf("ParseMigrationVersion(%q) = %d, nil; want error", tc.filename, got) t.Errorf("ParseMigrationVersion(%q) = %d, nil; want error", tc.filename, got)
} }
return return
} }
if err != nil { if err != nil {
t.Errorf("ParseMigrationVersion(%q) unexpected error: %v", tc.filename, err) t.Errorf("ParseMigrationVersion(%q) unexpected error: %v", tc.filename, err)
return return
} }
if got != tc.wantVer { if got != tc.wantVer {
t.Errorf("ParseMigrationVersion(%q) = %d; want %d", tc.filename, got, tc.wantVer) t.Errorf("ParseMigrationVersion(%q) = %d; want %d", tc.filename, got, tc.wantVer)
} }
@@ -148,7 +158,8 @@ func TestApplyMigrations_Idempotent(t *testing.T) {
t.Fatalf("failed to open database: %v", err) t.Fatalf("failed to open database: %v", err)
} }
defer func() { defer func() {
if err := conn.Close(); err != nil { err := conn.Close()
if err != nil {
t.Errorf("failed to close database: %v", err) t.Errorf("failed to close database: %v", err)
} }
}() }()
@@ -191,7 +202,8 @@ func TestBootstrapMigrationsTable_FreshDatabase(t *testing.T) {
t.Fatalf("failed to open database: %v", err) t.Fatalf("failed to open database: %v", err)
} }
defer func() { defer func() {
if err := conn.Close(); err != nil { err := conn.Close()
if err != nil {
t.Errorf("failed to close database: %v", err) t.Errorf("failed to close database: %v", err)
} }
}() }()
@@ -206,6 +218,7 @@ func TestBootstrapMigrationsTable_FreshDatabase(t *testing.T) {
).Scan(&tableBefore); err != nil { ).Scan(&tableBefore); err != nil {
t.Fatalf("failed to check for table before bootstrap: %v", err) t.Fatalf("failed to check for table before bootstrap: %v", err)
} }
if tableBefore != 0 { if tableBefore != 0 {
t.Fatal("schema_migrations table should not exist before bootstrap") t.Fatal("schema_migrations table should not exist before bootstrap")
} }
@@ -222,6 +235,7 @@ func TestBootstrapMigrationsTable_FreshDatabase(t *testing.T) {
).Scan(&tableAfter); err != nil { ).Scan(&tableAfter); err != nil {
t.Fatalf("failed to check for table after bootstrap: %v", err) t.Fatalf("failed to check for table after bootstrap: %v", err)
} }
if tableAfter != 1 { if tableAfter != 1 {
t.Fatalf("schema_migrations table should exist after bootstrap, got count=%d", tableAfter) t.Fatalf("schema_migrations table should exist after bootstrap, got count=%d", tableAfter)
} }
@@ -233,6 +247,7 @@ func TestBootstrapMigrationsTable_FreshDatabase(t *testing.T) {
).Scan(&version); err != nil { ).Scan(&version); err != nil {
t.Fatalf("version 0 row not found in schema_migrations: %v", err) t.Fatalf("version 0 row not found in schema_migrations: %v", err)
} }
if version != 0 { if version != 0 {
t.Errorf("expected version 0, got %d", version) t.Errorf("expected version 0, got %d", version)
} }

View File

@@ -6,15 +6,16 @@ import (
"os" "os"
) )
// Fatal prints an error message to stderr and exits with status 1 // Fatalf prints an error message to stderr and exits with status 1
func Fatal(format string, args ...interface{}) { func Fatalf(format string, args ...any) {
fmt.Fprintf(os.Stderr, "FATAL: "+format+"\n", args...) fmt.Fprintf(os.Stderr, "FATAL: "+format+"\n", args...)
os.Exit(1) os.Exit(1)
} }
// CloseRows closes rows and exits on error // CloseRows closes rows and exits on error
func CloseRows(rows *sql.Rows) { func CloseRows(rows *sql.Rows) {
if err := rows.Close(); err != nil { err := rows.Close()
Fatal("failed to close rows: %v", err) if err != nil {
Fatalf("failed to close rows: %v", err)
} }
} }

View File

@@ -4,6 +4,7 @@ import (
"context" "context"
"database/sql" "database/sql"
"fmt" "fmt"
"strings"
"sneak.berlin/go/vaultik/internal/types" "sneak.berlin/go/vaultik/internal/types"
) )
@@ -84,6 +85,7 @@ func (r *FileChunkRepository) GetByPathTx(ctx context.Context, tx *sql.Tx, path
` `
LogSQL("GetByPathTx", query, path) LogSQL("GetByPathTx", query, path)
rows, err := tx.QueryContext(ctx, query, path) rows, err := tx.QueryContext(ctx, query, path)
if err != nil { if err != nil {
return nil, fmt.Errorf("querying file chunks: %w", err) return nil, fmt.Errorf("querying file chunks: %w", err)
@@ -92,23 +94,30 @@ func (r *FileChunkRepository) GetByPathTx(ctx context.Context, tx *sql.Tx, path
fileChunks, err := r.scanFileChunks(rows) fileChunks, err := r.scanFileChunks(rows)
LogSQL("GetByPathTx", "Complete", path, "count", len(fileChunks)) LogSQL("GetByPathTx", "Complete", path, "count", len(fileChunks))
return fileChunks, err return fileChunks, err
} }
// scanFileChunks is a helper that scans file chunk rows // scanFileChunks is a helper that scans file chunk rows
func (r *FileChunkRepository) scanFileChunks(rows *sql.Rows) ([]*FileChunk, error) { func (r *FileChunkRepository) scanFileChunks(rows *sql.Rows) ([]*FileChunk, error) {
var fileChunks []*FileChunk var fileChunks []*FileChunk
for rows.Next() { for rows.Next() {
var fc FileChunk var (
var fileIDStr, chunkHashStr string fc FileChunk
fileIDStr, chunkHashStr string
)
err := rows.Scan(&fileIDStr, &fc.Idx, &chunkHashStr) err := rows.Scan(&fileIDStr, &fc.Idx, &chunkHashStr)
if err != nil { if err != nil {
return nil, fmt.Errorf("scanning file chunk: %w", err) return nil, fmt.Errorf("scanning file chunk: %w", err)
} }
fc.FileID, err = types.ParseFileID(fileIDStr) fc.FileID, err = types.ParseFileID(fileIDStr)
if err != nil { if err != nil {
return nil, fmt.Errorf("parsing file ID: %w", err) return nil, fmt.Errorf("parsing file ID: %w", err)
} }
fc.ChunkHash = types.ChunkHash(chunkHashStr) fc.ChunkHash = types.ChunkHash(chunkHashStr)
fileChunks = append(fileChunks, &fc) fileChunks = append(fileChunks, &fc)
} }
@@ -161,14 +170,13 @@ func (r *FileChunkRepository) DeleteByFileIDs(ctx context.Context, tx *sql.Tx, f
const batchSize = 500 const batchSize = 500
for i := 0; i < len(fileIDs); i += batchSize { for i := 0; i < len(fileIDs); i += batchSize {
end := i + batchSize end := min(i+batchSize, len(fileIDs))
if end > len(fileIDs) {
end = len(fileIDs)
}
batch := fileIDs[i:end] batch := fileIDs[i:end]
query := "DELETE FROM file_chunks WHERE file_id IN (?" + repeatPlaceholder(len(batch)-1) + ")" query := "DELETE FROM file_chunks WHERE file_id IN (?" + repeatPlaceholder(len(batch)-1) + ")"
args := make([]interface{}, len(batch))
args := make([]any, len(batch))
for j, id := range batch { for j, id := range batch {
args[j] = id.String() args[j] = id.String()
} }
@@ -179,6 +187,7 @@ func (r *FileChunkRepository) DeleteByFileIDs(ctx context.Context, tx *sql.Tx, f
} else { } else {
_, err = r.db.ExecWithLog(ctx, query, args...) _, err = r.db.ExecWithLog(ctx, query, args...)
} }
if err != nil { if err != nil {
return fmt.Errorf("batch deleting file_chunks: %w", err) return fmt.Errorf("batch deleting file_chunks: %w", err)
} }
@@ -199,22 +208,29 @@ func (r *FileChunkRepository) CreateBatch(ctx context.Context, tx *sql.Tx, fcs [
const batchSize = 300 const batchSize = 300
for i := 0; i < len(fcs); i += batchSize { for i := 0; i < len(fcs); i += batchSize {
end := i + batchSize end := min(i+batchSize, len(fcs))
if end > len(fcs) {
end = len(fcs)
}
batch := fcs[i:end] batch := fcs[i:end]
// Build the query with multiple value sets // Build the query with multiple value sets
query := "INSERT INTO file_chunks (file_id, idx, chunk_hash) VALUES " query := "INSERT INTO file_chunks (file_id, idx, chunk_hash) VALUES "
args := make([]interface{}, 0, len(batch)*3)
args := make([]any, 0, len(batch)*3)
var querySb211 strings.Builder
for j, fc := range batch { for j, fc := range batch {
if j > 0 { if j > 0 {
query += ", " querySb211.WriteString(", ")
} }
query += "(?, ?, ?)"
querySb211.WriteString("(?, ?, ?)")
args = append(args, fc.FileID.String(), fc.Idx, fc.ChunkHash.String()) args = append(args, fc.FileID.String(), fc.Idx, fc.ChunkHash.String())
} }
query += querySb211.String()
query += " ON CONFLICT(file_id, idx) DO NOTHING" query += " ON CONFLICT(file_id, idx) DO NOTHING"
var err error var err error
@@ -223,6 +239,7 @@ func (r *FileChunkRepository) CreateBatch(ctx context.Context, tx *sql.Tx, fcs [
} else { } else {
_, err = r.db.ExecWithLog(ctx, query, args...) _, err = r.db.ExecWithLog(ctx, query, args...)
} }
if err != nil { if err != nil {
return fmt.Errorf("batch inserting file_chunks: %w", err) return fmt.Errorf("batch inserting file_chunks: %w", err)
} }
@@ -236,6 +253,7 @@ func (r *FileChunkRepository) GetByFile(ctx context.Context, path string) ([]*Fi
LogSQL("GetByFile", "Starting", path) LogSQL("GetByFile", "Starting", path)
result, err := r.GetByPath(ctx, path) result, err := r.GetByPath(ctx, path)
LogSQL("GetByFile", "Complete", path, "count", len(result)) LogSQL("GetByFile", "Complete", path, "count", len(result))
return result, err return result, err
} }
@@ -244,5 +262,6 @@ func (r *FileChunkRepository) GetByFileTx(ctx context.Context, tx *sql.Tx, path
LogSQL("GetByFileTx", "Starting", path) LogSQL("GetByFileTx", "Starting", path)
result, err := r.GetByPathTx(ctx, tx, path) result, err := r.GetByPathTx(ctx, tx, path)
LogSQL("GetByFileTx", "Complete", path, "count", len(result)) LogSQL("GetByFileTx", "Complete", path, "count", len(result))
return result, err return result, err
} }

View File

@@ -28,6 +28,7 @@ func TestFileChunkRepository(t *testing.T) {
GID: 1000, GID: 1000,
LinkTarget: "", LinkTarget: "",
} }
err := fileRepo.Create(ctx, nil, file) err := fileRepo.Create(ctx, nil, file)
if err != nil { if err != nil {
t.Fatalf("failed to create file: %v", err) t.Fatalf("failed to create file: %v", err)
@@ -36,11 +37,13 @@ func TestFileChunkRepository(t *testing.T) {
// Create chunks first // Create chunks first
chunks := []types.ChunkHash{"chunk1", "chunk2", "chunk3"} chunks := []types.ChunkHash{"chunk1", "chunk2", "chunk3"}
chunkRepo := NewChunkRepository(db) chunkRepo := NewChunkRepository(db)
for _, chunkHash := range chunks { for _, chunkHash := range chunks {
chunk := &Chunk{ chunk := &Chunk{
ChunkHash: chunkHash, ChunkHash: chunkHash,
Size: 1024, Size: 1024,
} }
err = chunkRepo.Create(ctx, nil, chunk) err = chunkRepo.Create(ctx, nil, chunk)
if err != nil { if err != nil {
t.Fatalf("failed to create chunk %s: %v", chunkHash, err) t.Fatalf("failed to create chunk %s: %v", chunkHash, err)
@@ -65,6 +68,7 @@ func TestFileChunkRepository(t *testing.T) {
Idx: 1, Idx: 1,
ChunkHash: types.ChunkHash("chunk2"), ChunkHash: types.ChunkHash("chunk2"),
} }
err = repo.Create(ctx, nil, fc2) err = repo.Create(ctx, nil, fc2)
if err != nil { if err != nil {
t.Fatalf("failed to create second file chunk: %v", err) t.Fatalf("failed to create second file chunk: %v", err)
@@ -75,6 +79,7 @@ func TestFileChunkRepository(t *testing.T) {
Idx: 2, Idx: 2,
ChunkHash: types.ChunkHash("chunk3"), ChunkHash: types.ChunkHash("chunk3"),
} }
err = repo.Create(ctx, nil, fc3) err = repo.Create(ctx, nil, fc3)
if err != nil { if err != nil {
t.Fatalf("failed to create third file chunk: %v", err) t.Fatalf("failed to create third file chunk: %v", err)
@@ -85,6 +90,7 @@ func TestFileChunkRepository(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("failed to get file chunks: %v", err) t.Fatalf("failed to get file chunks: %v", err)
} }
if len(fileChunks) != 3 { if len(fileChunks) != 3 {
t.Errorf("expected 3 chunks, got %d", len(fileChunks)) t.Errorf("expected 3 chunks, got %d", len(fileChunks))
} }
@@ -112,6 +118,7 @@ func TestFileChunkRepository(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("failed to get deleted file chunks: %v", err) t.Fatalf("failed to get deleted file chunks: %v", err)
} }
if len(fileChunks) != 0 { if len(fileChunks) != 0 {
t.Errorf("expected 0 chunks after delete, got %d", len(fileChunks)) t.Errorf("expected 0 chunks after delete, got %d", len(fileChunks))
} }
@@ -140,22 +147,26 @@ func TestFileChunkRepositoryMultipleFiles(t *testing.T) {
GID: 1000, GID: 1000,
LinkTarget: "", LinkTarget: "",
} }
err := fileRepo.Create(ctx, nil, file) err := fileRepo.Create(ctx, nil, file)
if err != nil { if err != nil {
t.Fatalf("failed to create file %s: %v", path, err) t.Fatalf("failed to create file %s: %v", path, err)
} }
files[i] = file files[i] = file
} }
// Create all chunks first // Create all chunks first
chunkRepo := NewChunkRepository(db) chunkRepo := NewChunkRepository(db)
for i := range files { for i := range files {
for j := 0; j < 2; j++ { for j := range 2 {
chunkHash := types.ChunkHash(fmt.Sprintf("file%d_chunk%d", i, j)) chunkHash := types.ChunkHash(fmt.Sprintf("file%d_chunk%d", i, j))
chunk := &Chunk{ chunk := &Chunk{
ChunkHash: chunkHash, ChunkHash: chunkHash,
Size: 1024, Size: 1024,
} }
err := chunkRepo.Create(ctx, nil, chunk) err := chunkRepo.Create(ctx, nil, chunk)
if err != nil { if err != nil {
t.Fatalf("failed to create chunk %s: %v", chunkHash, err) t.Fatalf("failed to create chunk %s: %v", chunkHash, err)
@@ -165,12 +176,13 @@ func TestFileChunkRepositoryMultipleFiles(t *testing.T) {
// Create chunks for multiple files // Create chunks for multiple files
for i, file := range files { for i, file := range files {
for j := 0; j < 2; j++ { for j := range 2 {
fc := &FileChunk{ fc := &FileChunk{
FileID: file.ID, FileID: file.ID,
Idx: j, Idx: j,
ChunkHash: types.ChunkHash(fmt.Sprintf("file%d_chunk%d", i, j)), ChunkHash: types.ChunkHash(fmt.Sprintf("file%d_chunk%d", i, j)),
} }
err := repo.Create(ctx, nil, fc) err := repo.Create(ctx, nil, fc)
if err != nil { if err != nil {
t.Fatalf("failed to create file chunk: %v", err) t.Fatalf("failed to create file chunk: %v", err)
@@ -184,6 +196,7 @@ func TestFileChunkRepositoryMultipleFiles(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("failed to get chunks for file %d: %v", i, err) t.Fatalf("failed to get chunks for file %d: %v", i, err)
} }
if len(chunks) != 2 { if len(chunks) != 2 {
t.Errorf("expected 2 chunks for file %d, got %d", i, len(chunks)) t.Errorf("expected 2 chunks for file %d, got %d", i, len(chunks))
} }

View File

@@ -3,7 +3,9 @@ package database
import ( import (
"context" "context"
"database/sql" "database/sql"
"errors"
"fmt" "fmt"
"strings"
"time" "time"
"sneak.berlin/go/vaultik/internal/log" "sneak.berlin/go/vaultik/internal/log"
@@ -38,8 +40,11 @@ func (r *FileRepository) Create(ctx context.Context, tx *sql.Tx, file *File) err
RETURNING id RETURNING id
` `
var idStr string var (
var err error idStr string
err error
)
if tx != nil { if tx != nil {
LogSQL("Execute", query, file.ID.String(), file.Path.String(), file.SourcePath.String(), file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID, file.LinkTarget.String()) LogSQL("Execute", query, file.ID.String(), file.Path.String(), file.SourcePath.String(), file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID, file.LinkTarget.String())
err = tx.QueryRowContext(ctx, query, file.ID.String(), file.Path.String(), file.SourcePath.String(), file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID, file.LinkTarget.String()).Scan(&idStr) err = tx.QueryRowContext(ctx, query, file.ID.String(), file.Path.String(), file.SourcePath.String(), file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID, file.LinkTarget.String()).Scan(&idStr)
@@ -68,9 +73,10 @@ func (r *FileRepository) GetByPath(ctx context.Context, path string) (*File, err
` `
file, err := r.scanFile(r.db.conn.QueryRowContext(ctx, query, path)) file, err := r.scanFile(r.db.conn.QueryRowContext(ctx, query, path))
if err == sql.ErrNoRows { if errors.Is(err, sql.ErrNoRows) {
return nil, nil return nil, nil
} }
if err != nil { if err != nil {
return nil, fmt.Errorf("querying file: %w", err) return nil, fmt.Errorf("querying file: %w", err)
} }
@@ -87,9 +93,10 @@ func (r *FileRepository) GetByID(ctx context.Context, id types.FileID) (*File, e
` `
file, err := r.scanFile(r.db.conn.QueryRowContext(ctx, query, id.String())) file, err := r.scanFile(r.db.conn.QueryRowContext(ctx, query, id.String()))
if err == sql.ErrNoRows { if errors.Is(err, sql.ErrNoRows) {
return nil, nil return nil, nil
} }
if err != nil { if err != nil {
return nil, fmt.Errorf("querying file: %w", err) return nil, fmt.Errorf("querying file: %w", err)
} }
@@ -108,9 +115,10 @@ func (r *FileRepository) GetByPathTx(ctx context.Context, tx *sql.Tx, path strin
file, err := r.scanFile(tx.QueryRowContext(ctx, query, path)) file, err := r.scanFile(tx.QueryRowContext(ctx, query, path))
LogSQL("GetByPathTx Scan complete", query, path) LogSQL("GetByPathTx Scan complete", query, path)
if err == sql.ErrNoRows { if errors.Is(err, sql.ErrNoRows) {
return nil, nil return nil, nil
} }
if err != nil { if err != nil {
return nil, fmt.Errorf("querying file: %w", err) return nil, fmt.Errorf("querying file: %w", err)
} }
@@ -120,10 +128,12 @@ func (r *FileRepository) GetByPathTx(ctx context.Context, tx *sql.Tx, path strin
// scanFile is a helper that scans a single file row // scanFile is a helper that scans a single file row
func (r *FileRepository) scanFile(row *sql.Row) (*File, error) { func (r *FileRepository) scanFile(row *sql.Row) (*File, error) {
var file File var (
var idStr, pathStr, sourcePathStr string file File
var mtimeUnix int64 idStr, pathStr, sourcePathStr string
var linkTarget sql.NullString mtimeUnix int64
linkTarget sql.NullString
)
err := row.Scan( err := row.Scan(
&idStr, &idStr,
@@ -144,8 +154,10 @@ func (r *FileRepository) scanFile(row *sql.Row) (*File, error) {
if err != nil { if err != nil {
return nil, fmt.Errorf("parsing file ID: %w", err) return nil, fmt.Errorf("parsing file ID: %w", err)
} }
file.Path = types.FilePath(pathStr) file.Path = types.FilePath(pathStr)
file.SourcePath = types.SourcePath(sourcePathStr) file.SourcePath = types.SourcePath(sourcePathStr)
file.MTime = time.Unix(mtimeUnix, 0).UTC() file.MTime = time.Unix(mtimeUnix, 0).UTC()
if linkTarget.Valid { if linkTarget.Valid {
file.LinkTarget = types.FilePath(linkTarget.String) file.LinkTarget = types.FilePath(linkTarget.String)
@@ -156,10 +168,12 @@ func (r *FileRepository) scanFile(row *sql.Row) (*File, error) {
// scanFileRows is a helper that scans a file row from rows iterator // scanFileRows is a helper that scans a file row from rows iterator
func (r *FileRepository) scanFileRows(rows *sql.Rows) (*File, error) { func (r *FileRepository) scanFileRows(rows *sql.Rows) (*File, error) {
var file File var (
var idStr, pathStr, sourcePathStr string file File
var mtimeUnix int64 idStr, pathStr, sourcePathStr string
var linkTarget sql.NullString mtimeUnix int64
linkTarget sql.NullString
)
err := rows.Scan( err := rows.Scan(
&idStr, &idStr,
@@ -180,8 +194,10 @@ func (r *FileRepository) scanFileRows(rows *sql.Rows) (*File, error) {
if err != nil { if err != nil {
return nil, fmt.Errorf("parsing file ID: %w", err) return nil, fmt.Errorf("parsing file ID: %w", err)
} }
file.Path = types.FilePath(pathStr) file.Path = types.FilePath(pathStr)
file.SourcePath = types.SourcePath(sourcePathStr) file.SourcePath = types.SourcePath(sourcePathStr)
file.MTime = time.Unix(mtimeUnix, 0).UTC() file.MTime = time.Unix(mtimeUnix, 0).UTC()
if linkTarget.Valid { if linkTarget.Valid {
file.LinkTarget = types.FilePath(linkTarget.String) file.LinkTarget = types.FilePath(linkTarget.String)
@@ -205,11 +221,13 @@ func (r *FileRepository) ListModifiedSince(ctx context.Context, since time.Time)
defer CloseRows(rows) defer CloseRows(rows)
var files []*File var files []*File
for rows.Next() { for rows.Next() {
file, err := r.scanFileRows(rows) file, err := r.scanFileRows(rows)
if err != nil { if err != nil {
return nil, fmt.Errorf("scanning file: %w", err) return nil, fmt.Errorf("scanning file: %w", err)
} }
files = append(files, file) files = append(files, file)
} }
@@ -266,11 +284,13 @@ func (r *FileRepository) ListByPrefix(ctx context.Context, prefix string) ([]*Fi
defer CloseRows(rows) defer CloseRows(rows)
var files []*File var files []*File
for rows.Next() { for rows.Next() {
file, err := r.scanFileRows(rows) file, err := r.scanFileRows(rows)
if err != nil { if err != nil {
return nil, fmt.Errorf("scanning file: %w", err) return nil, fmt.Errorf("scanning file: %w", err)
} }
files = append(files, file) files = append(files, file)
} }
@@ -292,11 +312,13 @@ func (r *FileRepository) ListAll(ctx context.Context) ([]*File, error) {
defer CloseRows(rows) defer CloseRows(rows)
var files []*File var files []*File
for rows.Next() { for rows.Next() {
file, err := r.scanFileRows(rows) file, err := r.scanFileRows(rows)
if err != nil { if err != nil {
return nil, fmt.Errorf("scanning file: %w", err) return nil, fmt.Errorf("scanning file: %w", err)
} }
files = append(files, file) files = append(files, file)
} }
@@ -314,21 +336,28 @@ func (r *FileRepository) CreateBatch(ctx context.Context, tx *sql.Tx, files []*F
const batchSize = 100 const batchSize = 100
for i := 0; i < len(files); i += batchSize { for i := 0; i < len(files); i += batchSize {
end := i + batchSize end := min(i+batchSize, len(files))
if end > len(files) {
end = len(files)
}
batch := files[i:end] batch := files[i:end]
query := `INSERT INTO files (id, path, source_path, mtime, size, mode, uid, gid, link_target) VALUES ` query := `INSERT INTO files (id, path, source_path, mtime, size, mode, uid, gid, link_target) VALUES `
args := make([]interface{}, 0, len(batch)*9)
args := make([]any, 0, len(batch)*9)
var querySb325 strings.Builder
for j, f := range batch { for j, f := range batch {
if j > 0 { if j > 0 {
query += ", " querySb325.WriteString(", ")
} }
query += "(?, ?, ?, ?, ?, ?, ?, ?, ?)"
querySb325.WriteString("(?, ?, ?, ?, ?, ?, ?, ?, ?)")
args = append(args, f.ID.String(), f.Path.String(), f.SourcePath.String(), f.MTime.Unix(), f.Size, f.Mode, f.UID, f.GID, f.LinkTarget.String()) args = append(args, f.ID.String(), f.Path.String(), f.SourcePath.String(), f.MTime.Unix(), f.Size, f.Mode, f.UID, f.GID, f.LinkTarget.String())
} }
query += querySb325.String()
query += ` ON CONFLICT(path) DO UPDATE SET query += ` ON CONFLICT(path) DO UPDATE SET
source_path = excluded.source_path, source_path = excluded.source_path,
mtime = excluded.mtime, mtime = excluded.mtime,
@@ -344,6 +373,7 @@ func (r *FileRepository) CreateBatch(ctx context.Context, tx *sql.Tx, files []*F
} else { } else {
_, err = r.db.ExecWithLog(ctx, query, args...) _, err = r.db.ExecWithLog(ctx, query, args...)
} }
if err != nil { if err != nil {
return fmt.Errorf("batch inserting files: %w", err) return fmt.Errorf("batch inserting files: %w", err)
} }

View File

@@ -3,7 +3,7 @@ package database
import ( import (
"context" "context"
"database/sql" "database/sql"
"fmt" "errors"
"os" "os"
"path/filepath" "path/filepath"
"testing" "testing"
@@ -20,7 +20,8 @@ func setupTestDB(t *testing.T) (*DB, func()) {
} }
cleanup := func() { cleanup := func() {
if err := db.Close(); err != nil { err := db.Close()
if err != nil {
t.Errorf("failed to close database: %v", err) t.Errorf("failed to close database: %v", err)
} }
} }
@@ -56,18 +57,23 @@ func TestFileRepository(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("failed to get file: %v", err) t.Fatalf("failed to get file: %v", err)
} }
if retrieved == nil { if retrieved == nil {
t.Fatal("expected file, got nil") t.Fatal("expected file, got nil")
} }
if retrieved.Path != file.Path { if retrieved.Path != file.Path {
t.Errorf("path mismatch: got %s, want %s", retrieved.Path, file.Path) t.Errorf("path mismatch: got %s, want %s", retrieved.Path, file.Path)
} }
if !retrieved.MTime.Equal(file.MTime) { if !retrieved.MTime.Equal(file.MTime) {
t.Errorf("mtime mismatch: got %v, want %v", retrieved.MTime, file.MTime) t.Errorf("mtime mismatch: got %v, want %v", retrieved.MTime, file.MTime)
} }
if retrieved.Size != file.Size { if retrieved.Size != file.Size {
t.Errorf("size mismatch: got %d, want %d", retrieved.Size, file.Size) t.Errorf("size mismatch: got %d, want %d", retrieved.Size, file.Size)
} }
if retrieved.Mode != file.Mode { if retrieved.Mode != file.Mode {
t.Errorf("mode mismatch: got %o, want %o", retrieved.Mode, file.Mode) t.Errorf("mode mismatch: got %o, want %o", retrieved.Mode, file.Mode)
} }
@@ -75,6 +81,7 @@ func TestFileRepository(t *testing.T) {
// Test Update (upsert) // Test Update (upsert)
file.Size = 2048 file.Size = 2048
file.MTime = time.Now().Truncate(time.Second) file.MTime = time.Now().Truncate(time.Second)
err = repo.Create(ctx, nil, file) err = repo.Create(ctx, nil, file)
if err != nil { if err != nil {
t.Fatalf("failed to update file: %v", err) t.Fatalf("failed to update file: %v", err)
@@ -84,6 +91,7 @@ func TestFileRepository(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("failed to get updated file: %v", err) t.Fatalf("failed to get updated file: %v", err)
} }
if retrieved.Size != 2048 { if retrieved.Size != 2048 {
t.Errorf("size not updated: got %d, want %d", retrieved.Size, 2048) t.Errorf("size not updated: got %d, want %d", retrieved.Size, 2048)
} }
@@ -93,6 +101,7 @@ func TestFileRepository(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("failed to list files: %v", err) t.Fatalf("failed to list files: %v", err)
} }
if len(files) != 1 { if len(files) != 1 {
t.Errorf("expected 1 file, got %d", len(files)) t.Errorf("expected 1 file, got %d", len(files))
} }
@@ -107,6 +116,7 @@ func TestFileRepository(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("error getting deleted file: %v", err) t.Fatalf("error getting deleted file: %v", err)
} }
if retrieved != nil { if retrieved != nil {
t.Error("expected nil for deleted file") t.Error("expected nil for deleted file")
} }
@@ -139,9 +149,11 @@ func TestFileRepositorySymlink(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("failed to get symlink: %v", err) t.Fatalf("failed to get symlink: %v", err)
} }
if !retrieved.IsSymlink() { if !retrieved.IsSymlink() {
t.Error("expected IsSymlink() to be true") t.Error("expected IsSymlink() to be true")
} }
if retrieved.LinkTarget != symlink.LinkTarget { if retrieved.LinkTarget != symlink.LinkTarget {
t.Errorf("link target mismatch: got %s, want %s", retrieved.LinkTarget, symlink.LinkTarget) t.Errorf("link target mismatch: got %s, want %s", retrieved.LinkTarget, symlink.LinkTarget)
} }
@@ -165,12 +177,13 @@ func TestFileRepositoryTransaction(t *testing.T) {
GID: 1000, GID: 1000,
} }
if err := repos.Files.Create(ctx, tx, file); err != nil { err := repos.Files.Create(ctx, tx, file)
if err != nil {
return err return err
} }
// Return error to trigger rollback // Return error to trigger rollback
return fmt.Errorf("test rollback") return errors.New("test rollback")
}) })
if err == nil || err.Error() != "test rollback" { if err == nil || err.Error() != "test rollback" {
@@ -182,6 +195,7 @@ func TestFileRepositoryTransaction(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("error checking for file: %v", err) t.Fatalf("error checking for file: %v", err)
} }
if retrieved != nil { if retrieved != nil {
t.Error("file should not exist after rollback") t.Error("file should not exist after rollback")
} }

View File

@@ -27,15 +27,18 @@ func NewLocalMetaRepository(db *DB) *LocalMetaRepository {
// "unset" (bind on first use) from "set to something" (compare). // "unset" (bind on first use) from "set to something" (compare).
func (r *LocalMetaRepository) Get(ctx context.Context, key string) (string, error) { func (r *LocalMetaRepository) Get(ctx context.Context, key string) (string, error) {
var value string var value string
err := r.db.conn.QueryRowContext(ctx, err := r.db.conn.QueryRowContext(ctx,
"SELECT value FROM local_meta WHERE key = ?", key, "SELECT value FROM local_meta WHERE key = ?", key,
).Scan(&value) ).Scan(&value)
if errors.Is(err, sql.ErrNoRows) { if errors.Is(err, sql.ErrNoRows) {
return "", nil return "", nil
} }
if err != nil { if err != nil {
return "", fmt.Errorf("reading local_meta %q: %w", key, err) return "", fmt.Errorf("reading local_meta %q: %w", key, err)
} }
return value, nil return value, nil
} }
@@ -49,5 +52,6 @@ func (r *LocalMetaRepository) Set(ctx context.Context, key, value string) error
if err != nil { if err != nil {
return fmt.Errorf("writing local_meta %q: %w", key, err) return fmt.Errorf("writing local_meta %q: %w", key, err)
} }
return nil return nil
} }

View File

@@ -11,18 +11,20 @@ import (
func TestLocalMetaEmptyOnFresh(t *testing.T) { func TestLocalMetaEmptyOnFresh(t *testing.T) {
db, err := database.NewTestDB() db, err := database.NewTestDB()
require.NoError(t, err) require.NoError(t, err)
defer func() { _ = db.Close() }() defer func() { _ = db.Close() }()
repos := database.NewRepositories(db) repos := database.NewRepositories(db)
got, err := repos.LocalMeta.Get(context.Background(), database.LocalMetaKeyStorageURL) got, err := repos.LocalMeta.Get(context.Background(), database.LocalMetaKeyStorageURL)
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, "", got, "fresh DB must return empty for unset keys, not error") require.Empty(t, got, "fresh DB must return empty for unset keys, not error")
} }
func TestLocalMetaSetGetRoundTrip(t *testing.T) { func TestLocalMetaSetGetRoundTrip(t *testing.T) {
db, err := database.NewTestDB() db, err := database.NewTestDB()
require.NoError(t, err) require.NoError(t, err)
defer func() { _ = db.Close() }() defer func() { _ = db.Close() }()
repos := database.NewRepositories(db) repos := database.NewRepositories(db)
@@ -38,6 +40,7 @@ func TestLocalMetaSetGetRoundTrip(t *testing.T) {
func TestLocalMetaSetOverwrites(t *testing.T) { func TestLocalMetaSetOverwrites(t *testing.T) {
db, err := database.NewTestDB() db, err := database.NewTestDB()
require.NoError(t, err) require.NoError(t, err)
defer func() { _ = db.Close() }() defer func() { _ = db.Close() }()
repos := database.NewRepositories(db) repos := database.NewRepositories(db)

View File

@@ -34,11 +34,16 @@ func provideDatabase(lc fx.Lifecycle, cfg *config.Config) (*DB, error) {
lc.Append(fx.Hook{ lc.Append(fx.Hook{
OnStop: func(ctx context.Context) error { OnStop: func(ctx context.Context) error {
log.Debug("Database module OnStop hook called") log.Debug("Database module OnStop hook called")
if err := db.Close(); err != nil {
err := db.Close()
if err != nil {
log.Error("Failed to close database in OnStop hook", "error", err) log.Error("Failed to close database in OnStop hook", "error", err)
return err return err
} }
log.Debug("Database closed successfully in OnStop hook") log.Debug("Database closed successfully in OnStop hook")
return nil return nil
}, },
}) })

View File

@@ -50,21 +50,26 @@ type TxFunc func(ctx context.Context, tx *sql.Tx) error
// This method should be used for all write operations to ensure atomicity. // This method should be used for all write operations to ensure atomicity.
func (r *Repositories) WithTx(ctx context.Context, fn TxFunc) error { func (r *Repositories) WithTx(ctx context.Context, fn TxFunc) error {
LogSQL("WithTx", "Beginning transaction", "") LogSQL("WithTx", "Beginning transaction", "")
tx, err := r.db.BeginTx(ctx, nil) tx, err := r.db.BeginTx(ctx, nil)
if err != nil { if err != nil {
return fmt.Errorf("beginning transaction: %w", err) return fmt.Errorf("beginning transaction: %w", err)
} }
LogSQL("WithTx", "Transaction started", "") LogSQL("WithTx", "Transaction started", "")
defer func() { defer func() {
if p := recover(); p != nil { if p := recover(); p != nil {
if rollbackErr := tx.Rollback(); rollbackErr != nil { rollbackErr := tx.Rollback()
Fatal("failed to rollback transaction: %v", rollbackErr) if rollbackErr != nil {
Fatalf("failed to rollback transaction: %v", rollbackErr)
} }
panic(p) panic(p)
} else if err != nil { } else if err != nil {
if rollbackErr := tx.Rollback(); rollbackErr != nil { rollbackErr := tx.Rollback()
Fatal("failed to rollback transaction: %v", rollbackErr) if rollbackErr != nil {
Fatalf("failed to rollback transaction: %v", rollbackErr)
} }
} }
}() }()
@@ -90,6 +95,7 @@ func (r *Repositories) WithReadTx(ctx context.Context, fn TxFunc) error {
opts := &sql.TxOptions{ opts := &sql.TxOptions{
ReadOnly: true, ReadOnly: true,
} }
tx, err := r.db.BeginTx(ctx, opts) tx, err := r.db.BeginTx(ctx, opts)
if err != nil { if err != nil {
return fmt.Errorf("beginning read transaction: %w", err) return fmt.Errorf("beginning read transaction: %w", err)
@@ -97,13 +103,16 @@ func (r *Repositories) WithReadTx(ctx context.Context, fn TxFunc) error {
defer func() { defer func() {
if p := recover(); p != nil { if p := recover(); p != nil {
if rollbackErr := tx.Rollback(); rollbackErr != nil { rollbackErr := tx.Rollback()
Fatal("failed to rollback transaction: %v", rollbackErr) if rollbackErr != nil {
Fatalf("failed to rollback transaction: %v", rollbackErr)
} }
panic(p) panic(p)
} else if err != nil { } else if err != nil {
if rollbackErr := tx.Rollback(); rollbackErr != nil { rollbackErr := tx.Rollback()
Fatal("failed to rollback transaction: %v", rollbackErr) if rollbackErr != nil {
Fatalf("failed to rollback transaction: %v", rollbackErr)
} }
} }
}() }()

View File

@@ -3,7 +3,7 @@ package database
import ( import (
"context" "context"
"database/sql" "database/sql"
"fmt" "errors"
"testing" "testing"
"time" "time"
@@ -28,7 +28,9 @@ func TestRepositoriesTransaction(t *testing.T) {
UID: 1000, UID: 1000,
GID: 1000, GID: 1000,
} }
if err := repos.Files.Create(ctx, tx, file); err != nil {
err := repos.Files.Create(ctx, tx, file)
if err != nil {
return err return err
} }
@@ -37,7 +39,9 @@ func TestRepositoriesTransaction(t *testing.T) {
ChunkHash: types.ChunkHash("tx_chunk1"), ChunkHash: types.ChunkHash("tx_chunk1"),
Size: 512, Size: 512,
} }
if err := repos.Chunks.Create(ctx, tx, chunk1); err != nil {
err = repos.Chunks.Create(ctx, tx, chunk1)
if err != nil {
return err return err
} }
@@ -45,7 +49,9 @@ func TestRepositoriesTransaction(t *testing.T) {
ChunkHash: types.ChunkHash("tx_chunk2"), ChunkHash: types.ChunkHash("tx_chunk2"),
Size: 512, Size: 512,
} }
if err := repos.Chunks.Create(ctx, tx, chunk2); err != nil {
err = repos.Chunks.Create(ctx, tx, chunk2)
if err != nil {
return err return err
} }
@@ -55,7 +61,9 @@ func TestRepositoriesTransaction(t *testing.T) {
Idx: 0, Idx: 0,
ChunkHash: chunk1.ChunkHash, ChunkHash: chunk1.ChunkHash,
} }
if err := repos.FileChunks.Create(ctx, tx, fc1); err != nil {
err = repos.FileChunks.Create(ctx, tx, fc1)
if err != nil {
return err return err
} }
@@ -64,7 +72,9 @@ func TestRepositoriesTransaction(t *testing.T) {
Idx: 1, Idx: 1,
ChunkHash: chunk2.ChunkHash, ChunkHash: chunk2.ChunkHash,
} }
if err := repos.FileChunks.Create(ctx, tx, fc2); err != nil {
err = repos.FileChunks.Create(ctx, tx, fc2)
if err != nil {
return err return err
} }
@@ -74,7 +84,9 @@ func TestRepositoriesTransaction(t *testing.T) {
Hash: types.BlobHash("tx_blob1"), Hash: types.BlobHash("tx_blob1"),
CreatedTS: time.Now().Truncate(time.Second), CreatedTS: time.Now().Truncate(time.Second),
} }
if err := repos.Blobs.Create(ctx, tx, blob); err != nil {
err = repos.Blobs.Create(ctx, tx, blob)
if err != nil {
return err return err
} }
@@ -85,7 +97,9 @@ func TestRepositoriesTransaction(t *testing.T) {
Offset: 0, Offset: 0,
Length: 512, Length: 512,
} }
if err := repos.BlobChunks.Create(ctx, tx, bc1); err != nil {
err = repos.BlobChunks.Create(ctx, tx, bc1)
if err != nil {
return err return err
} }
@@ -95,13 +109,14 @@ func TestRepositoriesTransaction(t *testing.T) {
Offset: 512, Offset: 512,
Length: 512, Length: 512,
} }
if err := repos.BlobChunks.Create(ctx, tx, bc2); err != nil {
err = repos.BlobChunks.Create(ctx, tx, bc2)
if err != nil {
return err return err
} }
return nil return nil
}) })
if err != nil { if err != nil {
t.Fatalf("transaction failed: %v", err) t.Fatalf("transaction failed: %v", err)
} }
@@ -111,6 +126,7 @@ func TestRepositoriesTransaction(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("failed to get file: %v", err) t.Fatalf("failed to get file: %v", err)
} }
if file == nil { if file == nil {
t.Error("expected file after transaction") t.Error("expected file after transaction")
} }
@@ -119,6 +135,7 @@ func TestRepositoriesTransaction(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("failed to get file chunks: %v", err) t.Fatalf("failed to get file chunks: %v", err)
} }
if len(chunks) != 2 { if len(chunks) != 2 {
t.Errorf("expected 2 file chunks, got %d", len(chunks)) t.Errorf("expected 2 file chunks, got %d", len(chunks))
} }
@@ -127,6 +144,7 @@ func TestRepositoriesTransaction(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("failed to get blob: %v", err) t.Fatalf("failed to get blob: %v", err)
} }
if blob == nil { if blob == nil {
t.Error("expected blob after transaction") t.Error("expected blob after transaction")
} }
@@ -150,7 +168,9 @@ func TestRepositoriesTransactionRollback(t *testing.T) {
UID: 1000, UID: 1000,
GID: 1000, GID: 1000,
} }
if err := repos.Files.Create(ctx, tx, file); err != nil {
err := repos.Files.Create(ctx, tx, file)
if err != nil {
return err return err
} }
@@ -159,12 +179,14 @@ func TestRepositoriesTransactionRollback(t *testing.T) {
ChunkHash: types.ChunkHash("rollback_chunk"), ChunkHash: types.ChunkHash("rollback_chunk"),
Size: 1024, Size: 1024,
} }
if err := repos.Chunks.Create(ctx, tx, chunk); err != nil {
err = repos.Chunks.Create(ctx, tx, chunk)
if err != nil {
return err return err
} }
// Return error to trigger rollback // Return error to trigger rollback
return fmt.Errorf("intentional rollback") return errors.New("intentional rollback")
}) })
if err == nil || err.Error() != "intentional rollback" { if err == nil || err.Error() != "intentional rollback" {
@@ -176,6 +198,7 @@ func TestRepositoriesTransactionRollback(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("error checking for file: %v", err) t.Fatalf("error checking for file: %v", err)
} }
if file != nil { if file != nil {
t.Error("file should not exist after rollback") t.Error("file should not exist after rollback")
} }
@@ -184,6 +207,7 @@ func TestRepositoriesTransactionRollback(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("error checking for chunk: %v", err) t.Fatalf("error checking for chunk: %v", err)
} }
if chunk != nil { if chunk != nil {
t.Error("chunk should not exist after rollback") t.Error("chunk should not exist after rollback")
} }
@@ -205,6 +229,7 @@ func TestRepositoriesReadTransaction(t *testing.T) {
UID: 1000, UID: 1000,
GID: 1000, GID: 1000,
} }
err := repos.Files.Create(ctx, nil, file) err := repos.Files.Create(ctx, nil, file)
if err != nil { if err != nil {
t.Fatalf("failed to create file: %v", err) t.Fatalf("failed to create file: %v", err)
@@ -212,8 +237,10 @@ func TestRepositoriesReadTransaction(t *testing.T) {
// Test read-only transaction // Test read-only transaction
var retrievedFile *File var retrievedFile *File
err = repos.WithReadTx(ctx, func(ctx context.Context, tx *sql.Tx) error { err = repos.WithReadTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
var err error var err error
retrievedFile, err = repos.Files.GetByPathTx(ctx, tx, "/test/read_file.txt") retrievedFile, err = repos.Files.GetByPathTx(ctx, tx, "/test/read_file.txt")
if err != nil { if err != nil {
return err return err
@@ -232,7 +259,6 @@ func TestRepositoriesReadTransaction(t *testing.T) {
return nil return nil
}) })
if err != nil { if err != nil {
t.Fatalf("read transaction failed: %v", err) t.Fatalf("read transaction failed: %v", err)
} }

View File

@@ -3,6 +3,7 @@ package database
import ( import (
"context" "context"
"database/sql" "database/sql"
"errors"
"fmt" "fmt"
"testing" "testing"
"time" "time"
@@ -39,6 +40,7 @@ func TestFileRepositoryUUIDGeneration(t *testing.T) {
} }
uuids := make(map[string]bool) uuids := make(map[string]bool)
for _, file := range files { for _, file := range files {
err := repo.Create(ctx, nil, file) err := repo.Create(ctx, nil, file)
if err != nil { if err != nil {
@@ -54,6 +56,7 @@ func TestFileRepositoryUUIDGeneration(t *testing.T) {
if uuids[file.ID.String()] { if uuids[file.ID.String()] {
t.Errorf("duplicate UUID generated: %s", file.ID) t.Errorf("duplicate UUID generated: %s", file.ID)
} }
uuids[file.ID.String()] = true uuids[file.ID.String()] = true
} }
} }
@@ -90,16 +93,19 @@ func TestFileRepositoryGetByID(t *testing.T) {
if retrieved.ID != file.ID { if retrieved.ID != file.ID {
t.Errorf("ID mismatch: expected %s, got %s", file.ID, retrieved.ID) t.Errorf("ID mismatch: expected %s, got %s", file.ID, retrieved.ID)
} }
if retrieved.Path != file.Path { if retrieved.Path != file.Path {
t.Errorf("Path mismatch: expected %s, got %s", file.Path, retrieved.Path) t.Errorf("Path mismatch: expected %s, got %s", file.Path, retrieved.Path)
} }
// Test non-existent ID // Test non-existent ID
nonExistentID := types.NewFileID() // Generate a new UUID that won't exist in the database nonExistentID := types.NewFileID() // Generate a new UUID that won't exist in the database
nonExistent, err := repo.GetByID(ctx, nonExistentID) nonExistent, err := repo.GetByID(ctx, nonExistentID)
if err != nil { if err != nil {
t.Fatalf("GetByID should not return error for non-existent ID: %v", err) t.Fatalf("GetByID should not return error for non-existent ID: %v", err)
} }
if nonExistent != nil { if nonExistent != nil {
t.Error("expected nil for non-existent ID") t.Error("expected nil for non-existent ID")
} }
@@ -135,6 +141,7 @@ func TestOrphanedFileCleanup(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("failed to create file1: %v", err) t.Fatalf("failed to create file1: %v", err)
} }
err = repos.Files.Create(ctx, nil, file2) err = repos.Files.Create(ctx, nil, file2)
if err != nil { if err != nil {
t.Fatalf("failed to create file2: %v", err) t.Fatalf("failed to create file2: %v", err)
@@ -146,6 +153,7 @@ func TestOrphanedFileCleanup(t *testing.T) {
Hostname: "test-host", Hostname: "test-host",
StartedAt: time.Now(), StartedAt: time.Now(),
} }
err = repos.Snapshots.Create(ctx, nil, snapshot) err = repos.Snapshots.Create(ctx, nil, snapshot)
if err != nil { if err != nil {
t.Fatalf("failed to create snapshot: %v", err) t.Fatalf("failed to create snapshot: %v", err)
@@ -168,6 +176,7 @@ func TestOrphanedFileCleanup(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("error getting file: %v", err) t.Fatalf("error getting file: %v", err)
} }
if orphanedFile != nil { if orphanedFile != nil {
t.Error("orphaned file should have been deleted") t.Error("orphaned file should have been deleted")
} }
@@ -177,6 +186,7 @@ func TestOrphanedFileCleanup(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("error getting file: %v", err) t.Fatalf("error getting file: %v", err)
} }
if referencedFile == nil { if referencedFile == nil {
t.Error("referenced file should not have been deleted") t.Error("referenced file should not have been deleted")
} }
@@ -204,6 +214,7 @@ func TestOrphanedChunkCleanup(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("failed to create chunk1: %v", err) t.Fatalf("failed to create chunk1: %v", err)
} }
err = repos.Chunks.Create(ctx, nil, chunk2) err = repos.Chunks.Create(ctx, nil, chunk2)
if err != nil { if err != nil {
t.Fatalf("failed to create chunk2: %v", err) t.Fatalf("failed to create chunk2: %v", err)
@@ -218,6 +229,7 @@ func TestOrphanedChunkCleanup(t *testing.T) {
UID: 1000, UID: 1000,
GID: 1000, GID: 1000,
} }
err = repos.Files.Create(ctx, nil, file) err = repos.Files.Create(ctx, nil, file)
if err != nil { if err != nil {
t.Fatalf("failed to create file: %v", err) t.Fatalf("failed to create file: %v", err)
@@ -229,6 +241,7 @@ func TestOrphanedChunkCleanup(t *testing.T) {
Idx: 0, Idx: 0,
ChunkHash: chunk2.ChunkHash, ChunkHash: chunk2.ChunkHash,
} }
err = repos.FileChunks.Create(ctx, nil, fc) err = repos.FileChunks.Create(ctx, nil, fc)
if err != nil { if err != nil {
t.Fatalf("failed to create file chunk: %v", err) t.Fatalf("failed to create file chunk: %v", err)
@@ -245,6 +258,7 @@ func TestOrphanedChunkCleanup(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("error getting chunk: %v", err) t.Fatalf("error getting chunk: %v", err)
} }
if orphanedChunk != nil { if orphanedChunk != nil {
t.Error("orphaned chunk should have been deleted") t.Error("orphaned chunk should have been deleted")
} }
@@ -254,6 +268,7 @@ func TestOrphanedChunkCleanup(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("error getting chunk: %v", err) t.Fatalf("error getting chunk: %v", err)
} }
if referencedChunk == nil { if referencedChunk == nil {
t.Error("referenced chunk should not have been deleted") t.Error("referenced chunk should not have been deleted")
} }
@@ -283,6 +298,7 @@ func TestOrphanedBlobCleanup(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("failed to create blob1: %v", err) t.Fatalf("failed to create blob1: %v", err)
} }
err = repos.Blobs.Create(ctx, nil, blob2) err = repos.Blobs.Create(ctx, nil, blob2)
if err != nil { if err != nil {
t.Fatalf("failed to create blob2: %v", err) t.Fatalf("failed to create blob2: %v", err)
@@ -294,6 +310,7 @@ func TestOrphanedBlobCleanup(t *testing.T) {
Hostname: "test-host", Hostname: "test-host",
StartedAt: time.Now(), StartedAt: time.Now(),
} }
err = repos.Snapshots.Create(ctx, nil, snapshot) err = repos.Snapshots.Create(ctx, nil, snapshot)
if err != nil { if err != nil {
t.Fatalf("failed to create snapshot: %v", err) t.Fatalf("failed to create snapshot: %v", err)
@@ -316,6 +333,7 @@ func TestOrphanedBlobCleanup(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("error getting blob: %v", err) t.Fatalf("error getting blob: %v", err)
} }
if orphanedBlob != nil { if orphanedBlob != nil {
t.Error("orphaned blob should have been deleted") t.Error("orphaned blob should have been deleted")
} }
@@ -325,6 +343,7 @@ func TestOrphanedBlobCleanup(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("error getting blob: %v", err) t.Fatalf("error getting blob: %v", err)
} }
if referencedBlob == nil { if referencedBlob == nil {
t.Error("referenced blob should not have been deleted") t.Error("referenced blob should not have been deleted")
} }
@@ -347,6 +366,7 @@ func TestFileChunkRepositoryWithUUIDs(t *testing.T) {
UID: 1000, UID: 1000,
GID: 1000, GID: 1000,
} }
err := repos.Files.Create(ctx, nil, file) err := repos.Files.Create(ctx, nil, file)
if err != nil { if err != nil {
t.Fatalf("failed to create file: %v", err) t.Fatalf("failed to create file: %v", err)
@@ -359,6 +379,7 @@ func TestFileChunkRepositoryWithUUIDs(t *testing.T) {
ChunkHash: chunkHash, ChunkHash: chunkHash,
Size: 1024, Size: 1024,
} }
err = repos.Chunks.Create(ctx, nil, chunk) err = repos.Chunks.Create(ctx, nil, chunk)
if err != nil { if err != nil {
t.Fatalf("failed to create chunk: %v", err) t.Fatalf("failed to create chunk: %v", err)
@@ -370,6 +391,7 @@ func TestFileChunkRepositoryWithUUIDs(t *testing.T) {
Idx: i, Idx: i,
ChunkHash: chunkHash, ChunkHash: chunkHash,
} }
err = repos.FileChunks.Create(ctx, nil, fc) err = repos.FileChunks.Create(ctx, nil, fc)
if err != nil { if err != nil {
t.Fatalf("failed to create file chunk: %v", err) t.Fatalf("failed to create file chunk: %v", err)
@@ -381,6 +403,7 @@ func TestFileChunkRepositoryWithUUIDs(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("failed to get file chunks: %v", err) t.Fatalf("failed to get file chunks: %v", err)
} }
if len(fileChunks) != 3 { if len(fileChunks) != 3 {
t.Errorf("expected 3 chunks, got %d", len(fileChunks)) t.Errorf("expected 3 chunks, got %d", len(fileChunks))
} }
@@ -395,6 +418,7 @@ func TestFileChunkRepositoryWithUUIDs(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("failed to get file chunks after delete: %v", err) t.Fatalf("failed to get file chunks after delete: %v", err)
} }
if len(fileChunks) != 0 { if len(fileChunks) != 0 {
t.Errorf("expected 0 chunks after delete, got %d", len(fileChunks)) t.Errorf("expected 0 chunks after delete, got %d", len(fileChunks))
} }
@@ -430,6 +454,7 @@ func TestChunkFileRepositoryWithUUIDs(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("failed to create file1: %v", err) t.Fatalf("failed to create file1: %v", err)
} }
err = repos.Files.Create(ctx, nil, file2) err = repos.Files.Create(ctx, nil, file2)
if err != nil { if err != nil {
t.Fatalf("failed to create file2: %v", err) t.Fatalf("failed to create file2: %v", err)
@@ -440,6 +465,7 @@ func TestChunkFileRepositoryWithUUIDs(t *testing.T) {
ChunkHash: types.ChunkHash("shared-chunk"), ChunkHash: types.ChunkHash("shared-chunk"),
Size: 1024, Size: 1024,
} }
err = repos.Chunks.Create(ctx, nil, chunk) err = repos.Chunks.Create(ctx, nil, chunk)
if err != nil { if err != nil {
t.Fatalf("failed to create chunk: %v", err) t.Fatalf("failed to create chunk: %v", err)
@@ -463,6 +489,7 @@ func TestChunkFileRepositoryWithUUIDs(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("failed to create chunk file 1: %v", err) t.Fatalf("failed to create chunk file 1: %v", err)
} }
err = repos.ChunkFiles.Create(ctx, nil, cf2) err = repos.ChunkFiles.Create(ctx, nil, cf2)
if err != nil { if err != nil {
t.Fatalf("failed to create chunk file 2: %v", err) t.Fatalf("failed to create chunk file 2: %v", err)
@@ -473,6 +500,7 @@ func TestChunkFileRepositoryWithUUIDs(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("failed to get chunk files: %v", err) t.Fatalf("failed to get chunk files: %v", err)
} }
if len(chunkFiles) != 2 { if len(chunkFiles) != 2 {
t.Errorf("expected 2 files for chunk, got %d", len(chunkFiles)) t.Errorf("expected 2 files for chunk, got %d", len(chunkFiles))
} }
@@ -482,6 +510,7 @@ func TestChunkFileRepositoryWithUUIDs(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("failed to get chunks by file ID: %v", err) t.Fatalf("failed to get chunks by file ID: %v", err)
} }
if len(chunkFiles) != 1 { if len(chunkFiles) != 1 {
t.Errorf("expected 1 chunk for file, got %d", len(chunkFiles)) t.Errorf("expected 1 chunk for file, got %d", len(chunkFiles))
} }
@@ -528,15 +557,19 @@ func TestSnapshotRepositoryExtendedFields(t *testing.T) {
if retrieved.VaultikVersion != snapshot.VaultikVersion { if retrieved.VaultikVersion != snapshot.VaultikVersion {
t.Errorf("version mismatch: expected %s, got %s", snapshot.VaultikVersion, retrieved.VaultikVersion) t.Errorf("version mismatch: expected %s, got %s", snapshot.VaultikVersion, retrieved.VaultikVersion)
} }
if retrieved.VaultikGitRevision != snapshot.VaultikGitRevision { if retrieved.VaultikGitRevision != snapshot.VaultikGitRevision {
t.Errorf("git revision mismatch: expected %s, got %s", snapshot.VaultikGitRevision, retrieved.VaultikGitRevision) t.Errorf("git revision mismatch: expected %s, got %s", snapshot.VaultikGitRevision, retrieved.VaultikGitRevision)
} }
if retrieved.CompressionLevel != snapshot.CompressionLevel { if retrieved.CompressionLevel != snapshot.CompressionLevel {
t.Errorf("compression level mismatch: expected %d, got %d", snapshot.CompressionLevel, retrieved.CompressionLevel) t.Errorf("compression level mismatch: expected %d, got %d", snapshot.CompressionLevel, retrieved.CompressionLevel)
} }
if retrieved.BlobUncompressedSize != snapshot.BlobUncompressedSize { if retrieved.BlobUncompressedSize != snapshot.BlobUncompressedSize {
t.Errorf("uncompressed size mismatch: expected %d, got %d", snapshot.BlobUncompressedSize, retrieved.BlobUncompressedSize) t.Errorf("uncompressed size mismatch: expected %d, got %d", snapshot.BlobUncompressedSize, retrieved.BlobUncompressedSize)
} }
if retrieved.UploadDurationMs != snapshot.UploadDurationMs { if retrieved.UploadDurationMs != snapshot.UploadDurationMs {
t.Errorf("upload duration mismatch: expected %d, got %d", snapshot.UploadDurationMs, retrieved.UploadDurationMs) t.Errorf("upload duration mismatch: expected %d, got %d", snapshot.UploadDurationMs, retrieved.UploadDurationMs)
} }
@@ -566,6 +599,7 @@ func TestComplexOrphanedDataScenario(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("failed to create snapshot1: %v", err) t.Fatalf("failed to create snapshot1: %v", err)
} }
err = repos.Snapshots.Create(ctx, nil, snapshot2) err = repos.Snapshots.Create(ctx, nil, snapshot2)
if err != nil { if err != nil {
t.Fatalf("failed to create snapshot2: %v", err) t.Fatalf("failed to create snapshot2: %v", err)
@@ -582,6 +616,7 @@ func TestComplexOrphanedDataScenario(t *testing.T) {
UID: 1000, UID: 1000,
GID: 1000, GID: 1000,
} }
err = repos.Files.Create(ctx, nil, files[i]) err = repos.Files.Create(ctx, nil, files[i])
if err != nil { if err != nil {
t.Fatalf("failed to create file%d: %v", i, err) t.Fatalf("failed to create file%d: %v", i, err)
@@ -598,14 +633,17 @@ func TestComplexOrphanedDataScenario(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
err = repos.Snapshots.AddFileByID(ctx, nil, snapshot1.ID.String(), files[1].ID) err = repos.Snapshots.AddFileByID(ctx, nil, snapshot1.ID.String(), files[1].ID)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
err = repos.Snapshots.AddFileByID(ctx, nil, snapshot2.ID.String(), files[1].ID) err = repos.Snapshots.AddFileByID(ctx, nil, snapshot2.ID.String(), files[1].ID)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
err = repos.Snapshots.AddFileByID(ctx, nil, snapshot2.ID.String(), files[2].ID) err = repos.Snapshots.AddFileByID(ctx, nil, snapshot2.ID.String(), files[2].ID)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@@ -616,6 +654,7 @@ func TestComplexOrphanedDataScenario(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
err = repos.Snapshots.Delete(ctx, snapshot1.ID.String()) err = repos.Snapshots.Delete(ctx, snapshot1.ID.String())
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@@ -633,6 +672,7 @@ func TestComplexOrphanedDataScenario(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("error getting file0: %v", err) t.Fatalf("error getting file0: %v", err)
} }
if file0 != nil { if file0 != nil {
t.Error("file0 should have been deleted") t.Error("file0 should have been deleted")
} }
@@ -642,6 +682,7 @@ func TestComplexOrphanedDataScenario(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("error getting file1: %v", err) t.Fatalf("error getting file1: %v", err)
} }
if file1 == nil { if file1 == nil {
t.Error("file1 should still exist") t.Error("file1 should still exist")
} }
@@ -651,6 +692,7 @@ func TestComplexOrphanedDataScenario(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("error getting file2: %v", err) t.Fatalf("error getting file2: %v", err)
} }
if file2 == nil { if file2 == nil {
t.Error("file2 should still exist") t.Error("file2 should still exist")
} }
@@ -673,17 +715,19 @@ func TestCascadeDelete(t *testing.T) {
UID: 1000, UID: 1000,
GID: 1000, GID: 1000,
} }
err := repos.Files.Create(ctx, nil, file) err := repos.Files.Create(ctx, nil, file)
if err != nil { if err != nil {
t.Fatalf("failed to create file: %v", err) t.Fatalf("failed to create file: %v", err)
} }
// Create chunks and file-chunk mappings // Create chunks and file-chunk mappings
for i := 0; i < 3; i++ { for i := range 3 {
chunk := &Chunk{ chunk := &Chunk{
ChunkHash: types.ChunkHash(fmt.Sprintf("cascade-chunk-%d", i)), ChunkHash: types.ChunkHash(fmt.Sprintf("cascade-chunk-%d", i)),
Size: 1024, Size: 1024,
} }
err = repos.Chunks.Create(ctx, nil, chunk) err = repos.Chunks.Create(ctx, nil, chunk)
if err != nil { if err != nil {
t.Fatalf("failed to create chunk: %v", err) t.Fatalf("failed to create chunk: %v", err)
@@ -694,6 +738,7 @@ func TestCascadeDelete(t *testing.T) {
Idx: i, Idx: i,
ChunkHash: chunk.ChunkHash, ChunkHash: chunk.ChunkHash,
} }
err = repos.FileChunks.Create(ctx, nil, fc) err = repos.FileChunks.Create(ctx, nil, fc)
if err != nil { if err != nil {
t.Fatalf("failed to create file chunk: %v", err) t.Fatalf("failed to create file chunk: %v", err)
@@ -705,6 +750,7 @@ func TestCascadeDelete(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
if len(fileChunks) != 3 { if len(fileChunks) != 3 {
t.Errorf("expected 3 file chunks, got %d", len(fileChunks)) t.Errorf("expected 3 file chunks, got %d", len(fileChunks))
} }
@@ -720,6 +766,7 @@ func TestCascadeDelete(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
if len(fileChunks) != 0 { if len(fileChunks) != 0 {
t.Errorf("expected 0 file chunks after cascade delete, got %d", len(fileChunks)) t.Errorf("expected 0 file chunks after cascade delete, got %d", len(fileChunks))
} }
@@ -744,6 +791,7 @@ func TestTransactionIsolation(t *testing.T) {
UID: 1000, UID: 1000,
GID: 1000, GID: 1000,
} }
err := repos.Files.Create(ctx, tx, file) err := repos.Files.Create(ctx, tx, file)
if err != nil { if err != nil {
return err return err
@@ -754,9 +802,8 @@ func TestTransactionIsolation(t *testing.T) {
// For now, we'll just test that rollback works // For now, we'll just test that rollback works
// Return an error to trigger rollback // Return an error to trigger rollback
return fmt.Errorf("intentional rollback") return errors.New("intentional rollback")
}) })
if err == nil { if err == nil {
t.Fatal("expected error from transaction") t.Fatal("expected error from transaction")
} }
@@ -766,6 +813,7 @@ func TestTransactionIsolation(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
if len(files) != 0 { if len(files) != 0 {
t.Error("file should not exist after rollback") t.Error("file should not exist after rollback")
} }
@@ -790,13 +838,14 @@ func TestConcurrentOrphanedCleanup(t *testing.T) {
Hostname: "test-host", Hostname: "test-host",
StartedAt: time.Now(), StartedAt: time.Now(),
} }
err := repos.Snapshots.Create(ctx, nil, snapshot) err := repos.Snapshots.Create(ctx, nil, snapshot)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
// Create many files, some orphaned // Create many files, some orphaned
for i := 0; i < 20; i++ { for i := range 20 {
file := &File{ file := &File{
Path: types.FilePath(fmt.Sprintf("/concurrent-%d.txt", i)), Path: types.FilePath(fmt.Sprintf("/concurrent-%d.txt", i)),
MTime: time.Now().Truncate(time.Second), MTime: time.Now().Truncate(time.Second),
@@ -805,6 +854,7 @@ func TestConcurrentOrphanedCleanup(t *testing.T) {
UID: 1000, UID: 1000,
GID: 1000, GID: 1000,
} }
err = repos.Files.Create(ctx, nil, file) err = repos.Files.Create(ctx, nil, file)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@@ -822,14 +872,15 @@ func TestConcurrentOrphanedCleanup(t *testing.T) {
// Run multiple cleanup operations concurrently // Run multiple cleanup operations concurrently
// Note: SQLite has limited support for concurrent writes, so we expect some to fail // Note: SQLite has limited support for concurrent writes, so we expect some to fail
done := make(chan error, 3) done := make(chan error, 3)
for i := 0; i < 3; i++ {
for range 3 {
go func() { go func() {
done <- repos.Files.DeleteOrphaned(ctx) done <- repos.Files.DeleteOrphaned(ctx)
}() }()
} }
// Wait for all to complete // Wait for all to complete
for i := 0; i < 3; i++ { for i := range 3 {
err := <-done err := <-done
if err != nil { if err != nil {
t.Errorf("cleanup %d failed: %v", i, err) t.Errorf("cleanup %d failed: %v", i, err)
@@ -850,10 +901,12 @@ func TestConcurrentOrphanedCleanup(t *testing.T) {
// Verify all remaining files are even-numbered // Verify all remaining files are even-numbered
for _, file := range files { for _, file := range files {
var num int var num int
_, err := fmt.Sscanf(file.Path.String(), "/concurrent-%d.txt", &num) _, err := fmt.Sscanf(file.Path.String(), "/concurrent-%d.txt", &num)
if err != nil { if err != nil {
t.Logf("failed to parse file number from %s: %v", file.Path, err) t.Logf("failed to parse file number from %s: %v", file.Path, err)
} }
if num%2 != 0 { if num%2 != 0 {
t.Errorf("odd-numbered file %s should have been deleted", file.Path) t.Errorf("odd-numbered file %s should have been deleted", file.Path)
} }

View File

@@ -36,12 +36,14 @@ func TestOrphanedFileCleanupDebug(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("failed to create file1: %v", err) t.Fatalf("failed to create file1: %v", err)
} }
t.Logf("Created file1 with ID: %s", file1.ID) t.Logf("Created file1 with ID: %s", file1.ID)
err = repos.Files.Create(ctx, nil, file2) err = repos.Files.Create(ctx, nil, file2)
if err != nil { if err != nil {
t.Fatalf("failed to create file2: %v", err) t.Fatalf("failed to create file2: %v", err)
} }
t.Logf("Created file2 with ID: %s", file2.ID) t.Logf("Created file2 with ID: %s", file2.ID)
// Create a snapshot and reference only file2 // Create a snapshot and reference only file2
@@ -50,18 +52,22 @@ func TestOrphanedFileCleanupDebug(t *testing.T) {
Hostname: "test-host", Hostname: "test-host",
StartedAt: time.Now(), StartedAt: time.Now(),
} }
err = repos.Snapshots.Create(ctx, nil, snapshot) err = repos.Snapshots.Create(ctx, nil, snapshot)
if err != nil { if err != nil {
t.Fatalf("failed to create snapshot: %v", err) t.Fatalf("failed to create snapshot: %v", err)
} }
t.Logf("Created snapshot: %s", snapshot.ID) t.Logf("Created snapshot: %s", snapshot.ID)
// Check snapshot_files before adding // Check snapshot_files before adding
var count int var count int
err = db.conn.QueryRow("SELECT COUNT(*) FROM snapshot_files").Scan(&count) err = db.conn.QueryRow("SELECT COUNT(*) FROM snapshot_files").Scan(&count)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
t.Logf("snapshot_files count before add: %d", count) t.Logf("snapshot_files count before add: %d", count)
// Add file2 to snapshot // Add file2 to snapshot
@@ -69,6 +75,7 @@ func TestOrphanedFileCleanupDebug(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("failed to add file to snapshot: %v", err) t.Fatalf("failed to add file to snapshot: %v", err)
} }
t.Logf("Added file2 to snapshot") t.Logf("Added file2 to snapshot")
// Check snapshot_files after adding // Check snapshot_files after adding
@@ -76,6 +83,7 @@ func TestOrphanedFileCleanupDebug(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
t.Logf("snapshot_files count after add: %d", count) t.Logf("snapshot_files count after add: %d", count)
// Check which files are referenced // Check which files are referenced
@@ -84,16 +92,22 @@ func TestOrphanedFileCleanupDebug(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
defer func() { defer func() {
if err := rows.Close(); err != nil { err := rows.Close()
if err != nil {
t.Logf("failed to close rows: %v", err) t.Logf("failed to close rows: %v", err)
} }
}() }()
t.Log("Files in snapshot_files:") t.Log("Files in snapshot_files:")
for rows.Next() { for rows.Next() {
var fileID string var fileID string
if err := rows.Scan(&fileID); err != nil {
err := rows.Scan(&fileID)
if err != nil {
t.Fatal(err) t.Fatal(err)
} }
t.Logf(" - %s", fileID) t.Logf(" - %s", fileID)
} }
@@ -102,6 +116,7 @@ func TestOrphanedFileCleanupDebug(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
t.Logf("Files count before cleanup: %d", count) t.Logf("Files count before cleanup: %d", count)
// Run orphaned cleanup // Run orphaned cleanup
@@ -109,6 +124,7 @@ func TestOrphanedFileCleanupDebug(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("failed to delete orphaned files: %v", err) t.Fatalf("failed to delete orphaned files: %v", err)
} }
t.Log("Ran orphaned cleanup") t.Log("Ran orphaned cleanup")
// Check files after cleanup // Check files after cleanup
@@ -116,6 +132,7 @@ func TestOrphanedFileCleanupDebug(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
t.Logf("Files count after cleanup: %d", count) t.Logf("Files count after cleanup: %d", count)
// List remaining files // List remaining files
@@ -123,7 +140,9 @@ func TestOrphanedFileCleanupDebug(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
t.Log("Remaining files:") t.Log("Remaining files:")
for _, f := range files { for _, f := range files {
t.Logf(" - ID: %s, Path: %s", f.ID, f.Path) t.Logf(" - ID: %s, Path: %s", f.ID, f.Path)
} }
@@ -133,10 +152,12 @@ func TestOrphanedFileCleanupDebug(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("error getting file: %v", err) t.Fatalf("error getting file: %v", err)
} }
if orphanedFile != nil { if orphanedFile != nil {
t.Error("orphaned file should have been deleted") t.Error("orphaned file should have been deleted")
// Let's check why it wasn't deleted // Let's check why it wasn't deleted
var exists bool var exists bool
err = db.conn.QueryRow(` err = db.conn.QueryRow(`
SELECT EXISTS( SELECT EXISTS(
SELECT 1 FROM snapshot_files SELECT 1 FROM snapshot_files
@@ -145,6 +166,7 @@ func TestOrphanedFileCleanupDebug(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
t.Logf("File1 exists in snapshot_files: %v", exists) t.Logf("File1 exists in snapshot_files: %v", exists)
} else { } else {
t.Log("Orphaned file was correctly deleted") t.Log("Orphaned file was correctly deleted")
@@ -155,6 +177,7 @@ func TestOrphanedFileCleanupDebug(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("error getting file: %v", err) t.Fatalf("error getting file: %v", err)
} }
if referencedFile == nil { if referencedFile == nil {
t.Error("referenced file should not have been deleted") t.Error("referenced file should not have been deleted")
} else { } else {

View File

@@ -98,6 +98,7 @@ func TestFileRepositoryEdgeCases(t *testing.T) {
if (err != nil) != tt.wantErr { if (err != nil) != tt.wantErr {
t.Errorf("Create() error = %v, wantErr %v", err, tt.wantErr) t.Errorf("Create() error = %v, wantErr %v", err, tt.wantErr)
} }
if err != nil && tt.errMsg != "" && !strings.Contains(err.Error(), tt.errMsg) { if err != nil && tt.errMsg != "" && !strings.Contains(err.Error(), tt.errMsg) {
t.Errorf("Create() error = %v, want error containing %q", err, tt.errMsg) t.Errorf("Create() error = %v, want error containing %q", err, tt.errMsg)
} }
@@ -136,6 +137,7 @@ func TestDuplicateHandling(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("failed to create file1: %v", err) t.Fatalf("failed to create file1: %v", err)
} }
originalID := file1.ID originalID := file1.ID
// Create with same path should update the existing record (UPSERT behavior) // Create with same path should update the existing record (UPSERT behavior)
@@ -190,6 +192,7 @@ func TestDuplicateHandling(t *testing.T) {
UID: 1000, UID: 1000,
GID: 1000, GID: 1000,
} }
err := repos.Files.Create(ctx, nil, file) err := repos.Files.Create(ctx, nil, file)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@@ -199,6 +202,7 @@ func TestDuplicateHandling(t *testing.T) {
ChunkHash: types.ChunkHash("test-chunk-dup"), ChunkHash: types.ChunkHash("test-chunk-dup"),
Size: 1024, Size: 1024,
} }
err = repos.Chunks.Create(ctx, nil, chunk) err = repos.Chunks.Create(ctx, nil, chunk)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@@ -325,6 +329,7 @@ func TestLargeDatasets(t *testing.T) {
Hostname: "test-host", Hostname: "test-host",
StartedAt: time.Now(), StartedAt: time.Now(),
} }
err := repos.Snapshots.Create(ctx, nil, snapshot) err := repos.Snapshots.Create(ctx, nil, snapshot)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@@ -332,11 +337,12 @@ func TestLargeDatasets(t *testing.T) {
// Create many files // Create many files
const fileCount = 1000 const fileCount = 1000
fileIDs := make([]types.FileID, fileCount) fileIDs := make([]types.FileID, fileCount)
t.Run("create many files", func(t *testing.T) { t.Run("create many files", func(t *testing.T) {
start := time.Now() start := time.Now()
for i := 0; i < fileCount; i++ { for i := range fileCount {
file := &File{ file := &File{
Path: types.FilePath(fmt.Sprintf("/large/file%05d.txt", i)), Path: types.FilePath(fmt.Sprintf("/large/file%05d.txt", i)),
MTime: time.Now(), MTime: time.Now(),
@@ -345,10 +351,12 @@ func TestLargeDatasets(t *testing.T) {
UID: uint32(1000 + (i % 10)), UID: uint32(1000 + (i % 10)),
GID: uint32(1000 + (i % 10)), GID: uint32(1000 + (i % 10)),
} }
err := repos.Files.Create(ctx, nil, file) err := repos.Files.Create(ctx, nil, file)
if err != nil { if err != nil {
t.Fatalf("failed to create file %d: %v", i, err) t.Fatalf("failed to create file %d: %v", i, err)
} }
fileIDs[i] = file.ID fileIDs[i] = file.ID
// Add half to snapshot // Add half to snapshot
@@ -359,29 +367,35 @@ func TestLargeDatasets(t *testing.T) {
} }
} }
} }
t.Logf("Created %d files in %v", fileCount, time.Since(start)) t.Logf("Created %d files in %v", fileCount, time.Since(start))
}) })
// Test ListByPrefix performance // Test ListByPrefix performance
t.Run("list by prefix performance", func(t *testing.T) { t.Run("list by prefix performance", func(t *testing.T) {
start := time.Now() start := time.Now()
files, err := repos.Files.ListByPrefix(ctx, "/large/") files, err := repos.Files.ListByPrefix(ctx, "/large/")
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
if len(files) != fileCount { if len(files) != fileCount {
t.Errorf("expected %d files, got %d", fileCount, len(files)) t.Errorf("expected %d files, got %d", fileCount, len(files))
} }
t.Logf("Listed %d files in %v", len(files), time.Since(start)) t.Logf("Listed %d files in %v", len(files), time.Since(start))
}) })
// Test orphaned cleanup performance // Test orphaned cleanup performance
t.Run("orphaned cleanup performance", func(t *testing.T) { t.Run("orphaned cleanup performance", func(t *testing.T) {
start := time.Now() start := time.Now()
err := repos.Files.DeleteOrphaned(ctx) err := repos.Files.DeleteOrphaned(ctx)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
t.Logf("Cleaned up orphaned files in %v", time.Since(start)) t.Logf("Cleaned up orphaned files in %v", time.Since(start))
// Verify correct number remain // Verify correct number remain
@@ -389,6 +403,7 @@ func TestLargeDatasets(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
if len(files) != fileCount/2 { if len(files) != fileCount/2 {
t.Errorf("expected %d files after cleanup, got %d", fileCount/2, len(files)) t.Errorf("expected %d files after cleanup, got %d", fileCount/2, len(files))
} }
@@ -409,6 +424,7 @@ func TestErrorPropagation(t *testing.T) {
if err != nil { if err != nil {
t.Errorf("GetByID should not return error for non-existent ID, got: %v", err) t.Errorf("GetByID should not return error for non-existent ID, got: %v", err)
} }
if file != nil { if file != nil {
t.Error("expected nil file for non-existent ID") t.Error("expected nil file for non-existent ID")
} }
@@ -420,6 +436,7 @@ func TestErrorPropagation(t *testing.T) {
if err != nil { if err != nil {
t.Errorf("GetByPath should not return error for non-existent path, got: %v", err) t.Errorf("GetByPath should not return error for non-existent path, got: %v", err)
} }
if file != nil { if file != nil {
t.Error("expected nil file for non-existent path") t.Error("expected nil file for non-existent path")
} }
@@ -432,10 +449,12 @@ func TestErrorPropagation(t *testing.T) {
Idx: 0, Idx: 0,
ChunkHash: types.ChunkHash("some-chunk"), ChunkHash: types.ChunkHash("some-chunk"),
} }
err := repos.FileChunks.Create(ctx, nil, fc) err := repos.FileChunks.Create(ctx, nil, fc)
if err == nil { if err == nil {
t.Error("expected error for invalid foreign key") t.Error("expected error for invalid foreign key")
} }
if !strings.Contains(err.Error(), "FOREIGN KEY") { if !strings.Contains(err.Error(), "FOREIGN KEY") {
t.Errorf("expected foreign key error, got: %v", err) t.Errorf("expected foreign key error, got: %v", err)
} }
@@ -475,6 +494,7 @@ func TestQueryInjection(t *testing.T) {
// Verify tables still exist // Verify tables still exist
var count int var count int
err := db.conn.QueryRow("SELECT COUNT(*) FROM files").Scan(&count) err := db.conn.QueryRow("SELECT COUNT(*) FROM files").Scan(&count)
if err != nil { if err != nil {
t.Fatal("files table was damaged by injection") t.Fatal("files table was damaged by injection")

View File

@@ -4,6 +4,7 @@ import (
"context" "context"
"database/sql" "database/sql"
"fmt" "fmt"
"strings"
"time" "time"
"sneak.berlin/go/vaultik/internal/types" "sneak.berlin/go/vaultik/internal/types"
@@ -26,6 +27,7 @@ func (r *SnapshotRepository) Create(ctx context.Context, tx *sql.Tx, snapshot *S
` `
var completedAt *int64 var completedAt *int64
if snapshot.CompletedAt != nil { if snapshot.CompletedAt != nil {
ts := snapshot.CompletedAt.Unix() ts := snapshot.CompletedAt.Unix()
completedAt = &ts completedAt = &ts
@@ -84,9 +86,11 @@ func (r *SnapshotRepository) UpdateCounts(ctx context.Context, tx *sql.Tx, snaps
func (r *SnapshotRepository) UpdateExtendedStats(ctx context.Context, tx *sql.Tx, snapshotID string, blobUncompressedSize int64, compressionLevel int, uploadDurationMs int64) error { func (r *SnapshotRepository) UpdateExtendedStats(ctx context.Context, tx *sql.Tx, snapshotID string, blobUncompressedSize int64, compressionLevel int, uploadDurationMs int64) error {
// Calculate compression ratio based on uncompressed vs compressed sizes // Calculate compression ratio based on uncompressed vs compressed sizes
var compressionRatio float64 var compressionRatio float64
if blobUncompressedSize > 0 { if blobUncompressedSize > 0 {
// Get current blob_size from DB to calculate ratio // Get current blob_size from DB to calculate ratio
var blobSize int64 var blobSize int64
queryGet := `SELECT blob_size FROM snapshots WHERE id = ?` queryGet := `SELECT blob_size FROM snapshots WHERE id = ?`
if tx != nil { if tx != nil {
err := tx.QueryRowContext(ctx, queryGet, snapshotID).Scan(&blobSize) err := tx.QueryRowContext(ctx, queryGet, snapshotID).Scan(&blobSize)
@@ -99,6 +103,7 @@ func (r *SnapshotRepository) UpdateExtendedStats(ctx context.Context, tx *sql.Tx
return fmt.Errorf("getting blob size: %w", err) return fmt.Errorf("getting blob size: %w", err)
} }
} }
compressionRatio = float64(blobSize) / float64(blobUncompressedSize) compressionRatio = float64(blobSize) / float64(blobUncompressedSize)
} else { } else {
compressionRatio = 1.0 compressionRatio = 1.0
@@ -124,6 +129,7 @@ func (r *SnapshotRepository) UpdateExtendedStats(ctx context.Context, tx *sql.Tx
if err != nil { if err != nil {
return fmt.Errorf("updating extended stats: %w", err) return fmt.Errorf("updating extended stats: %w", err)
} }
return nil return nil
} }
@@ -136,9 +142,11 @@ func (r *SnapshotRepository) GetByID(ctx context.Context, snapshotID string) (*S
WHERE id = ? WHERE id = ?
` `
var snapshot Snapshot var (
var startedAtUnix int64 snapshot Snapshot
var completedAtUnix *int64 startedAtUnix int64
completedAtUnix *int64
)
err := r.db.conn.QueryRowContext(ctx, query, snapshotID).Scan( err := r.db.conn.QueryRowContext(ctx, query, snapshotID).Scan(
&snapshot.ID, &snapshot.ID,
@@ -162,6 +170,7 @@ func (r *SnapshotRepository) GetByID(ctx context.Context, snapshotID string) (*S
if err == sql.ErrNoRows { if err == sql.ErrNoRows {
return nil, nil return nil, nil
} }
if err != nil { if err != nil {
return nil, fmt.Errorf("querying snapshot: %w", err) return nil, fmt.Errorf("querying snapshot: %w", err)
} }
@@ -190,10 +199,13 @@ func (r *SnapshotRepository) ListRecent(ctx context.Context, limit int) ([]*Snap
defer CloseRows(rows) defer CloseRows(rows)
var snapshots []*Snapshot var snapshots []*Snapshot
for rows.Next() { for rows.Next() {
var snapshot Snapshot var (
var startedAtUnix int64 snapshot Snapshot
var completedAtUnix *int64 startedAtUnix int64
completedAtUnix *int64
)
err := rows.Scan( err := rows.Scan(
&snapshot.ID, &snapshot.ID,
@@ -301,28 +313,35 @@ func (r *SnapshotRepository) AddFilesByIDBatch(ctx context.Context, tx *sql.Tx,
const batchSize = 400 const batchSize = 400
for i := 0; i < len(fileIDs); i += batchSize { for i := 0; i < len(fileIDs); i += batchSize {
end := i + batchSize end := min(i+batchSize, len(fileIDs))
if end > len(fileIDs) {
end = len(fileIDs)
}
batch := fileIDs[i:end] batch := fileIDs[i:end]
query := "INSERT OR IGNORE INTO snapshot_files (snapshot_id, file_id) VALUES " query := "INSERT OR IGNORE INTO snapshot_files (snapshot_id, file_id) VALUES "
args := make([]interface{}, 0, len(batch)*2)
args := make([]any, 0, len(batch)*2)
var querySb312 strings.Builder
for j, fileID := range batch { for j, fileID := range batch {
if j > 0 { if j > 0 {
query += ", " querySb312.WriteString(", ")
} }
query += "(?, ?)"
querySb312.WriteString("(?, ?)")
args = append(args, snapshotID, fileID.String()) args = append(args, snapshotID, fileID.String())
} }
query += querySb312.String()
var err error var err error
if tx != nil { if tx != nil {
_, err = tx.ExecContext(ctx, query, args...) _, err = tx.ExecContext(ctx, query, args...)
} else { } else {
_, err = r.db.ExecWithLog(ctx, query, args...) _, err = r.db.ExecWithLog(ctx, query, args...)
} }
if err != nil { if err != nil {
return fmt.Errorf("batch adding files to snapshot: %w", err) return fmt.Errorf("batch adding files to snapshot: %w", err)
} }
@@ -353,18 +372,22 @@ func (r *SnapshotRepository) PopulateReferencedBlobs(ctx context.Context, tx *sq
AND blobs.blob_hash IS NOT NULL AND blobs.blob_hash IS NOT NULL
` `
var result sql.Result var (
var err error result sql.Result
err error
)
if tx != nil { if tx != nil {
result, err = tx.ExecContext(ctx, query, snapshotID, snapshotID) result, err = tx.ExecContext(ctx, query, snapshotID, snapshotID)
} else { } else {
result, err = r.db.ExecWithLog(ctx, query, snapshotID, snapshotID) result, err = r.db.ExecWithLog(ctx, query, snapshotID, snapshotID)
} }
if err != nil { if err != nil {
return 0, fmt.Errorf("populating referenced blobs: %w", err) return 0, fmt.Errorf("populating referenced blobs: %w", err)
} }
n, _ := result.RowsAffected() n, _ := result.RowsAffected()
return n, nil return n, nil
} }
@@ -405,11 +428,15 @@ func (r *SnapshotRepository) GetBlobHashes(ctx context.Context, snapshotID strin
defer CloseRows(rows) defer CloseRows(rows)
var blobs []string var blobs []string
for rows.Next() { for rows.Next() {
var blobHash string var blobHash string
if err := rows.Scan(&blobHash); err != nil {
err := rows.Scan(&blobHash)
if err != nil {
return nil, fmt.Errorf("scanning blob hash: %w", err) return nil, fmt.Errorf("scanning blob hash: %w", err)
} }
blobs = append(blobs, blobHash) blobs = append(blobs, blobHash)
} }
@@ -426,6 +453,7 @@ func (r *SnapshotRepository) GetSnapshotTotalCompressedSize(ctx context.Context,
` `
var totalSize int64 var totalSize int64
err := r.db.conn.QueryRowContext(ctx, query, snapshotID).Scan(&totalSize) err := r.db.conn.QueryRowContext(ctx, query, snapshotID).Scan(&totalSize)
if err != nil { if err != nil {
return 0, fmt.Errorf("querying total compressed size: %w", err) return 0, fmt.Errorf("querying total compressed size: %w", err)
@@ -449,6 +477,7 @@ func (r *SnapshotRepository) GetSnapshotUncompressedChunkSize(ctx context.Contex
` `
var totalSize int64 var totalSize int64
err := r.db.conn.QueryRowContext(ctx, query, snapshotID).Scan(&totalSize) err := r.db.conn.QueryRowContext(ctx, query, snapshotID).Scan(&totalSize)
if err != nil { if err != nil {
return 0, fmt.Errorf("querying uncompressed chunk size: %w", err) return 0, fmt.Errorf("querying uncompressed chunk size: %w", err)
@@ -485,6 +514,7 @@ func (r *SnapshotRepository) GetSnapshotNewChunkSize(ctx context.Context, snapsh
` `
var totalSize int64 var totalSize int64
err := r.db.conn.QueryRowContext(ctx, query, snapshotID, snapshotID, snapshotID).Scan(&totalSize) err := r.db.conn.QueryRowContext(ctx, query, snapshotID, snapshotID, snapshotID).Scan(&totalSize)
if err != nil { if err != nil {
return 0, fmt.Errorf("querying new chunk size: %w", err) return 0, fmt.Errorf("querying new chunk size: %w", err)
@@ -509,10 +539,13 @@ func (r *SnapshotRepository) GetIncompleteSnapshots(ctx context.Context) ([]*Sna
defer CloseRows(rows) defer CloseRows(rows)
var snapshots []*Snapshot var snapshots []*Snapshot
for rows.Next() { for rows.Next() {
var snapshot Snapshot var (
var startedAtUnix int64 snapshot Snapshot
var completedAtUnix *int64 startedAtUnix int64
completedAtUnix *int64
)
err := rows.Scan( err := rows.Scan(
&snapshot.ID, &snapshot.ID,
@@ -560,10 +593,13 @@ func (r *SnapshotRepository) GetIncompleteByHostname(ctx context.Context, hostna
defer CloseRows(rows) defer CloseRows(rows)
var snapshots []*Snapshot var snapshots []*Snapshot
for rows.Next() { for rows.Next() {
var snapshot Snapshot var (
var startedAtUnix int64 snapshot Snapshot
var completedAtUnix *int64 startedAtUnix int64
completedAtUnix *int64
)
err := rows.Scan( err := rows.Scan(
&snapshot.ID, &snapshot.ID,

View File

@@ -52,15 +52,19 @@ func TestSnapshotRepository(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("failed to get snapshot: %v", err) t.Fatalf("failed to get snapshot: %v", err)
} }
if retrieved == nil { if retrieved == nil {
t.Fatal("expected snapshot, got nil") t.Fatal("expected snapshot, got nil")
} }
if retrieved.ID != snapshot.ID { if retrieved.ID != snapshot.ID {
t.Errorf("ID mismatch: got %s, want %s", retrieved.ID, snapshot.ID) t.Errorf("ID mismatch: got %s, want %s", retrieved.ID, snapshot.ID)
} }
if retrieved.Hostname != snapshot.Hostname { if retrieved.Hostname != snapshot.Hostname {
t.Errorf("hostname mismatch: got %s, want %s", retrieved.Hostname, snapshot.Hostname) t.Errorf("hostname mismatch: got %s, want %s", retrieved.Hostname, snapshot.Hostname)
} }
if retrieved.FileCount != snapshot.FileCount { if retrieved.FileCount != snapshot.FileCount {
t.Errorf("file count mismatch: got %d, want %d", retrieved.FileCount, snapshot.FileCount) t.Errorf("file count mismatch: got %d, want %d", retrieved.FileCount, snapshot.FileCount)
} }
@@ -75,21 +79,27 @@ func TestSnapshotRepository(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("failed to get updated snapshot: %v", err) t.Fatalf("failed to get updated snapshot: %v", err)
} }
if retrieved.FileCount != 200 { if retrieved.FileCount != 200 {
t.Errorf("file count not updated: got %d, want %d", retrieved.FileCount, 200) t.Errorf("file count not updated: got %d, want %d", retrieved.FileCount, 200)
} }
if retrieved.ChunkCount != 1000 { if retrieved.ChunkCount != 1000 {
t.Errorf("chunk count not updated: got %d, want %d", retrieved.ChunkCount, 1000) t.Errorf("chunk count not updated: got %d, want %d", retrieved.ChunkCount, 1000)
} }
if retrieved.BlobCount != 20 { if retrieved.BlobCount != 20 {
t.Errorf("blob count not updated: got %d, want %d", retrieved.BlobCount, 20) t.Errorf("blob count not updated: got %d, want %d", retrieved.BlobCount, 20)
} }
if retrieved.TotalSize != twoHundredMebibytes { if retrieved.TotalSize != twoHundredMebibytes {
t.Errorf("total size not updated: got %d, want %d", retrieved.TotalSize, twoHundredMebibytes) t.Errorf("total size not updated: got %d, want %d", retrieved.TotalSize, twoHundredMebibytes)
} }
if retrieved.BlobSize != sixtyMebibytes { if retrieved.BlobSize != sixtyMebibytes {
t.Errorf("blob size not updated: got %d, want %d", retrieved.BlobSize, sixtyMebibytes) t.Errorf("blob size not updated: got %d, want %d", retrieved.BlobSize, sixtyMebibytes)
} }
expectedRatio := compressionRatioPoint3 // 0.3 expectedRatio := compressionRatioPoint3 // 0.3
if math.Abs(retrieved.CompressionRatio-expectedRatio) > 0.001 { if math.Abs(retrieved.CompressionRatio-expectedRatio) > 0.001 {
t.Errorf("compression ratio not updated: got %f, want %f", retrieved.CompressionRatio, expectedRatio) t.Errorf("compression ratio not updated: got %f, want %f", retrieved.CompressionRatio, expectedRatio)
@@ -108,6 +118,7 @@ func TestSnapshotRepository(t *testing.T) {
ChunkCount: int64(500 * i), ChunkCount: int64(500 * i),
BlobCount: int64(10 * i), BlobCount: int64(10 * i),
} }
err := repo.Create(ctx, nil, s) err := repo.Create(ctx, nil, s)
if err != nil { if err != nil {
t.Fatalf("failed to create snapshot %d: %v", i, err) t.Fatalf("failed to create snapshot %d: %v", i, err)
@@ -119,12 +130,13 @@ func TestSnapshotRepository(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("failed to list recent snapshots: %v", err) t.Fatalf("failed to list recent snapshots: %v", err)
} }
if len(recent) != 3 { if len(recent) != 3 {
t.Errorf("expected 3 recent snapshots, got %d", len(recent)) t.Errorf("expected 3 recent snapshots, got %d", len(recent))
} }
// Verify order (most recent first) // Verify order (most recent first)
for i := 0; i < len(recent)-1; i++ { for i := range len(recent) - 1 {
if recent[i].StartedAt.Before(recent[i+1].StartedAt) { if recent[i].StartedAt.Before(recent[i+1].StartedAt) {
t.Error("snapshots not in descending order") t.Error("snapshots not in descending order")
} }
@@ -143,6 +155,7 @@ func TestSnapshotRepositoryNotFound(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("unexpected error: %v", err) t.Fatalf("unexpected error: %v", err)
} }
if snapshot != nil { if snapshot != nil {
t.Error("expected nil for non-existent snapshot") t.Error("expected nil for non-existent snapshot")
} }

View File

@@ -53,6 +53,7 @@ func (r *UploadRepository) GetByBlobHash(ctx context.Context, blobHash string) (
` `
var upload Upload var upload Upload
err := r.conn.QueryRowContext(ctx, query, blobHash).Scan( err := r.conn.QueryRowContext(ctx, query, blobHash).Scan(
&upload.BlobHash, &upload.BlobHash,
&upload.UploadedAt, &upload.UploadedAt,
@@ -63,6 +64,7 @@ func (r *UploadRepository) GetByBlobHash(ctx context.Context, blobHash string) (
if err == sql.ErrNoRows { if err == sql.ErrNoRows {
return nil, nil return nil, nil
} }
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -84,17 +86,22 @@ func (r *UploadRepository) GetRecentUploads(ctx context.Context, limit int) ([]*
return nil, err return nil, err
} }
defer func() { defer func() {
if err := rows.Close(); err != nil { err := rows.Close()
if err != nil {
log.Error("failed to close rows", "error", err) log.Error("failed to close rows", "error", err)
} }
}() }()
var uploads []*Upload var uploads []*Upload
for rows.Next() { for rows.Next() {
var upload Upload var upload Upload
if err := rows.Scan(&upload.BlobHash, &upload.UploadedAt, &upload.Size, &upload.DurationMs); err != nil {
err := rows.Scan(&upload.BlobHash, &upload.UploadedAt, &upload.Size, &upload.DurationMs)
if err != nil {
return nil, err return nil, err
} }
uploads = append(uploads, &upload) uploads = append(uploads, &upload)
} }
@@ -115,6 +122,7 @@ func (r *UploadRepository) GetUploadStats(ctx context.Context, since time.Time)
` `
var stats UploadStats var stats UploadStats
err := r.conn.QueryRowContext(ctx, query, since).Scan( err := r.conn.QueryRowContext(ctx, query, since).Scan(
&stats.Count, &stats.Count,
&stats.TotalSize, &stats.TotalSize,
@@ -138,10 +146,13 @@ type UploadStats struct {
// GetCountBySnapshot returns the count of uploads for a specific snapshot // GetCountBySnapshot returns the count of uploads for a specific snapshot
func (r *UploadRepository) GetCountBySnapshot(ctx context.Context, snapshotID string) (int64, error) { func (r *UploadRepository) GetCountBySnapshot(ctx context.Context, snapshotID string) (int64, error) {
query := `SELECT COUNT(*) FROM uploads WHERE snapshot_id = ?` query := `SELECT COUNT(*) FROM uploads WHERE snapshot_id = ?`
var count int64 var count int64
err := r.conn.QueryRowContext(ctx, query, snapshotID).Scan(&count) err := r.conn.QueryRowContext(ctx, query, snapshotID).Scan(&count)
if err != nil { if err != nil {
return 0, err return 0, err
} }
return count, nil return count, nil
} }

View File

@@ -50,5 +50,6 @@ func (g *Globals) ShortCommit() string {
if len(g.Commit) > 12 { if len(g.Commit) > 12 {
return g.Commit[:12] return g.Commit[:12]
} }
return g.Commit return g.Commit
} }

View File

@@ -84,6 +84,7 @@ func getCaller(skip int) string {
if !ok { if !ok {
return "unknown" return "unknown"
} }
return fmt.Sprintf("%s:%d", filepath.Base(file), line) return fmt.Sprintf("%s:%d", filepath.Base(file), line)
} }
@@ -94,6 +95,7 @@ func Fatal(msg string, args ...any) {
args = append(args, "caller", getCaller(2)) args = append(args, "caller", getCaller(2))
logger.Error(msg, args...) logger.Error(msg, args...)
} }
os.Exit(1) os.Exit(1)
} }
@@ -172,6 +174,7 @@ func With(args ...any) *slog.Logger {
if logger != nil { if logger != nil {
return logger.With(args...) return logger.With(args...)
} }
return slog.Default() return slog.Default()
} }

View File

@@ -33,6 +33,7 @@ func NewTTYHandler(out io.Writer, opts *slog.HandlerOptions) *TTYHandler {
if opts == nil { if opts == nil {
opts = &slog.HandlerOptions{} opts = &slog.HandlerOptions{}
} }
return &TTYHandler{ return &TTYHandler{
out: out, out: out,
opts: *opts, opts: *opts,
@@ -54,7 +55,9 @@ func (h *TTYHandler) Handle(_ context.Context, r slog.Record) error {
// Level and color // Level and color
level := r.Level.String() level := r.Level.String()
var levelColor string var levelColor string
switch r.Level { switch r.Level {
case slog.LevelDebug: case slog.LevelDebug:
levelColor = colorGray levelColor = colorGray
@@ -96,10 +99,12 @@ func (h *TTYHandler) Handle(_ context.Context, r slog.Record) error {
_, _ = fmt.Fprintf(h.out, " %s%s%s=%s%s%s", _, _ = fmt.Fprintf(h.out, " %s%s%s=%s%s%s",
colorCyan, a.Key, colorReset, colorCyan, a.Key, colorReset,
colorBlue, value, colorReset) colorBlue, value, colorReset)
return true return true
}) })
_, _ = fmt.Fprintln(h.out) _, _ = fmt.Fprintln(h.out)
return nil return nil
} }
@@ -122,6 +127,7 @@ func formatDuration(d time.Duration) string {
} else if d < time.Minute { } else if d < time.Minute {
return fmt.Sprintf("%.1fs", d.Seconds()) return fmt.Sprintf("%.1fs", d.Seconds())
} }
return d.String() return d.String()
} }
@@ -131,10 +137,12 @@ func formatBytes(b int64) string {
if b < unit { if b < unit {
return fmt.Sprintf("%d B", b) return fmt.Sprintf("%d B", b)
} }
div, exp := int64(unit), 0 div, exp := int64(unit), 0
for n := b / unit; n >= unit; n /= unit { for n := b / unit; n >= unit; n /= unit {
div *= unit div *= unit
exp++ exp++
} }
return fmt.Sprintf("%.1f %cB", float64(b)/float64(div), "KMGTPE"[exp]) return fmt.Sprintf("%.1f %cB", float64(b)/float64(div), "KMGTPE"[exp])
} }

View File

@@ -77,6 +77,7 @@ func (l *Lock) Release() error {
} }
l.path = "" // Prevent double-release l.path = "" // Prevent double-release
return nil return nil
} }
@@ -104,5 +105,6 @@ func isProcessRunning(pid int) bool {
// On Unix, FindProcess always succeeds. We need to send signal 0 to check. // On Unix, FindProcess always succeeds. We need to send signal 0 to check.
err = process.Signal(syscall.Signal(0)) err = process.Signal(syscall.Signal(0))
return err == nil return err == nil
} }

View File

@@ -40,6 +40,7 @@ func TestAcquireBlocksSecondInstance(t *testing.T) {
// Acquire first lock // Acquire first lock
lock1, err := Acquire(tmpDir) lock1, err := Acquire(tmpDir)
require.NoError(t, err) require.NoError(t, err)
require.NotNil(t, lock1) require.NotNil(t, lock1)
defer func() { _ = lock1.Release() }() defer func() { _ = lock1.Release() }()
@@ -61,6 +62,7 @@ func TestAcquireWithStaleLock(t *testing.T) {
// Should be able to acquire lock (stale lock is cleaned up) // Should be able to acquire lock (stale lock is cleaned up)
lock, err := Acquire(tmpDir) lock, err := Acquire(tmpDir)
require.NoError(t, err) require.NoError(t, err)
require.NotNil(t, lock) require.NotNil(t, lock)
defer func() { _ = lock.Release() }() defer func() { _ = lock.Release() }()
@@ -88,6 +90,7 @@ func TestReleaseIsIdempotent(t *testing.T) {
func TestReleaseNilLock(t *testing.T) { func TestReleaseNilLock(t *testing.T) {
var lock *Lock var lock *Lock
err := lock.Release() err := lock.Release()
assert.NoError(t, err) assert.NoError(t, err)
} }
@@ -98,6 +101,7 @@ func TestAcquireCreatesDirectory(t *testing.T) {
lock, err := Acquire(nestedDir) lock, err := Acquire(nestedDir)
require.NoError(t, err) require.NoError(t, err)
require.NotNil(t, lock) require.NotNil(t, lock)
defer func() { _ = lock.Release() }() defer func() { _ = lock.Release() }()

View File

@@ -42,7 +42,7 @@ type Config struct {
// Used to suppress SDK warnings about checksums. // Used to suppress SDK warnings about checksums.
type nopLogger struct{} type nopLogger struct{}
func (nopLogger) Logf(classification logging.Classification, format string, v ...interface{}) {} func (nopLogger) Logf(classification logging.Classification, format string, v ...any) {}
// NewClient creates a new S3 client with the provided configuration. // NewClient creates a new S3 client with the provided configuration.
// It establishes a connection to the S3-compatible storage service and // It establishes a connection to the S3-compatible storage service and
@@ -92,6 +92,7 @@ func (c *Client) PutObject(ctx context.Context, key string, data io.Reader) erro
Key: aws.String(fullKey), Key: aws.String(fullKey),
Body: data, Body: data,
}) })
return err return err
} }
@@ -137,6 +138,7 @@ func (c *Client) PutObjectWithProgress(ctx context.Context, key string, data io.
// close the returned reader when done to avoid resource leaks. // close the returned reader when done to avoid resource leaks.
func (c *Client) GetObject(ctx context.Context, key string) (io.ReadCloser, error) { func (c *Client) GetObject(ctx context.Context, key string) (io.ReadCloser, error) {
fullKey := c.prefix + key fullKey := c.prefix + key
result, err := c.s3Client.GetObject(ctx, &s3.GetObjectInput{ result, err := c.s3Client.GetObject(ctx, &s3.GetObjectInput{
Bucket: aws.String(c.bucket), Bucket: aws.String(c.bucket),
Key: aws.String(fullKey), Key: aws.String(fullKey),
@@ -144,6 +146,7 @@ func (c *Client) GetObject(ctx context.Context, key string) (io.ReadCloser, erro
if err != nil { if err != nil {
return nil, err return nil, err
} }
return result.Body, nil return result.Body, nil
} }
@@ -156,6 +159,7 @@ func (c *Client) DeleteObject(ctx context.Context, key string) error {
Bucket: aws.String(c.bucket), Bucket: aws.String(c.bucket),
Key: aws.String(fullKey), Key: aws.String(fullKey),
}) })
return err return err
} }
@@ -168,6 +172,7 @@ func (c *Client) ListObjects(ctx context.Context, prefix string) ([]string, erro
fullPrefix := c.prefix + prefix fullPrefix := c.prefix + prefix
var keys []string var keys []string
paginator := s3.NewListObjectsV2Paginator(c.s3Client, &s3.ListObjectsV2Input{ paginator := s3.NewListObjectsV2Paginator(c.s3Client, &s3.ListObjectsV2Input{
Bucket: aws.String(c.bucket), Bucket: aws.String(c.bucket),
Prefix: aws.String(fullPrefix), Prefix: aws.String(fullPrefix),
@@ -186,6 +191,7 @@ func (c *Client) ListObjects(ctx context.Context, prefix string) ([]string, erro
if len(key) > len(c.prefix) { if len(key) > len(c.prefix) {
key = key[len(c.prefix):] key = key[len(c.prefix):]
} }
keys = append(keys, key) keys = append(keys, key)
} }
} }
@@ -200,18 +206,23 @@ func (c *Client) ListObjects(ctx context.Context, prefix string) ([]string, erro
// Note: This method returns false for any error, not just "not found". // Note: This method returns false for any error, not just "not found".
func (c *Client) HeadObject(ctx context.Context, key string) (bool, error) { func (c *Client) HeadObject(ctx context.Context, key string) (bool, error) {
fullKey := c.prefix + key fullKey := c.prefix + key
_, err := c.s3Client.HeadObject(ctx, &s3.HeadObjectInput{ _, err := c.s3Client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: aws.String(c.bucket), Bucket: aws.String(c.bucket),
Key: aws.String(fullKey), Key: aws.String(fullKey),
}) })
if err != nil { if err != nil {
var notFound *s3types.NotFound var (
var noSuchKey *s3types.NoSuchKey notFound *s3types.NotFound
noSuchKey *s3types.NoSuchKey
)
if errors.As(err, &notFound) || errors.As(err, &noSuchKey) { if errors.As(err, &notFound) || errors.As(err, &noSuchKey) {
return false, nil return false, nil
} }
return false, err return false, err
} }
return true, nil return true, nil
} }
@@ -247,6 +258,7 @@ func (c *Client) ListObjectsStream(ctx context.Context, prefix string, recursive
page, err := paginator.NextPage(ctx) page, err := paginator.NextPage(ctx)
if err != nil { if err != nil {
ch <- ObjectInfo{Err: err} ch <- ObjectInfo{Err: err}
return return
} }
@@ -257,6 +269,7 @@ func (c *Client) ListObjectsStream(ctx context.Context, prefix string, recursive
if len(key) > len(c.prefix) { if len(key) > len(c.prefix) {
key = key[len(c.prefix):] key = key[len(c.prefix):]
} }
ch <- ObjectInfo{ ch <- ObjectInfo{
Key: key, Key: key,
Size: *obj.Size, Size: *obj.Size,
@@ -275,6 +288,7 @@ func (c *Client) ListObjectsStream(ctx context.Context, prefix string, recursive
// Returns an error if the object doesn't exist or if the operation fails. // Returns an error if the object doesn't exist or if the operation fails.
func (c *Client) StatObject(ctx context.Context, key string) (*ObjectInfo, error) { func (c *Client) StatObject(ctx context.Context, key string) (*ObjectInfo, error) {
fullKey := c.prefix + key fullKey := c.prefix + key
result, err := c.s3Client.HeadObject(ctx, &s3.HeadObjectInput{ result, err := c.s3Client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: aws.String(c.bucket), Bucket: aws.String(c.bucket),
Key: aws.String(fullKey), Key: aws.String(fullKey),
@@ -313,6 +327,7 @@ func (c *Client) Endpoint() string {
if c.endpoint == "" { if c.endpoint == "" {
return "s3.amazonaws.com" return "s3.amazonaws.com"
} }
return c.endpoint return c.endpoint
} }
@@ -329,11 +344,14 @@ func (pr *progressReader) Read(p []byte) (int, error) {
n, err := pr.reader.Read(p) n, err := pr.reader.Read(p)
if n > 0 { if n > 0 {
atomic.AddInt64(&pr.read, int64(n)) atomic.AddInt64(&pr.read, int64(n))
if pr.callback != nil { if pr.callback != nil {
if callbackErr := pr.callback(atomic.LoadInt64(&pr.read)); callbackErr != nil { callbackErr := pr.callback(atomic.LoadInt64(&pr.read))
if callbackErr != nil {
return n, callbackErr return n, callbackErr
} }
} }
} }
return n, err return n, err
} }

View File

@@ -12,7 +12,8 @@ import (
func TestClient(t *testing.T) { func TestClient(t *testing.T) {
ts := NewTestServer(t) ts := NewTestServer(t)
defer func() { defer func() {
if err := ts.Cleanup(); err != nil { err := ts.Cleanup()
if err != nil {
t.Errorf("cleanup failed: %v", err) t.Errorf("cleanup failed: %v", err)
} }
}() }()
@@ -35,6 +36,7 @@ func TestClient(t *testing.T) {
// Test PutObject // Test PutObject
testKey := "foo/bar.txt" testKey := "foo/bar.txt"
testData := []byte("test data") testData := []byte("test data")
err = client.PutObject(ctx, testKey, bytes.NewReader(testData)) err = client.PutObject(ctx, testKey, bytes.NewReader(testData))
if err != nil { if err != nil {
t.Fatalf("failed to put object: %v", err) t.Fatalf("failed to put object: %v", err)
@@ -46,7 +48,8 @@ func TestClient(t *testing.T) {
t.Fatalf("failed to get object: %v", err) t.Fatalf("failed to get object: %v", err)
} }
defer func() { defer func() {
if err := reader.Close(); err != nil { err := reader.Close()
if err != nil {
t.Errorf("failed to close reader: %v", err) t.Errorf("failed to close reader: %v", err)
} }
}() }()
@@ -65,6 +68,7 @@ func TestClient(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("failed to head object: %v", err) t.Fatalf("failed to head object: %v", err)
} }
if !exists { if !exists {
t.Error("expected object to exist") t.Error("expected object to exist")
} }
@@ -74,9 +78,11 @@ func TestClient(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("failed to list objects: %v", err) t.Fatalf("failed to list objects: %v", err)
} }
if len(keys) != 1 { if len(keys) != 1 {
t.Errorf("expected 1 key, got %d", len(keys)) t.Errorf("expected 1 key, got %d", len(keys))
} }
if keys[0] != testKey { if keys[0] != testKey {
t.Errorf("unexpected key: got %s, want %s", keys[0], testKey) t.Errorf("unexpected key: got %s, want %s", keys[0], testKey)
} }
@@ -92,6 +98,7 @@ func TestClient(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("failed to head object after deletion: %v", err) t.Fatalf("failed to head object after deletion: %v", err)
} }
if exists { if exists {
t.Error("expected object to not exist after deletion") t.Error("expected object to not exist after deletion")
} }

View File

@@ -57,7 +57,8 @@ func NewTestServer(t *testing.T) *TestServer {
// Start server in background // Start server in background
go func() { go func() {
if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed { err := server.ListenAndServe()
if err != nil && err != http.ErrServerClosed {
t.Logf("test server error: %v", err) t.Logf("test server error: %v", err)
} }
}() }()
@@ -77,7 +78,7 @@ func NewTestServer(t *testing.T) *TestServer {
"", "",
)), )),
config.WithClientLogMode(aws.LogRetries|aws.LogRequestWithBody|aws.LogResponseWithBody), config.WithClientLogMode(aws.LogRetries|aws.LogRequestWithBody|aws.LogResponseWithBody),
config.WithLogger(logging.LoggerFunc(func(classification logging.Classification, format string, v ...interface{}) { config.WithLogger(logging.LoggerFunc(func(classification logging.Classification, format string, v ...any) {
// Capture logs to buffer instead of stdout // Capture logs to buffer instead of stdout
fmt.Fprintf(logBuf, "SDK %s %s %s\n", fmt.Fprintf(logBuf, "SDK %s %s %s\n",
time.Now().Format("2006/01/02 15:04:05"), time.Now().Format("2006/01/02 15:04:05"),
@@ -125,7 +126,8 @@ func (ts *TestServer) Cleanup() error {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel() defer cancel()
if err := ts.server.Shutdown(ctx); err != nil { err := ts.server.Shutdown(ctx)
if err != nil {
return err return err
} }
@@ -141,7 +143,8 @@ func (ts *TestServer) Client() *s3.Client {
func TestBasicS3Operations(t *testing.T) { func TestBasicS3Operations(t *testing.T) {
ts := NewTestServer(t) ts := NewTestServer(t)
defer func() { defer func() {
if err := ts.Cleanup(); err != nil { err := ts.Cleanup()
if err != nil {
t.Errorf("cleanup failed: %v", err) t.Errorf("cleanup failed: %v", err)
} }
}() }()
@@ -172,7 +175,8 @@ func TestBasicS3Operations(t *testing.T) {
t.Fatalf("failed to get object: %v", err) t.Fatalf("failed to get object: %v", err)
} }
defer func() { defer func() {
if err := result.Body.Close(); err != nil { err := result.Body.Close()
if err != nil {
t.Errorf("failed to close body: %v", err) t.Errorf("failed to close body: %v", err)
} }
}() }()
@@ -192,7 +196,8 @@ func TestBasicS3Operations(t *testing.T) {
func TestBlobOperations(t *testing.T) { func TestBlobOperations(t *testing.T) {
ts := NewTestServer(t) ts := NewTestServer(t)
defer func() { defer func() {
if err := ts.Cleanup(); err != nil { err := ts.Cleanup()
if err != nil {
t.Errorf("cleanup failed: %v", err) t.Errorf("cleanup failed: %v", err)
} }
}() }()
@@ -255,7 +260,8 @@ func TestBlobOperations(t *testing.T) {
func TestMetadataOperations(t *testing.T) { func TestMetadataOperations(t *testing.T) {
ts := NewTestServer(t) ts := NewTestServer(t)
defer func() { defer func() {
if err := ts.Cleanup(); err != nil { err := ts.Cleanup()
if err != nil {
t.Errorf("cleanup failed: %v", err) t.Errorf("cleanup failed: %v", err)
} }
}() }()

View File

@@ -4,6 +4,8 @@ import (
"context" "context"
"crypto/sha256" "crypto/sha256"
"database/sql" "database/sql"
"encoding/hex"
"errors"
"fmt" "fmt"
"io" "io"
"io/fs" "io/fs"
@@ -30,6 +32,7 @@ func NewMockS3Client() *MockS3Client {
func (m *MockS3Client) PutBlob(ctx context.Context, hash string, data []byte) error { func (m *MockS3Client) PutBlob(ctx context.Context, hash string, data []byte) error {
m.storage[hash] = data m.storage[hash] = data
return nil return nil
} }
@@ -38,11 +41,13 @@ func (m *MockS3Client) GetBlob(ctx context.Context, hash string) ([]byte, error)
if !ok { if !ok {
return nil, fmt.Errorf("blob not found: %s", hash) return nil, fmt.Errorf("blob not found: %s", hash)
} }
return data, nil return data, nil
} }
func (m *MockS3Client) BlobExists(ctx context.Context, hash string) (bool, error) { func (m *MockS3Client) BlobExists(ctx context.Context, hash string) (bool, error) {
_, ok := m.storage[hash] _, ok := m.storage[hash]
return ok, nil return ok, nil
} }
@@ -81,12 +86,15 @@ func TestBackupWithInMemoryFS(t *testing.T) {
// Initialize the database // Initialize the database
ctx := context.Background() ctx := context.Background()
db, err := database.New(ctx, dbPath) db, err := database.New(ctx, dbPath)
if err != nil { if err != nil {
t.Fatalf("Failed to create database: %v", err) t.Fatalf("Failed to create database: %v", err)
} }
defer func() { defer func() {
if err := db.Close(); err != nil { err := db.Close()
if err != nil {
t.Logf("Failed to close database: %v", err) t.Logf("Failed to close database: %v", err)
} }
}() }()
@@ -142,12 +150,14 @@ func TestBackupWithInMemoryFS(t *testing.T) {
if !expectedFiles[file.Path.String()] { if !expectedFiles[file.Path.String()] {
t.Errorf("Unexpected file in database: %s", file.Path) t.Errorf("Unexpected file in database: %s", file.Path)
} }
delete(expectedFiles, file.Path.String()) delete(expectedFiles, file.Path.String())
// Verify file metadata // Verify file metadata
fsFile := testFS[file.Path.String()] fsFile := testFS[file.Path.String()]
if fsFile == nil { if fsFile == nil {
t.Errorf("File %s not found in test filesystem", file.Path) t.Errorf("File %s not found in test filesystem", file.Path)
continue continue
} }
@@ -187,6 +197,7 @@ func TestBackupWithInMemoryFS(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("Failed to get blob hashes: %v", err) t.Fatalf("Failed to get blob hashes: %v", err)
} }
if len(blobHashes) == 0 { if len(blobHashes) == 0 {
t.Error("Expected at least one blob to be created") t.Error("Expected at least one blob to be created")
} }
@@ -197,6 +208,7 @@ func TestBackupWithInMemoryFS(t *testing.T) {
if err != nil { if err != nil {
t.Errorf("Failed to check blob %s: %v", blobHash, err) t.Errorf("Failed to check blob %s: %v", blobHash, err)
} }
if !exists { if !exists {
t.Errorf("Blob %s not found in S3", blobHash) t.Errorf("Blob %s not found in S3", blobHash)
} }
@@ -229,12 +241,15 @@ func TestBackupDeduplication(t *testing.T) {
// Initialize the database // Initialize the database
ctx := context.Background() ctx := context.Background()
db, err := database.New(ctx, dbPath) db, err := database.New(ctx, dbPath)
if err != nil { if err != nil {
t.Fatalf("Failed to create database: %v", err) t.Fatalf("Failed to create database: %v", err)
} }
defer func() { defer func() {
if err := db.Close(); err != nil { err := db.Close()
if err != nil {
t.Logf("Failed to close database: %v", err) t.Logf("Failed to close database: %v", err)
} }
}() }()
@@ -348,6 +363,7 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
UID: 1000, // Default UID for test UID: 1000, // Default UID for test
GID: 1000, // Default GID for test GID: 1000, // Default GID for test
} }
err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
return b.repos.Files.Create(ctx, tx, file) return b.repos.Files.Create(ctx, tx, file)
}) })
@@ -364,7 +380,8 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
return err return err
} }
defer func() { defer func() {
if err := f.Close(); err != nil { err := f.Close()
if err != nil {
// Log but don't fail since we're already in an error path potentially // Log but don't fail since we're already in an error path potentially
fmt.Fprintf(os.Stderr, "Failed to close file: %v\n", err) fmt.Fprintf(os.Stderr, "Failed to close file: %v\n", err)
} }
@@ -376,9 +393,10 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
for { for {
n, err := f.Read(buffer) n, err := f.Read(buffer)
if err != nil && err != io.EOF { if err != nil && !errors.Is(err, io.EOF) {
return err return err
} }
if n == 0 { if n == 0 {
break break
} }
@@ -395,11 +413,13 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
ChunkHash: types.ChunkHash(chunkHash), ChunkHash: types.ChunkHash(chunkHash),
Size: int64(n), Size: int64(n),
} }
return b.repos.Chunks.Create(ctx, tx, chunk) return b.repos.Chunks.Create(ctx, tx, chunk)
}) })
if err != nil { if err != nil {
return err return err
} }
processedChunks[chunkHash] = true processedChunks[chunkHash] = true
} }
@@ -410,6 +430,7 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
Idx: chunkIndex, Idx: chunkIndex,
ChunkHash: types.ChunkHash(chunkHash), ChunkHash: types.ChunkHash(chunkHash),
} }
return b.repos.FileChunks.Create(ctx, tx, fileChunk) return b.repos.FileChunks.Create(ctx, tx, fileChunk)
}) })
if err != nil { if err != nil {
@@ -424,6 +445,7 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
FileOffset: int64(chunkIndex * defaultChunkSize), FileOffset: int64(chunkIndex * defaultChunkSize),
Length: int64(n), Length: int64(n),
} }
return b.repos.ChunkFiles.Create(ctx, tx, chunkFile) return b.repos.ChunkFiles.Create(ctx, tx, chunkFile)
}) })
if err != nil { if err != nil {
@@ -435,7 +457,6 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
return nil return nil
}) })
if err != nil { if err != nil {
return "", err return "", err
} }
@@ -464,12 +485,14 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
// Create blob entry in a short transaction // Create blob entry in a short transaction
blobID := types.NewBlobID() blobID := types.NewBlobID()
err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
blob := &database.Blob{ blob := &database.Blob{
ID: blobID, ID: blobID,
Hash: types.BlobHash(blobHash), Hash: types.BlobHash(blobHash),
CreatedTS: time.Now(), CreatedTS: time.Now(),
} }
return b.repos.Blobs.Create(ctx, tx, blob) return b.repos.Blobs.Create(ctx, tx, blob)
}) })
if err != nil { if err != nil {
@@ -487,6 +510,7 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
Offset: 0, Offset: 0,
Length: chunk.Size, Length: chunk.Size,
} }
return b.repos.BlobChunks.Create(ctx, tx, blobChunk) return b.repos.BlobChunks.Create(ctx, tx, blobChunk)
}) })
if err != nil { if err != nil {
@@ -506,7 +530,6 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
return b.repos.Snapshots.UpdateCounts(ctx, tx, snapshotID, fileCount, chunkCount, blobCount, totalSize, blobSize) return b.repos.Snapshots.UpdateCounts(ctx, tx, snapshotID, fileCount, chunkCount, blobCount, totalSize, blobSize)
}) })
if err != nil { if err != nil {
return "", err return "", err
} }
@@ -517,16 +540,18 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
func calculateHash(data []byte) string { func calculateHash(data []byte) string {
h := sha256.New() h := sha256.New()
h.Write(data) h.Write(data)
return fmt.Sprintf("%x", h.Sum(nil))
return hex.EncodeToString(h.Sum(nil))
} }
func generateLargeFileContent(size int) []byte { func generateLargeFileContent(size int) []byte {
data := make([]byte, size) data := make([]byte, size)
// Fill with pattern that changes every chunk to avoid deduplication // Fill with pattern that changes every chunk to avoid deduplication
for i := 0; i < size; i++ { for i := range size {
chunkNum := i / defaultChunkSize chunkNum := i / defaultChunkSize
data[i] = byte((i + chunkNum) % 256) data[i] = byte((i + chunkNum) % 256)
} }
return data return data
} }

View File

@@ -63,6 +63,7 @@ func setupExcludeTestFS(t *testing.T) afero.Fs {
} }
testTime := time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC) testTime := time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC)
for path, content := range files { for path, content := range files {
dir := filepath.Dir(path) dir := filepath.Dir(path)
err := fs.MkdirAll(dir, 0755) err := fs.MkdirAll(dir, 0755)
@@ -107,6 +108,7 @@ func createTestScanner(t *testing.T, fs afero.Fs, excludePatterns []string) (*sn
func createSnapshotRecord(t *testing.T, ctx context.Context, repos *database.Repositories, snapshotID string) { func createSnapshotRecord(t *testing.T, ctx context.Context, repos *database.Repositories, snapshotID string) {
t.Helper() t.Helper()
err := repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { err := repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
snap := &database.Snapshot{ snap := &database.Snapshot{
ID: types.SnapshotID(snapshotID), ID: types.SnapshotID(snapshotID),
@@ -121,6 +123,7 @@ func createSnapshotRecord(t *testing.T, ctx context.Context, repos *database.Rep
BlobSize: 0, BlobSize: 0,
CompressionRatio: 1.0, CompressionRatio: 1.0,
} }
return repos.Snapshots.Create(ctx, tx, snap) return repos.Snapshots.Create(ctx, tx, snap)
}) })
require.NoError(t, err) require.NoError(t, err)
@@ -128,8 +131,10 @@ func createSnapshotRecord(t *testing.T, ctx context.Context, repos *database.Rep
func TestExcludePatterns_ExcludeGitDirectory(t *testing.T) { func TestExcludePatterns_ExcludeGitDirectory(t *testing.T) {
fs := setupExcludeTestFS(t) fs := setupExcludeTestFS(t)
scanner, repos, cleanup := createTestScanner(t, fs, []string{".git"}) scanner, repos, cleanup := createTestScanner(t, fs, []string{".git"})
defer cleanup() defer cleanup()
require.NotNil(t, scanner) require.NotNil(t, scanner)
ctx := context.Background() ctx := context.Background()
@@ -148,8 +153,10 @@ func TestExcludePatterns_ExcludeGitDirectory(t *testing.T) {
func TestExcludePatterns_ExcludeByExtension(t *testing.T) { func TestExcludePatterns_ExcludeByExtension(t *testing.T) {
fs := setupExcludeTestFS(t) fs := setupExcludeTestFS(t)
scanner, repos, cleanup := createTestScanner(t, fs, []string{"*.log"}) scanner, repos, cleanup := createTestScanner(t, fs, []string{"*.log"})
defer cleanup() defer cleanup()
require.NotNil(t, scanner) require.NotNil(t, scanner)
ctx := context.Background() ctx := context.Background()
@@ -165,8 +172,10 @@ func TestExcludePatterns_ExcludeByExtension(t *testing.T) {
func TestExcludePatterns_ExcludeNodeModules(t *testing.T) { func TestExcludePatterns_ExcludeNodeModules(t *testing.T) {
fs := setupExcludeTestFS(t) fs := setupExcludeTestFS(t)
scanner, repos, cleanup := createTestScanner(t, fs, []string{"node_modules"}) scanner, repos, cleanup := createTestScanner(t, fs, []string{"node_modules"})
defer cleanup() defer cleanup()
require.NotNil(t, scanner) require.NotNil(t, scanner)
ctx := context.Background() ctx := context.Background()
@@ -182,8 +191,10 @@ func TestExcludePatterns_ExcludeNodeModules(t *testing.T) {
func TestExcludePatterns_MultiplePatterns(t *testing.T) { func TestExcludePatterns_MultiplePatterns(t *testing.T) {
fs := setupExcludeTestFS(t) fs := setupExcludeTestFS(t)
scanner, repos, cleanup := createTestScanner(t, fs, []string{".git", "node_modules", "*.log", ".DS_Store", "thumbs.db", "cache", "build"}) scanner, repos, cleanup := createTestScanner(t, fs, []string{".git", "node_modules", "*.log", ".DS_Store", "thumbs.db", "cache", "build"})
defer cleanup() defer cleanup()
require.NotNil(t, scanner) require.NotNil(t, scanner)
ctx := context.Background() ctx := context.Background()
@@ -199,8 +210,10 @@ func TestExcludePatterns_MultiplePatterns(t *testing.T) {
func TestExcludePatterns_NoExclusions(t *testing.T) { func TestExcludePatterns_NoExclusions(t *testing.T) {
fs := setupExcludeTestFS(t) fs := setupExcludeTestFS(t)
scanner, repos, cleanup := createTestScanner(t, fs, []string{}) scanner, repos, cleanup := createTestScanner(t, fs, []string{})
defer cleanup() defer cleanup()
require.NotNil(t, scanner) require.NotNil(t, scanner)
ctx := context.Background() ctx := context.Background()
@@ -215,8 +228,10 @@ func TestExcludePatterns_NoExclusions(t *testing.T) {
func TestExcludePatterns_ExcludeHiddenFiles(t *testing.T) { func TestExcludePatterns_ExcludeHiddenFiles(t *testing.T) {
fs := setupExcludeTestFS(t) fs := setupExcludeTestFS(t)
scanner, repos, cleanup := createTestScanner(t, fs, []string{".*"}) scanner, repos, cleanup := createTestScanner(t, fs, []string{".*"})
defer cleanup() defer cleanup()
require.NotNil(t, scanner) require.NotNil(t, scanner)
ctx := context.Background() ctx := context.Background()
@@ -232,8 +247,10 @@ func TestExcludePatterns_ExcludeHiddenFiles(t *testing.T) {
func TestExcludePatterns_DoubleStarGlob(t *testing.T) { func TestExcludePatterns_DoubleStarGlob(t *testing.T) {
fs := setupExcludeTestFS(t) fs := setupExcludeTestFS(t)
scanner, repos, cleanup := createTestScanner(t, fs, []string{"**/*.pack"}) scanner, repos, cleanup := createTestScanner(t, fs, []string{"**/*.pack"})
defer cleanup() defer cleanup()
require.NotNil(t, scanner) require.NotNil(t, scanner)
ctx := context.Background() ctx := context.Background()
@@ -249,8 +266,10 @@ func TestExcludePatterns_DoubleStarGlob(t *testing.T) {
func TestExcludePatterns_ExactFileName(t *testing.T) { func TestExcludePatterns_ExactFileName(t *testing.T) {
fs := setupExcludeTestFS(t) fs := setupExcludeTestFS(t)
scanner, repos, cleanup := createTestScanner(t, fs, []string{"thumbs.db", ".DS_Store"}) scanner, repos, cleanup := createTestScanner(t, fs, []string{"thumbs.db", ".DS_Store"})
defer cleanup() defer cleanup()
require.NotNil(t, scanner) require.NotNil(t, scanner)
ctx := context.Background() ctx := context.Background()
@@ -267,8 +286,10 @@ func TestExcludePatterns_ExactFileName(t *testing.T) {
func TestExcludePatterns_CaseSensitive(t *testing.T) { func TestExcludePatterns_CaseSensitive(t *testing.T) {
// Pattern matching should be case-sensitive // Pattern matching should be case-sensitive
fs := setupExcludeTestFS(t) fs := setupExcludeTestFS(t)
scanner, repos, cleanup := createTestScanner(t, fs, []string{"THUMBS.DB"}) scanner, repos, cleanup := createTestScanner(t, fs, []string{"THUMBS.DB"})
defer cleanup() defer cleanup()
require.NotNil(t, scanner) require.NotNil(t, scanner)
ctx := context.Background() ctx := context.Background()
@@ -287,6 +308,7 @@ func TestExcludePatterns_DirectoryWithTrailingSlash(t *testing.T) {
// Some users might add trailing slashes to directory patterns // Some users might add trailing slashes to directory patterns
scanner, repos, cleanup := createTestScanner(t, fs, []string{"cache/", "build/"}) scanner, repos, cleanup := createTestScanner(t, fs, []string{"cache/", "build/"})
defer cleanup() defer cleanup()
require.NotNil(t, scanner) require.NotNil(t, scanner)
ctx := context.Background() ctx := context.Background()
@@ -305,6 +327,7 @@ func TestExcludePatterns_PatternInSubdirectory(t *testing.T) {
// Exclude .hidden file specifically in src directory // Exclude .hidden file specifically in src directory
scanner, repos, cleanup := createTestScanner(t, fs, []string{"src/.hidden"}) scanner, repos, cleanup := createTestScanner(t, fs, []string{"src/.hidden"})
defer cleanup() defer cleanup()
require.NotNil(t, scanner) require.NotNil(t, scanner)
ctx := context.Background() ctx := context.Background()
@@ -343,6 +366,7 @@ func setupAnchoredTestFS(t *testing.T) afero.Fs {
} }
testTime := time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC) testTime := time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC)
for path, content := range files { for path, content := range files {
dir := filepath.Dir(path) dir := filepath.Dir(path)
err := fs.MkdirAll(dir, 0755) err := fs.MkdirAll(dir, 0755)
@@ -359,8 +383,10 @@ func setupAnchoredTestFS(t *testing.T) afero.Fs {
func TestExcludePatterns_AnchoredPattern(t *testing.T) { func TestExcludePatterns_AnchoredPattern(t *testing.T) {
// Pattern starting with / should only match from root of source dir // Pattern starting with / should only match from root of source dir
fs := setupAnchoredTestFS(t) fs := setupAnchoredTestFS(t)
scanner, repos, cleanup := createTestScanner(t, fs, []string{"/projectname"}) scanner, repos, cleanup := createTestScanner(t, fs, []string{"/projectname"})
defer cleanup() defer cleanup()
require.NotNil(t, scanner) require.NotNil(t, scanner)
ctx := context.Background() ctx := context.Background()
@@ -378,8 +404,10 @@ func TestExcludePatterns_AnchoredPattern(t *testing.T) {
func TestExcludePatterns_UnanchoredPattern(t *testing.T) { func TestExcludePatterns_UnanchoredPattern(t *testing.T) {
// Pattern without leading / should match anywhere in path // Pattern without leading / should match anywhere in path
fs := setupAnchoredTestFS(t) fs := setupAnchoredTestFS(t)
scanner, repos, cleanup := createTestScanner(t, fs, []string{"projectname"}) scanner, repos, cleanup := createTestScanner(t, fs, []string{"projectname"})
defer cleanup() defer cleanup()
require.NotNil(t, scanner) require.NotNil(t, scanner)
ctx := context.Background() ctx := context.Background()
@@ -398,8 +426,10 @@ func TestExcludePatterns_UnanchoredPattern(t *testing.T) {
func TestExcludePatterns_AnchoredPatternWithGlob(t *testing.T) { func TestExcludePatterns_AnchoredPatternWithGlob(t *testing.T) {
// Anchored pattern with glob // Anchored pattern with glob
fs := setupAnchoredTestFS(t) fs := setupAnchoredTestFS(t)
scanner, repos, cleanup := createTestScanner(t, fs, []string{"/src/*.go"}) scanner, repos, cleanup := createTestScanner(t, fs, []string{"/src/*.go"})
defer cleanup() defer cleanup()
require.NotNil(t, scanner) require.NotNil(t, scanner)
ctx := context.Background() ctx := context.Background()
@@ -416,8 +446,10 @@ func TestExcludePatterns_AnchoredPatternWithGlob(t *testing.T) {
func TestExcludePatterns_AnchoredPatternFile(t *testing.T) { func TestExcludePatterns_AnchoredPatternFile(t *testing.T) {
// Anchored pattern for exact file at root // Anchored pattern for exact file at root
fs := setupAnchoredTestFS(t) fs := setupAnchoredTestFS(t)
scanner, repos, cleanup := createTestScanner(t, fs, []string{"/file.txt"}) scanner, repos, cleanup := createTestScanner(t, fs, []string{"/file.txt"})
defer cleanup() defer cleanup()
require.NotNil(t, scanner) require.NotNil(t, scanner)
ctx := context.Background() ctx := context.Background()
@@ -435,8 +467,10 @@ func TestExcludePatterns_AnchoredPatternFile(t *testing.T) {
func TestExcludePatterns_UnanchoredPatternFile(t *testing.T) { func TestExcludePatterns_UnanchoredPatternFile(t *testing.T) {
// Unanchored pattern for file should match anywhere // Unanchored pattern for file should match anywhere
fs := setupAnchoredTestFS(t) fs := setupAnchoredTestFS(t)
scanner, repos, cleanup := createTestScanner(t, fs, []string{"file.txt"}) scanner, repos, cleanup := createTestScanner(t, fs, []string{"file.txt"})
defer cleanup() defer cleanup()
require.NotNil(t, scanner) require.NotNil(t, scanner)
ctx := context.Background() ctx := context.Background()

View File

@@ -30,9 +30,11 @@ func TestFileContentChange(t *testing.T) {
// Create test database // Create test database
db, err := database.NewTestDB() db, err := database.NewTestDB()
require.NoError(t, err) require.NoError(t, err)
defer func() { defer func() {
if err := db.Close(); err != nil { err := db.Close()
if err != nil {
t.Errorf("failed to close database: %v", err) t.Errorf("failed to close database: %v", err)
} }
}() }()
@@ -59,6 +61,7 @@ func TestFileContentChange(t *testing.T) {
VaultikVersion: "test", VaultikVersion: "test",
StartedAt: time.Now(), StartedAt: time.Now(),
} }
return repos.Snapshots.Create(ctx, tx, snapshot) return repos.Snapshots.Create(ctx, tx, snapshot)
}) })
require.NoError(t, err) require.NoError(t, err)
@@ -81,6 +84,7 @@ func TestFileContentChange(t *testing.T) {
// Modify the file // Modify the file
time.Sleep(10 * time.Millisecond) // Ensure mtime changes time.Sleep(10 * time.Millisecond) // Ensure mtime changes
err = afero.WriteFile(fs, "/test.txt", []byte("Modified content with different data"), 0644) err = afero.WriteFile(fs, "/test.txt", []byte("Modified content with different data"), 0644)
require.NoError(t, err) require.NoError(t, err)
@@ -93,6 +97,7 @@ func TestFileContentChange(t *testing.T) {
VaultikVersion: "test", VaultikVersion: "test",
StartedAt: time.Now(), StartedAt: time.Now(),
} }
return repos.Snapshots.Create(ctx, tx, snapshot) return repos.Snapshots.Create(ctx, tx, snapshot)
}) })
require.NoError(t, err) require.NoError(t, err)
@@ -130,6 +135,7 @@ func TestFileContentChange(t *testing.T) {
// Verify that chunk_files for old chunk no longer references this file // Verify that chunk_files for old chunk no longer references this file
oldChunkFiles, err := repos.ChunkFiles.GetByChunkHash(ctx, oldChunkHash) oldChunkFiles, err := repos.ChunkFiles.GetByChunkHash(ctx, oldChunkHash)
require.NoError(t, err) require.NoError(t, err)
for _, cf := range oldChunkFiles { for _, cf := range oldChunkFiles {
file, err := repos.Files.GetByID(ctx, cf.FileID) file, err := repos.Files.GetByID(ctx, cf.FileID)
require.NoError(t, err) require.NoError(t, err)
@@ -159,9 +165,11 @@ func TestMultipleFileChanges(t *testing.T) {
// Create test database // Create test database
db, err := database.NewTestDB() db, err := database.NewTestDB()
require.NoError(t, err) require.NoError(t, err)
defer func() { defer func() {
if err := db.Close(); err != nil { err := db.Close()
if err != nil {
t.Errorf("failed to close database: %v", err) t.Errorf("failed to close database: %v", err)
} }
}() }()
@@ -188,6 +196,7 @@ func TestMultipleFileChanges(t *testing.T) {
VaultikVersion: "test", VaultikVersion: "test",
StartedAt: time.Now(), StartedAt: time.Now(),
} }
return repos.Snapshots.Create(ctx, tx, snapshot) return repos.Snapshots.Create(ctx, tx, snapshot)
}) })
require.NoError(t, err) require.NoError(t, err)
@@ -200,6 +209,7 @@ func TestMultipleFileChanges(t *testing.T) {
// Modify two files // Modify two files
time.Sleep(10 * time.Millisecond) // Ensure mtime changes time.Sleep(10 * time.Millisecond) // Ensure mtime changes
err = afero.WriteFile(fs, "/file1.txt", []byte("Modified content 1"), 0644) err = afero.WriteFile(fs, "/file1.txt", []byte("Modified content 1"), 0644)
require.NoError(t, err) require.NoError(t, err)
err = afero.WriteFile(fs, "/file3.txt", []byte("Modified content 3"), 0644) err = afero.WriteFile(fs, "/file3.txt", []byte("Modified content 3"), 0644)
@@ -214,6 +224,7 @@ func TestMultipleFileChanges(t *testing.T) {
VaultikVersion: "test", VaultikVersion: "test",
StartedAt: time.Now(), StartedAt: time.Now(),
} }
return repos.Snapshots.Create(ctx, tx, snapshot) return repos.Snapshots.Create(ctx, tx, snapshot)
}) })
require.NoError(t, err) require.NoError(t, err)

View File

@@ -52,6 +52,7 @@ func EncodeManifest(manifest *Manifest, compressionLevel int) ([]byte, error) {
// Compress using zstd // Compress using zstd
var compressedBuf bytes.Buffer var compressedBuf bytes.Buffer
writer, err := zstd.NewWriter(&compressedBuf, zstd.WithEncoderLevel(zstd.EncoderLevelFromZstd(compressionLevel))) writer, err := zstd.NewWriter(&compressedBuf, zstd.WithEncoderLevel(zstd.EncoderLevelFromZstd(compressionLevel)))
if err != nil { if err != nil {
return nil, fmt.Errorf("creating zstd writer: %w", err) return nil, fmt.Errorf("creating zstd writer: %w", err)
@@ -59,6 +60,7 @@ func EncodeManifest(manifest *Manifest, compressionLevel int) ([]byte, error) {
if _, err := writer.Write(jsonData); err != nil { if _, err := writer.Write(jsonData); err != nil {
_ = writer.Close() _ = writer.Close()
return nil, fmt.Errorf("writing compressed data: %w", err) return nil, fmt.Errorf("writing compressed data: %w", err)
} }

View File

@@ -12,7 +12,9 @@ import (
func TestWrapPermissionError(t *testing.T) { func TestWrapPermissionError(t *testing.T) {
// Non-permission errors pass through unchanged. // Non-permission errors pass through unchanged.
plain := errors.New("disk on fire") plain := errors.New("disk on fire")
if got := wrapPermissionError("/some/path", plain); got != plain {
got := wrapPermissionError("/some/path", plain)
if !errors.Is(got, plain) {
t.Errorf("non-permission error should pass through, got %v", got) t.Errorf("non-permission error should pass through, got %v", got)
} }
@@ -23,6 +25,7 @@ func TestWrapPermissionError(t *testing.T) {
if !errors.Is(wrapped, os.ErrPermission) { if !errors.Is(wrapped, os.ErrPermission) {
t.Error("wrapped error should still match os.ErrPermission") t.Error("wrapped error should still match os.ErrPermission")
} }
if !strings.Contains(wrapped.Error(), "/Users/u/Library/Calendars") { if !strings.Contains(wrapped.Error(), "/Users/u/Library/Calendars") {
t.Error("wrapped error should name the offending path") t.Error("wrapped error should name the offending path")
} }
@@ -31,6 +34,7 @@ func TestWrapPermissionError(t *testing.T) {
if !strings.Contains(wrapped.Error(), "Full Disk Access") { if !strings.Contains(wrapped.Error(), "Full Disk Access") {
t.Errorf("macOS permission error should mention Full Disk Access:\n%s", wrapped.Error()) t.Errorf("macOS permission error should mention Full Disk Access:\n%s", wrapped.Error())
} }
if !strings.Contains(wrapped.Error(), "System Settings") { if !strings.Contains(wrapped.Error(), "System Settings") {
t.Errorf("macOS permission error should point at System Settings:\n%s", wrapped.Error()) t.Errorf("macOS permission error should point at System Settings:\n%s", wrapped.Error())
} }

View File

@@ -153,6 +153,7 @@ func (pr *ProgressReporter) printSummaryStatus() {
if uploadInfo, ok := pr.stats.CurrentUpload.Load().(*UploadInfo); ok && uploadInfo != nil { if uploadInfo, ok := pr.stats.CurrentUpload.Load().(*UploadInfo); ok && uploadInfo != nil {
// Show upload progress instead // Show upload progress instead
pr.printUploadProgress(uploadInfo) pr.printUploadProgress(uploadInfo)
return return
} }
@@ -175,16 +176,18 @@ func (pr *ProgressReporter) printSummaryStatus() {
// Calculate ETA if we have total size and are processing // Calculate ETA if we have total size and are processing
etaStr := "" etaStr := ""
if totalSize > 0 && bytesProcessed > 0 { if totalSize > 0 && bytesProcessed > 0 {
processStart, ok := pr.stats.ProcessStartTime.Load().(time.Time) processStart, ok := pr.stats.ProcessStartTime.Load().(time.Time)
if ok && !processStart.IsZero() { if ok && !processStart.IsZero() {
processElapsed := time.Since(processStart) processElapsed := time.Since(processStart)
rate := float64(bytesProcessed) / processElapsed.Seconds() rate := float64(bytesProcessed) / processElapsed.Seconds()
if rate > 0 { if rate > 0 {
remainingBytes := totalSize - bytesProcessed remainingBytes := totalSize - bytesProcessed
remainingSeconds := float64(remainingBytes) / rate remainingSeconds := float64(remainingBytes) / rate
eta := time.Duration(remainingSeconds * float64(time.Second)) eta := time.Duration(remainingSeconds * float64(time.Second))
etaStr = fmt.Sprintf(" | ETA: %s", formatDuration(eta)) etaStr = " | ETA: " + formatDuration(eta)
} }
} }
} }
@@ -206,7 +209,7 @@ func (pr *ProgressReporter) printSummaryStatus() {
) )
if currentFile != "" { if currentFile != "" {
status += fmt.Sprintf(" | Current: %s", truncatePath(currentFile, 40)) status += " | Current: " + truncatePath(currentFile, 40)
} }
log.Info(status) log.Info(status)
@@ -242,6 +245,7 @@ func (pr *ProgressReporter) printDetailedStatus() {
processStart, ok := pr.stats.ProcessStartTime.Load().(time.Time) processStart, ok := pr.stats.ProcessStartTime.Load().(time.Time)
if ok && !processStart.IsZero() { if ok && !processStart.IsZero() {
processElapsed := time.Since(processStart) processElapsed := time.Since(processStart)
processRate := float64(bytesProcessed) / processElapsed.Seconds() processRate := float64(bytesProcessed) / processElapsed.Seconds()
if processRate > 0 { if processRate > 0 {
remainingBytes := totalSize - bytesProcessed remainingBytes := totalSize - bytesProcessed
@@ -276,9 +280,11 @@ func (pr *ProgressReporter) printDetailedStatus() {
log.Info("Total uploaded to remote", log.Info("Total uploaded to remote",
"uploaded", humanize.Bytes(uint64(bytesUploaded)), "uploaded", humanize.Bytes(uint64(bytesUploaded)),
"compression_ratio", formatRatio(bytesUploaded, bytesScanned)) "compression_ratio", formatRatio(bytesUploaded, bytesScanned))
if currentFile != "" { if currentFile != "" {
log.Info("Current file", "path", currentFile) log.Info("Current file", "path", currentFile)
} }
log.Notice("=============================") log.Notice("=============================")
} }
@@ -288,12 +294,15 @@ func formatDuration(d time.Duration) string {
if d < 0 { if d < 0 {
return "unknown" return "unknown"
} }
if d < time.Minute { if d < time.Minute {
return fmt.Sprintf("%ds", int(d.Seconds())) return fmt.Sprintf("%ds", int(d.Seconds()))
} }
if d < time.Hour { if d < time.Hour {
return fmt.Sprintf("%dm%ds", int(d.Minutes()), int(d.Seconds())%60) return fmt.Sprintf("%dm%ds", int(d.Minutes()), int(d.Seconds())%60)
} }
return fmt.Sprintf("%dh%dm", int(d.Hours()), int(d.Minutes())%60) return fmt.Sprintf("%dh%dm", int(d.Hours()), int(d.Minutes())%60)
} }
@@ -301,6 +310,7 @@ func formatPercent(numerator, denominator int64) string {
if denominator == 0 { if denominator == 0 {
return "0.0%" return "0.0%"
} }
return fmt.Sprintf("%.1f%%", float64(numerator)/float64(denominator)*100) return fmt.Sprintf("%.1f%%", float64(numerator)/float64(denominator)*100)
} }
@@ -308,7 +318,9 @@ func formatRatio(compressed, uncompressed int64) string {
if uncompressed == 0 { if uncompressed == 0 {
return "1.00" return "1.00"
} }
ratio := float64(compressed) / float64(uncompressed) ratio := float64(compressed) / float64(uncompressed)
return fmt.Sprintf("%.2f", ratio) return fmt.Sprintf("%.2f", ratio)
} }
@@ -353,6 +365,7 @@ func (pr *ProgressReporter) ReportUploadComplete(blobHash string, size int64, du
if duration < time.Millisecond { if duration < time.Millisecond {
duration = time.Millisecond duration = time.Millisecond
} }
bytesPerSec := float64(size) / duration.Seconds() bytesPerSec := float64(size) / duration.Seconds()
bitsPerSec := bytesPerSec * 8 bitsPerSec := bytesPerSec * 8
@@ -398,6 +411,7 @@ func (pr *ProgressReporter) ReportUploadProgress(blobHash string, bytesUploaded,
// Calculate ETA based on current speed // Calculate ETA based on current speed
etaStr := "unknown" etaStr := "unknown"
if instantSpeed > 0 && bytesUploaded < totalSize { if instantSpeed > 0 && bytesUploaded < totalSize {
remainingBytes := totalSize - bytesUploaded remainingBytes := totalSize - bytesUploaded
remainingSeconds := float64(remainingBytes) / instantSpeed remainingSeconds := float64(remainingBytes) / instantSpeed

View File

@@ -36,5 +36,6 @@ const remoteKeyPrefix = "vaultik|"
func RemoteSnapshotKey(snapshotID string) string { func RemoteSnapshotKey(snapshotID string) string {
first := sha256.Sum256([]byte(remoteKeyPrefix + snapshotID)) first := sha256.Sum256([]byte(remoteKeyPrefix + snapshotID))
second := sha256.Sum256(first[:]) second := sha256.Sum256(first[:])
return hex.EncodeToString(second[:]) return hex.EncodeToString(second[:])
} }

View File

@@ -119,6 +119,7 @@ func NewScanner(cfg ScannerConfig) *Scanner {
// Create encryptor (required for blob packing) // Create encryptor (required for blob packing)
if len(cfg.AgeRecipients) == 0 { if len(cfg.AgeRecipients) == 0 {
log.Error("No age recipients configured - encryption is required") log.Error("No age recipients configured - encryption is required")
return nil return nil
} }
@@ -130,9 +131,11 @@ func NewScanner(cfg ScannerConfig) *Scanner {
Repositories: cfg.Repositories, Repositories: cfg.Repositories,
Fs: cfg.FS, Fs: cfg.FS,
} }
packer, err := blob.NewPacker(packerCfg) packer, err := blob.NewPacker(packerCfg)
if err != nil { if err != nil {
log.Error("Failed to create packer", "error", err) log.Error("Failed to create packer", "error", err)
return nil return nil
} }
@@ -199,11 +202,14 @@ func (s *Scanner) Scan(ctx context.Context, path string, snapshotID string) (*Sc
// Phase 1: Scan directory, collect files to process, and track existing files // Phase 1: Scan directory, collect files to process, and track existing files
// (builds existingFiles map during walk to avoid double traversal) // (builds existingFiles map during walk to avoid double traversal)
log.Info("Phase 1/3: Scanning directory structure") log.Info("Phase 1/3: Scanning directory structure")
existingFiles := make(map[string]struct{}) existingFiles := make(map[string]struct{})
scanResult, err := s.scanPhase(ctx, path, result, existingFiles, knownFiles) scanResult, err := s.scanPhase(ctx, path, result, existingFiles, knownFiles)
if err != nil { if err != nil {
return nil, fmt.Errorf("scan phase failed: %w", err) return nil, fmt.Errorf("scan phase failed: %w", err)
} }
filesToProcess := scanResult.FilesToProcess filesToProcess := scanResult.FilesToProcess
// Phase 1b: Detect deleted files by comparing DB against scanned files // Phase 1b: Detect deleted files by comparing DB against scanned files
@@ -213,8 +219,10 @@ func (s *Scanner) Scan(ctx context.Context, path string, snapshotID string) (*Sc
// Phase 1c: Associate unchanged files with this snapshot (no new records needed) // Phase 1c: Associate unchanged files with this snapshot (no new records needed)
if len(scanResult.UnchangedFileIDs) > 0 { if len(scanResult.UnchangedFileIDs) > 0 {
s.ui.Begin("Associating %s unchanged files with the snapshot.", s.ui.Count(len(scanResult.UnchangedFileIDs))) s.ui.Beginf("Associating %s unchanged files with the snapshot.", s.ui.Count(len(scanResult.UnchangedFileIDs)))
if err := s.batchAddFilesToSnapshot(ctx, scanResult.UnchangedFileIDs); err != nil {
err := s.batchAddFilesToSnapshot(ctx, scanResult.UnchangedFileIDs)
if err != nil {
return nil, fmt.Errorf("associating unchanged files: %w", err) return nil, fmt.Errorf("associating unchanged files: %w", err)
} }
} }
@@ -224,13 +232,15 @@ func (s *Scanner) Scan(ctx context.Context, path string, snapshotID string) (*Sc
// Phase 2: Process files and create chunks // Phase 2: Process files and create chunks
if len(filesToProcess) > 0 { if len(filesToProcess) > 0 {
s.ui.Begin("Backing up %s snapshot source files (chunking, compressing, encrypting, uploading).", s.ui.Count(len(filesToProcess))) s.ui.Beginf("Backing up %s snapshot source files (chunking, compressing, encrypting, uploading).", s.ui.Count(len(filesToProcess)))
log.Info("Phase 2/3: Creating snapshot (chunking, compressing, encrypting, and uploading blobs)") log.Info("Phase 2/3: Creating snapshot (chunking, compressing, encrypting, and uploading blobs)")
if err := s.processPhase(ctx, filesToProcess, result); err != nil {
err := s.processPhase(ctx, filesToProcess, result)
if err != nil {
return nil, fmt.Errorf("process phase failed: %w", err) return nil, fmt.Errorf("process phase failed: %w", err)
} }
} else { } else {
s.ui.Info("Snapshot file backup skipped: no changed files (creating metadata-only snapshot).") s.ui.Infof("Snapshot file backup skipped: no changed files (creating metadata-only snapshot).")
log.Info("Phase 2/3: Skipping (no files need processing, metadata-only snapshot)") log.Info("Phase 2/3: Skipping (no files need processing, metadata-only snapshot)")
} }
@@ -243,18 +253,22 @@ func (s *Scanner) Scan(ctx context.Context, path string, snapshotID string) (*Sc
// loadDatabaseState loads known files and chunks from the database into memory for fast lookup // loadDatabaseState loads known files and chunks from the database into memory for fast lookup
// This avoids per-file and per-chunk database queries during the scan and process phases // This avoids per-file and per-chunk database queries during the scan and process phases
func (s *Scanner) loadDatabaseState(ctx context.Context, path string) (map[string]*database.File, error) { func (s *Scanner) loadDatabaseState(ctx context.Context, path string) (map[string]*database.File, error) {
s.ui.Begin("Loading known files from local index database.") s.ui.Beginf("Loading known files from local index database.")
knownFiles, err := s.loadKnownFiles(ctx, path) knownFiles, err := s.loadKnownFiles(ctx, path)
if err != nil { if err != nil {
return nil, fmt.Errorf("loading known files: %w", err) return nil, fmt.Errorf("loading known files: %w", err)
} }
s.ui.Complete("Loaded %s known files from local index database.", s.ui.Count(len(knownFiles)))
s.ui.Begin("Loading known chunks from local index database.") s.ui.Completef("Loaded %s known files from local index database.", s.ui.Count(len(knownFiles)))
s.ui.Beginf("Loading known chunks from local index database.")
if err := s.loadKnownChunks(ctx); err != nil { if err := s.loadKnownChunks(ctx); err != nil {
return nil, fmt.Errorf("loading known chunks: %w", err) return nil, fmt.Errorf("loading known chunks: %w", err)
} }
s.ui.Complete("Loaded %s known chunks from local index database.", s.ui.Count(len(s.knownChunks)))
s.ui.Completef("Loaded %s known chunks from local index database.", s.ui.Count(len(s.knownChunks)))
return knownFiles, nil return knownFiles, nil
} }
@@ -288,7 +302,8 @@ func (s *Scanner) summarizeScanPhase(result *ScanResult, filesToProcess []*FileT
s.ui.Count(result.FilesDeleted), s.ui.Count(result.FilesDeleted),
s.ui.Size(result.BytesDeleted)) s.ui.Size(result.BytesDeleted))
} }
s.ui.Complete("%s.", msg)
s.ui.Completef("%s.", msg)
} }
// finalizeScanResult populates final blob statistics in the scan result // finalizeScanResult populates final blob statistics in the scan result
@@ -337,6 +352,7 @@ func (s *Scanner) loadKnownChunks(ctx context.Context) error {
} }
s.knownChunksMu.Lock() s.knownChunksMu.Lock()
s.knownChunks = make(map[string]struct{}, len(chunks)) s.knownChunks = make(map[string]struct{}, len(chunks))
for _, c := range chunks { for _, c := range chunks {
s.knownChunks[c.ChunkHash.String()] = struct{}{} s.knownChunks[c.ChunkHash.String()] = struct{}{}
@@ -351,6 +367,7 @@ func (s *Scanner) chunkExists(hash string) bool {
s.knownChunksMu.RLock() s.knownChunksMu.RLock()
_, exists := s.knownChunks[hash] _, exists := s.knownChunks[hash]
s.knownChunksMu.RUnlock() s.knownChunksMu.RUnlock()
return exists return exists
} }
@@ -371,7 +388,9 @@ func (s *Scanner) addPendingChunkHash(hash string) {
// removePendingChunkHashes removes committed chunk hashes from the pending set // removePendingChunkHashes removes committed chunk hashes from the pending set
func (s *Scanner) removePendingChunkHashes(hashes []string) { func (s *Scanner) removePendingChunkHashes(hashes []string) {
log.Debug("removePendingChunkHashes: starting", "count", len(hashes)) log.Debug("removePendingChunkHashes: starting", "count", len(hashes))
start := time.Now() start := time.Now()
s.pendingChunkHashesMu.Lock() s.pendingChunkHashesMu.Lock()
for _, hash := range hashes { for _, hash := range hashes {
delete(s.pendingChunkHashes, hash) delete(s.pendingChunkHashes, hash)
@@ -385,6 +404,7 @@ func (s *Scanner) isChunkPending(hash string) bool {
s.pendingChunkHashesMu.Lock() s.pendingChunkHashesMu.Lock()
_, pending := s.pendingChunkHashes[hash] _, pending := s.pendingChunkHashes[hash]
s.pendingChunkHashesMu.Unlock() s.pendingChunkHashesMu.Unlock()
return pending return pending
} }
@@ -411,37 +431,45 @@ func (s *Scanner) flushPendingFiles(ctx context.Context) error {
return s.repos.WithTx(ctx, func(txCtx context.Context, tx *sql.Tx) error { return s.repos.WithTx(ctx, func(txCtx context.Context, tx *sql.Tx) error {
for _, data := range files { for _, data := range files {
// Create or update the file record // Create or update the file record
if err := s.repos.Files.Create(txCtx, tx, data.file); err != nil { err := s.repos.Files.Create(txCtx, tx, data.file)
if err != nil {
return fmt.Errorf("creating file record: %w", err) return fmt.Errorf("creating file record: %w", err)
} }
// Delete any existing file_chunks and chunk_files for this file // Delete any existing file_chunks and chunk_files for this file
if err := s.repos.FileChunks.DeleteByFileID(txCtx, tx, data.file.ID); err != nil { err = s.repos.FileChunks.DeleteByFileID(txCtx, tx, data.file.ID)
if err != nil {
return fmt.Errorf("deleting old file chunks: %w", err) return fmt.Errorf("deleting old file chunks: %w", err)
} }
if err := s.repos.ChunkFiles.DeleteByFileID(txCtx, tx, data.file.ID); err != nil {
err = s.repos.ChunkFiles.DeleteByFileID(txCtx, tx, data.file.ID)
if err != nil {
return fmt.Errorf("deleting old chunk files: %w", err) return fmt.Errorf("deleting old chunk files: %w", err)
} }
// Create file-chunk mappings // Create file-chunk mappings
for i := range data.fileChunks { for i := range data.fileChunks {
if err := s.repos.FileChunks.Create(txCtx, tx, &data.fileChunks[i]); err != nil { err := s.repos.FileChunks.Create(txCtx, tx, &data.fileChunks[i])
if err != nil {
return fmt.Errorf("creating file chunk: %w", err) return fmt.Errorf("creating file chunk: %w", err)
} }
} }
// Create chunk-file mappings // Create chunk-file mappings
for i := range data.chunkFiles { for i := range data.chunkFiles {
if err := s.repos.ChunkFiles.Create(txCtx, tx, &data.chunkFiles[i]); err != nil { err := s.repos.ChunkFiles.Create(txCtx, tx, &data.chunkFiles[i])
if err != nil {
return fmt.Errorf("creating chunk file: %w", err) return fmt.Errorf("creating chunk file: %w", err)
} }
} }
// Add file to snapshot // Add file to snapshot
if err := s.repos.Snapshots.AddFileByID(txCtx, tx, s.snapshotID, data.file.ID); err != nil { err = s.repos.Snapshots.AddFileByID(txCtx, tx, s.snapshotID, data.file.ID)
if err != nil {
return fmt.Errorf("adding file to snapshot: %w", err) return fmt.Errorf("adding file to snapshot: %w", err)
} }
} }
return nil return nil
}) })
} }
@@ -455,6 +483,7 @@ func (s *Scanner) flushAllPending(ctx context.Context) error {
// Files with pending chunks are kept in the queue for later flushing // Files with pending chunks are kept in the queue for later flushing
func (s *Scanner) flushCompletedPendingFiles(ctx context.Context) error { func (s *Scanner) flushCompletedPendingFiles(ctx context.Context) error {
flushStart := time.Now() flushStart := time.Now()
log.Debug("flushCompletedPendingFiles: starting") log.Debug("flushCompletedPendingFiles: starting")
// Partition pending files into those ready to flush and those still waiting // Partition pending files into those ready to flush and those still waiting
@@ -462,6 +491,7 @@ func (s *Scanner) flushCompletedPendingFiles(ctx context.Context) error {
if len(canFlush) == 0 { if len(canFlush) == 0 {
log.Debug("flushCompletedPendingFiles: nothing to flush") log.Debug("flushCompletedPendingFiles: nothing to flush")
return nil return nil
} }
@@ -474,10 +504,13 @@ func (s *Scanner) flushCompletedPendingFiles(ctx context.Context) error {
// Execute the batch flush in a single transaction // Execute the batch flush in a single transaction
log.Debug("flushCompletedPendingFiles: starting transaction") log.Debug("flushCompletedPendingFiles: starting transaction")
txStart := time.Now() txStart := time.Now()
err := s.executeBatchFileFlush(ctx, allFiles, allFileIDs, allFileChunks, allChunkFiles) err := s.executeBatchFileFlush(ctx, allFiles, allFileIDs, allFileChunks, allChunkFiles)
log.Debug("flushCompletedPendingFiles: transaction done", "duration", time.Since(txStart)) log.Debug("flushCompletedPendingFiles: transaction done", "duration", time.Since(txStart))
log.Debug("flushCompletedPendingFiles: total duration", "duration", time.Since(flushStart)) log.Debug("flushCompletedPendingFiles: total duration", "duration", time.Since(flushStart))
return err return err
} }
@@ -492,21 +525,27 @@ func (s *Scanner) partitionPendingByChunkStatus() (canFlush []pendingFileData, s
var stillPending []pendingFileData var stillPending []pendingFileData
log.Debug("flushCompletedPendingFiles: checking which files can flush") log.Debug("flushCompletedPendingFiles: checking which files can flush")
checkStart := time.Now() checkStart := time.Now()
for _, data := range s.pendingFiles { for _, data := range s.pendingFiles {
allChunksCommitted := true allChunksCommitted := true
for _, fc := range data.fileChunks { for _, fc := range data.fileChunks {
if s.isChunkPending(fc.ChunkHash.String()) { if s.isChunkPending(fc.ChunkHash.String()) {
allChunksCommitted = false allChunksCommitted = false
break break
} }
} }
if allChunksCommitted { if allChunksCommitted {
canFlush = append(canFlush, data) canFlush = append(canFlush, data)
} else { } else {
stillPending = append(stillPending, data) stillPending = append(stillPending, data)
} }
} }
log.Debug("flushCompletedPendingFiles: check done", "duration", time.Since(checkStart), "can_flush", len(canFlush), "still_pending", len(stillPending)) log.Debug("flushCompletedPendingFiles: check done", "duration", time.Since(checkStart), "can_flush", len(canFlush), "still_pending", len(stillPending))
s.pendingFiles = stillPending s.pendingFiles = stillPending
@@ -520,12 +559,15 @@ func (s *Scanner) partitionPendingByChunkStatus() (canFlush []pendingFileData, s
// mappings from the given pending file data for efficient batch database operations // mappings from the given pending file data for efficient batch database operations
func (s *Scanner) collectBatchFlushData(canFlush []pendingFileData) ([]*database.File, []types.FileID, []database.FileChunk, []database.ChunkFile) { func (s *Scanner) collectBatchFlushData(canFlush []pendingFileData) ([]*database.File, []types.FileID, []database.FileChunk, []database.ChunkFile) {
log.Debug("flushCompletedPendingFiles: collecting data for batch ops") log.Debug("flushCompletedPendingFiles: collecting data for batch ops")
collectStart := time.Now() collectStart := time.Now()
var allFileChunks []database.FileChunk var (
var allChunkFiles []database.ChunkFile allFileChunks []database.FileChunk
var allFileIDs []types.FileID allChunkFiles []database.ChunkFile
var allFiles []*database.File allFileIDs []types.FileID
allFiles []*database.File
)
for _, data := range canFlush { for _, data := range canFlush {
allFileChunks = append(allFileChunks, data.fileChunks...) allFileChunks = append(allFileChunks, data.fileChunks...)
@@ -551,52 +593,77 @@ func (s *Scanner) executeBatchFileFlush(ctx context.Context, allFiles []*databas
// Batch delete old file_chunks and chunk_files // Batch delete old file_chunks and chunk_files
log.Debug("flushCompletedPendingFiles: deleting old file_chunks") log.Debug("flushCompletedPendingFiles: deleting old file_chunks")
opStart := time.Now() opStart := time.Now()
if err := s.repos.FileChunks.DeleteByFileIDs(txCtx, tx, allFileIDs); err != nil {
err := s.repos.FileChunks.DeleteByFileIDs(txCtx, tx, allFileIDs)
if err != nil {
return fmt.Errorf("batch deleting old file chunks: %w", err) return fmt.Errorf("batch deleting old file chunks: %w", err)
} }
log.Debug("flushCompletedPendingFiles: deleted file_chunks", "duration", time.Since(opStart)) log.Debug("flushCompletedPendingFiles: deleted file_chunks", "duration", time.Since(opStart))
log.Debug("flushCompletedPendingFiles: deleting old chunk_files") log.Debug("flushCompletedPendingFiles: deleting old chunk_files")
opStart = time.Now() opStart = time.Now()
if err := s.repos.ChunkFiles.DeleteByFileIDs(txCtx, tx, allFileIDs); err != nil {
err = s.repos.ChunkFiles.DeleteByFileIDs(txCtx, tx, allFileIDs)
if err != nil {
return fmt.Errorf("batch deleting old chunk files: %w", err) return fmt.Errorf("batch deleting old chunk files: %w", err)
} }
log.Debug("flushCompletedPendingFiles: deleted chunk_files", "duration", time.Since(opStart)) log.Debug("flushCompletedPendingFiles: deleted chunk_files", "duration", time.Since(opStart))
// Batch create/update file records // Batch create/update file records
log.Debug("flushCompletedPendingFiles: creating files") log.Debug("flushCompletedPendingFiles: creating files")
opStart = time.Now() opStart = time.Now()
if err := s.repos.Files.CreateBatch(txCtx, tx, allFiles); err != nil {
err = s.repos.Files.CreateBatch(txCtx, tx, allFiles)
if err != nil {
return fmt.Errorf("batch creating file records: %w", err) return fmt.Errorf("batch creating file records: %w", err)
} }
log.Debug("flushCompletedPendingFiles: created files", "duration", time.Since(opStart)) log.Debug("flushCompletedPendingFiles: created files", "duration", time.Since(opStart))
// Batch insert file_chunks // Batch insert file_chunks
log.Debug("flushCompletedPendingFiles: inserting file_chunks") log.Debug("flushCompletedPendingFiles: inserting file_chunks")
opStart = time.Now() opStart = time.Now()
if err := s.repos.FileChunks.CreateBatch(txCtx, tx, allFileChunks); err != nil {
err = s.repos.FileChunks.CreateBatch(txCtx, tx, allFileChunks)
if err != nil {
return fmt.Errorf("batch creating file chunks: %w", err) return fmt.Errorf("batch creating file chunks: %w", err)
} }
log.Debug("flushCompletedPendingFiles: inserted file_chunks", "duration", time.Since(opStart)) log.Debug("flushCompletedPendingFiles: inserted file_chunks", "duration", time.Since(opStart))
// Batch insert chunk_files // Batch insert chunk_files
log.Debug("flushCompletedPendingFiles: inserting chunk_files") log.Debug("flushCompletedPendingFiles: inserting chunk_files")
opStart = time.Now() opStart = time.Now()
if err := s.repos.ChunkFiles.CreateBatch(txCtx, tx, allChunkFiles); err != nil {
err = s.repos.ChunkFiles.CreateBatch(txCtx, tx, allChunkFiles)
if err != nil {
return fmt.Errorf("batch creating chunk files: %w", err) return fmt.Errorf("batch creating chunk files: %w", err)
} }
log.Debug("flushCompletedPendingFiles: inserted chunk_files", "duration", time.Since(opStart)) log.Debug("flushCompletedPendingFiles: inserted chunk_files", "duration", time.Since(opStart))
// Batch add files to snapshot // Batch add files to snapshot
log.Debug("flushCompletedPendingFiles: adding files to snapshot") log.Debug("flushCompletedPendingFiles: adding files to snapshot")
opStart = time.Now() opStart = time.Now()
if err := s.repos.Snapshots.AddFilesByIDBatch(txCtx, tx, s.snapshotID, allFileIDs); err != nil {
err = s.repos.Snapshots.AddFilesByIDBatch(txCtx, tx, s.snapshotID, allFileIDs)
if err != nil {
return fmt.Errorf("batch adding files to snapshot: %w", err) return fmt.Errorf("batch adding files to snapshot: %w", err)
} }
log.Debug("flushCompletedPendingFiles: added files to snapshot", "duration", time.Since(opStart)) log.Debug("flushCompletedPendingFiles: added files to snapshot", "duration", time.Since(opStart))
log.Debug("flushCompletedPendingFiles: transaction complete") log.Debug("flushCompletedPendingFiles: transaction complete")
return nil return nil
}) })
} }
@@ -616,24 +683,31 @@ func (s *Scanner) scanPhase(ctx context.Context, path string, result *ScanResult
estimatedTotal := int64(len(knownFiles)) estimatedTotal := int64(len(knownFiles))
var filesToProcess []*FileToProcess var filesToProcess []*FileToProcess
var unchangedFileIDs []types.FileID // Just IDs - no new records needed var unchangedFileIDs []types.FileID // Just IDs - no new records needed
var mu sync.Mutex var mu sync.Mutex
// Set up periodic status output // Set up periodic status output
startTime := time.Now() startTime := time.Now()
lastStatusTime := time.Now() lastStatusTime := time.Now()
statusInterval := 15 * time.Second statusInterval := 15 * time.Second
var filesScanned int64 var filesScanned int64
log.Debug("Starting directory walk", "path", path) log.Debug("Starting directory walk", "path", path)
err := afero.Walk(s.fs, path, func(filePath string, info os.FileInfo, err error) error { err := afero.Walk(s.fs, path, func(filePath string, info os.FileInfo, err error) error {
if err != nil { if err != nil {
if s.skipErrors { if s.skipErrors {
log.Error("Failed to access file (skipping due to --skip-errors)", "path", filePath, "error", err) log.Error("Failed to access file (skipping due to --skip-errors)", "path", filePath, "error", err)
s.ui.Error("Failed to access %s: %v. Skipping (--skip-errors).", s.ui.Path(filePath), err) s.ui.Errorf("Failed to access %s: %v. Skipping (--skip-errors).", s.ui.Path(filePath), err)
return nil // Continue scanning return nil // Continue scanning
} }
log.Debug("Error accessing filesystem entry", "path", filePath, "error", err) log.Debug("Error accessing filesystem entry", "path", filePath, "error", err)
return wrapPermissionError(filePath, err) return wrapPermissionError(filePath, err)
} }
@@ -649,6 +723,7 @@ func (s *Scanner) scanPhase(ctx context.Context, path string, result *ScanResult
if info.IsDir() { if info.IsDir() {
return filepath.SkipDir return filepath.SkipDir
} }
return nil return nil
} }
@@ -657,7 +732,9 @@ func (s *Scanner) scanPhase(ctx context.Context, path string, result *ScanResult
file := s.buildSymlinkEntry(filePath, info) file := s.buildSymlinkEntry(filePath, info)
if file != nil { if file != nil {
existingFiles[filePath] = struct{}{} existingFiles[filePath] = struct{}{}
mu.Lock() mu.Lock()
filesToProcess = append(filesToProcess, &FileToProcess{ filesToProcess = append(filesToProcess, &FileToProcess{
Path: filePath, Path: filePath,
FileInfo: info, FileInfo: info,
@@ -667,6 +744,7 @@ func (s *Scanner) scanPhase(ctx context.Context, path string, result *ScanResult
mu.Unlock() mu.Unlock()
s.updateScanEntryStats(result, true, info) s.updateScanEntryStats(result, true, info)
} }
return nil return nil
} }
@@ -674,7 +752,9 @@ func (s *Scanner) scanPhase(ctx context.Context, path string, result *ScanResult
if info.IsDir() { if info.IsDir() {
file := s.buildDirectoryEntry(filePath, info) file := s.buildDirectoryEntry(filePath, info)
existingFiles[filePath] = struct{}{} existingFiles[filePath] = struct{}{}
mu.Lock() mu.Lock()
filesToProcess = append(filesToProcess, &FileToProcess{ filesToProcess = append(filesToProcess, &FileToProcess{
Path: filePath, Path: filePath,
FileInfo: info, FileInfo: info,
@@ -682,6 +762,7 @@ func (s *Scanner) scanPhase(ctx context.Context, path string, result *ScanResult
}) })
filesScanned++ filesScanned++
mu.Unlock() mu.Unlock()
return nil return nil
} }
@@ -708,6 +789,7 @@ func (s *Scanner) scanPhase(ctx context.Context, path string, result *ScanResult
// Unchanged file with existing ID - just need snapshot association // Unchanged file with existing ID - just need snapshot association
unchangedFileIDs = append(unchangedFileIDs, file.ID) unchangedFileIDs = append(unchangedFileIDs, file.ID)
} }
filesScanned++ filesScanned++
changedCount := len(filesToProcess) changedCount := len(filesToProcess)
mu.Unlock() mu.Unlock()
@@ -718,12 +800,12 @@ func (s *Scanner) scanPhase(ctx context.Context, path string, result *ScanResult
// Output periodic status // Output periodic status
if time.Since(lastStatusTime) >= statusInterval { if time.Since(lastStatusTime) >= statusInterval {
s.printScanProgressLine(filesScanned, changedCount, estimatedTotal, startTime) s.printScanProgressLine(filesScanned, changedCount, estimatedTotal, startTime)
lastStatusTime = time.Now() lastStatusTime = time.Now()
} }
return nil return nil
}) })
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -745,11 +827,13 @@ func (s *Scanner) updateScanEntryStats(result *ScanResult, needsProcessing bool,
} else { } else {
result.FilesSkipped++ result.FilesSkipped++
result.BytesSkipped += info.Size() result.BytesSkipped += info.Size()
if s.progress != nil { if s.progress != nil {
s.progress.GetStats().FilesSkipped.Add(1) s.progress.GetStats().FilesSkipped.Add(1)
s.progress.GetStats().BytesSkipped.Add(info.Size()) s.progress.GetStats().BytesSkipped.Add(info.Size())
} }
} }
result.FilesScanned++ result.FilesScanned++
if s.progress != nil { if s.progress != nil {
s.progress.GetStats().FilesScanned.Add(1) s.progress.GetStats().FilesScanned.Add(1)
@@ -768,16 +852,16 @@ func (s *Scanner) printScanProgressLine(filesScanned int64, changedCount int, es
if pct > 100 { if pct > 100 {
pct = 100 // Cap at 100% for display pct = 100 // Cap at 100% for display
} }
remaining := estimatedTotal - filesScanned
if remaining < 0 { remaining := max(estimatedTotal-filesScanned, 0)
remaining = 0
}
var eta time.Duration var eta time.Duration
if rate > 0 && remaining > 0 { if rate > 0 && remaining > 0 {
eta = time.Duration(float64(remaining)/rate) * time.Second eta = time.Duration(float64(remaining)/rate) * time.Second
} }
if eta > 0 { if eta > 0 {
s.ui.Progress("Snapshot source files enumeration: %s files (~%s), %s changed or new, %.0f files/sec, enumeration elapsed: %s, enumeration ETA: %s (est remain %s).", s.ui.Progressf("Snapshot source files enumeration: %s files (~%s), %s changed or new, %.0f files/sec, enumeration elapsed: %s, enumeration ETA: %s (est remain %s).",
s.ui.Count(int(filesScanned)), s.ui.Count(int(filesScanned)),
s.ui.Percent(pct), s.ui.Percent(pct),
s.ui.Count(changedCount), s.ui.Count(changedCount),
@@ -786,7 +870,7 @@ func (s *Scanner) printScanProgressLine(filesScanned int64, changedCount int, es
s.ui.Time(time.Now().Add(eta)), s.ui.Time(time.Now().Add(eta)),
s.ui.Duration(eta)) s.ui.Duration(eta))
} else { } else {
s.ui.Progress("Snapshot source files enumeration: %s files (~%s), %s changed or new, %.0f files/sec, enumeration elapsed: %s.", s.ui.Progressf("Snapshot source files enumeration: %s files (~%s), %s changed or new, %.0f files/sec, enumeration elapsed: %s.",
s.ui.Count(int(filesScanned)), s.ui.Count(int(filesScanned)),
s.ui.Percent(pct), s.ui.Percent(pct),
s.ui.Count(changedCount), s.ui.Count(changedCount),
@@ -794,7 +878,7 @@ func (s *Scanner) printScanProgressLine(filesScanned int64, changedCount int, es
s.ui.Duration(elapsed)) s.ui.Duration(elapsed))
} }
} else { } else {
s.ui.Progress("Snapshot source files enumeration: %s files seen, %s changed or new, %.0f files/sec, enumeration elapsed: %s.", s.ui.Progressf("Snapshot source files enumeration: %s files seen, %s changed or new, %.0f files/sec, enumeration elapsed: %s.",
s.ui.Count(int(filesScanned)), s.ui.Count(int(filesScanned)),
s.ui.Count(changedCount), s.ui.Count(changedCount),
rate, rate,
@@ -808,6 +892,7 @@ func (s *Scanner) buildSymlinkEntry(path string, info os.FileInfo) *database.Fil
target, err := os.Readlink(path) target, err := os.Readlink(path)
if err != nil { if err != nil {
log.Debug("Cannot read symlink target", "path", path, "error", err) log.Debug("Cannot read symlink target", "path", path, "error", err)
return nil return nil
} }
@@ -860,9 +945,11 @@ func (s *Scanner) buildDirectoryEntry(path string, info os.FileInfo) *database.F
// and associates it with the current snapshot. No chunking is performed. // and associates it with the current snapshot. No chunking is performed.
func (s *Scanner) recordNonRegularFile(ctx context.Context, ftp *FileToProcess) error { func (s *Scanner) recordNonRegularFile(ctx context.Context, ftp *FileToProcess) error {
return s.repos.WithTx(ctx, func(txCtx context.Context, tx *sql.Tx) error { return s.repos.WithTx(ctx, func(txCtx context.Context, tx *sql.Tx) error {
if err := s.repos.Files.Create(txCtx, tx, ftp.File); err != nil { err := s.repos.Files.Create(txCtx, tx, ftp.File)
if err != nil {
return fmt.Errorf("creating non-regular file record: %w", err) return fmt.Errorf("creating non-regular file record: %w", err)
} }
return s.repos.Snapshots.AddFileByID(txCtx, tx, s.snapshotID, ftp.File.ID) return s.repos.Snapshots.AddFileByID(txCtx, tx, s.snapshotID, ftp.File.ID)
}) })
} }
@@ -941,18 +1028,18 @@ func (s *Scanner) batchAddFilesToSnapshot(ctx context.Context, fileIDs []types.F
default: default:
} }
end := i + batchSize end := min(i+batchSize, len(fileIDs))
if end > len(fileIDs) {
end = len(fileIDs)
}
batch := fileIDs[i:end] batch := fileIDs[i:end]
err := s.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { err := s.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
for _, fileID := range batch { for _, fileID := range batch {
if err := s.repos.Snapshots.AddFileByID(ctx, tx, s.snapshotID, fileID); err != nil { err := s.repos.Snapshots.AddFileByID(ctx, tx, s.snapshotID, fileID)
if err != nil {
return fmt.Errorf("adding file to snapshot: %w", err) return fmt.Errorf("adding file to snapshot: %w", err)
} }
} }
return nil return nil
}) })
if err != nil { if err != nil {
@@ -964,15 +1051,16 @@ func (s *Scanner) batchAddFilesToSnapshot(ctx context.Context, fileIDs []types.F
elapsed := time.Since(startTime) elapsed := time.Since(startTime)
rate := float64(end) / elapsed.Seconds() rate := float64(end) / elapsed.Seconds()
pct := float64(end) / float64(len(fileIDs)) * 100 pct := float64(end) / float64(len(fileIDs)) * 100
s.ui.Progress("Snapshot unchanged-file association: %s/%s (%s), %.0f files/sec.", s.ui.Progressf("Snapshot unchanged-file association: %s/%s (%s), %.0f files/sec.",
s.ui.Count(end), s.ui.Count(len(fileIDs)), s.ui.Percent(pct), rate) s.ui.Count(end), s.ui.Count(len(fileIDs)), s.ui.Percent(pct), rate)
lastStatusTime = time.Now() lastStatusTime = time.Now()
} }
} }
elapsed := time.Since(startTime) elapsed := time.Since(startTime)
rate := float64(len(fileIDs)) / elapsed.Seconds() rate := float64(len(fileIDs)) / elapsed.Seconds()
s.ui.Complete("Associated %s unchanged files with the snapshot in %s (%.0f files/sec).", s.ui.Completef("Associated %s unchanged files with the snapshot in %s (%.0f files/sec).",
s.ui.Count(len(fileIDs)), s.ui.Duration(elapsed), rate) s.ui.Count(len(fileIDs)), s.ui.Duration(elapsed), rate)
return nil return nil
@@ -991,7 +1079,9 @@ func (s *Scanner) processPhase(ctx context.Context, filesToProcess []*FileToProc
statusInterval := 15 * time.Second statusInterval := 15 * time.Second
startTime := time.Now() startTime := time.Now()
filesProcessed := 0 filesProcessed := 0
var bytesProcessed int64 var bytesProcessed int64
totalFiles := len(filesToProcess) totalFiles := len(filesToProcess)
// Process each file // Process each file
@@ -1006,6 +1096,7 @@ func (s *Scanner) processPhase(ctx context.Context, filesToProcess []*FileToProc
if err != nil { if err != nil {
return err return err
} }
if skipped { if skipped {
continue continue
} }
@@ -1021,6 +1112,7 @@ func (s *Scanner) processPhase(ctx context.Context, filesToProcess []*FileToProc
// Output periodic status // Output periodic status
if time.Since(lastStatusTime) >= statusInterval { if time.Since(lastStatusTime) >= statusInterval {
s.printProcessingProgress(filesProcessed, totalFiles, bytesProcessed, totalBytes, startTime) s.printProcessingProgress(filesProcessed, totalFiles, bytesProcessed, totalBytes, startTime)
lastStatusTime = time.Now() lastStatusTime = time.Now()
} }
} }
@@ -1032,22 +1124,29 @@ func (s *Scanner) processPhase(ctx context.Context, filesToProcess []*FileToProc
// processFileWithErrorHandling wraps processFileStreaming with error recovery for // processFileWithErrorHandling wraps processFileStreaming with error recovery for
// deleted files and skip-errors mode. Returns (skipped, error). // deleted files and skip-errors mode. Returns (skipped, error).
func (s *Scanner) processFileWithErrorHandling(ctx context.Context, fileToProcess *FileToProcess, result *ScanResult) (bool, error) { func (s *Scanner) processFileWithErrorHandling(ctx context.Context, fileToProcess *FileToProcess, result *ScanResult) (bool, error) {
if err := s.processFileStreaming(ctx, fileToProcess, result); err != nil { err := s.processFileStreaming(ctx, fileToProcess, result)
if err != nil {
// Handle files that were deleted between scan and process phases // Handle files that were deleted between scan and process phases
if errors.Is(err, os.ErrNotExist) { if errors.Is(err, os.ErrNotExist) {
log.Warn("File was deleted during backup, skipping", "path", fileToProcess.Path) log.Warn("File was deleted during backup, skipping", "path", fileToProcess.Path)
result.FilesSkipped++ result.FilesSkipped++
return true, nil return true, nil
} }
// Skip file read errors if --skip-errors is enabled // Skip file read errors if --skip-errors is enabled
if s.skipErrors { if s.skipErrors {
log.Error("Failed to process file (skipping due to --skip-errors)", "path", fileToProcess.Path, "error", err) log.Error("Failed to process file (skipping due to --skip-errors)", "path", fileToProcess.Path, "error", err)
s.ui.Error("Failed to process %s: %v. Skipping (--skip-errors).", s.ui.Path(fileToProcess.Path), err) s.ui.Errorf("Failed to process %s: %v. Skipping (--skip-errors).", s.ui.Path(fileToProcess.Path), err)
result.FilesSkipped++ result.FilesSkipped++
return true, nil return true, nil
} }
return false, fmt.Errorf("processing file %s: %w", fileToProcess.Path, err) return false, fmt.Errorf("processing file %s: %w", fileToProcess.Path, err)
} }
return false, nil return false, nil
} }
@@ -1061,13 +1160,14 @@ func (s *Scanner) printProcessingProgress(filesProcessed, totalFiles int, bytesP
// Calculate ETA based on bytes (more accurate than files) // Calculate ETA based on bytes (more accurate than files)
remainingBytes := totalBytes - bytesProcessed remainingBytes := totalBytes - bytesProcessed
var eta time.Duration var eta time.Duration
if byteRate > 0 { if byteRate > 0 {
eta = time.Duration(float64(remainingBytes)/byteRate) * time.Second eta = time.Duration(float64(remainingBytes)/byteRate) * time.Second
} }
if eta > 0 { if eta > 0 {
s.ui.Progress("Snapshot backup: %s/%s files (%s), %s/%s, %s, %.0f files/sec, backup elapsed: %s, backup ETA: %s (est remain %s).", s.ui.Progressf("Snapshot backup: %s/%s files (%s), %s/%s, %s, %.0f files/sec, backup elapsed: %s, backup ETA: %s (est remain %s).",
s.ui.Count(filesProcessed), s.ui.Count(filesProcessed),
s.ui.Count(totalFiles), s.ui.Count(totalFiles),
s.ui.Percent(pct), s.ui.Percent(pct),
@@ -1079,7 +1179,7 @@ func (s *Scanner) printProcessingProgress(filesProcessed, totalFiles int, bytesP
s.ui.Time(time.Now().Add(eta)), s.ui.Time(time.Now().Add(eta)),
s.ui.Duration(eta)) s.ui.Duration(eta))
} else { } else {
s.ui.Progress("Snapshot backup: %s/%s files (%s), %s/%s, %s, %.0f files/sec, backup elapsed: %s.", s.ui.Progressf("Snapshot backup: %s/%s files (%s), %s/%s, %s, %.0f files/sec, backup elapsed: %s.",
s.ui.Count(filesProcessed), s.ui.Count(filesProcessed),
s.ui.Count(totalFiles), s.ui.Count(totalFiles),
s.ui.Percent(pct), s.ui.Percent(pct),
@@ -1097,15 +1197,19 @@ func (s *Scanner) finalizeProcessPhase(ctx context.Context, result *ScanResult)
// Final packer flush first - this commits remaining chunks to DB // Final packer flush first - this commits remaining chunks to DB
// and handleBlobReady will flush files whose chunks are now committed // and handleBlobReady will flush files whose chunks are now committed
s.packerMu.Lock() s.packerMu.Lock()
if err := s.packer.Flush(); err != nil {
err := s.packer.Flush()
if err != nil {
s.packerMu.Unlock() s.packerMu.Unlock()
return fmt.Errorf("flushing packer: %w", err) return fmt.Errorf("flushing packer: %w", err)
} }
s.packerMu.Unlock() s.packerMu.Unlock()
// Flush any remaining pending files (e.g., files with only pre-existing chunks // Flush any remaining pending files (e.g., files with only pre-existing chunks
// that didn't trigger a blob finalize) // that didn't trigger a blob finalize)
if err := s.flushAllPending(ctx); err != nil { err = s.flushAllPending(ctx)
if err != nil {
return fmt.Errorf("flushing remaining pending files: %w", err) return fmt.Errorf("flushing remaining pending files: %w", err)
} }
@@ -1119,6 +1223,7 @@ func (s *Scanner) finalizeProcessPhase(ctx context.Context, result *ScanResult)
if err != nil { if err != nil {
return fmt.Errorf("parsing blob ID: %w", err) return fmt.Errorf("parsing blob ID: %w", err)
} }
err = s.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { err = s.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
return s.repos.Snapshots.AddBlob(ctx, tx, s.snapshotID, blobID, types.BlobHash(b.Hash)) return s.repos.Snapshots.AddBlob(ctx, tx, s.snapshotID, blobID, types.BlobHash(b.Hash))
}) })
@@ -1126,6 +1231,7 @@ func (s *Scanner) finalizeProcessPhase(ctx context.Context, result *ScanResult)
return fmt.Errorf("storing blob metadata: %w", err) return fmt.Errorf("storing blob metadata: %w", err)
} }
} }
result.BlobsCreated += len(blobs) result.BlobsCreated += len(blobs)
} }
@@ -1148,14 +1254,17 @@ func (s *Scanner) handleBlobReady(blobWithReader *blob.BlobWithReader) error {
} }
blobPath := fmt.Sprintf("blobs/%s/%s/%s", finishedBlob.Hash[:2], finishedBlob.Hash[2:4], finishedBlob.Hash) blobPath := fmt.Sprintf("blobs/%s/%s/%s", finishedBlob.Hash[:2], finishedBlob.Hash[2:4], finishedBlob.Hash)
blobExists, err := s.uploadBlobIfNeeded(ctx, blobPath, blobWithReader, startTime) blobExists, err := s.uploadBlobIfNeeded(ctx, blobPath, blobWithReader, startTime)
if err != nil { if err != nil {
s.cleanupBlobTempFile(blobWithReader) s.cleanupBlobTempFile(blobWithReader)
return fmt.Errorf("uploading blob %s: %w", finishedBlob.Hash, err) return fmt.Errorf("uploading blob %s: %w", finishedBlob.Hash, err)
} }
if err := s.recordBlobMetadata(ctx, finishedBlob, blobExists, startTime); err != nil { if err := s.recordBlobMetadata(ctx, finishedBlob, blobExists, startTime); err != nil {
s.cleanupBlobTempFile(blobWithReader) s.cleanupBlobTempFile(blobWithReader)
return err return err
} }
@@ -1181,25 +1290,28 @@ func (s *Scanner) uploadBlobIfNeeded(ctx context.Context, blobPath string, blobW
if _, err := s.storage.Stat(ctx, blobPath); err == nil { if _, err := s.storage.Stat(ctx, blobPath); err == nil {
log.Info("Blob already exists in storage, skipping upload", log.Info("Blob already exists in storage, skipping upload",
"hash", finishedBlob.Hash, "size", humanize.Bytes(uint64(finishedBlob.Compressed))) "hash", finishedBlob.Hash, "size", humanize.Bytes(uint64(finishedBlob.Compressed)))
s.ui.Info("Blob %s (%s) already exists at %s. Skipping upload.", s.ui.Infof("Blob %s (%s) already exists at %s. Skipping upload.",
s.ui.Hex(finishedBlob.Hash), s.ui.Size(finishedBlob.Compressed), s.ui.Path(destination)) s.ui.Hex(finishedBlob.Hash), s.ui.Size(finishedBlob.Compressed), s.ui.Path(destination))
return true, nil return true, nil
} }
s.ui.Begin("Uploading blob %s (%s) to %s.", s.ui.Beginf("Uploading blob %s (%s) to %s.",
s.ui.Hex(finishedBlob.Hash), s.ui.Size(finishedBlob.Compressed), s.ui.Path(destination)) s.ui.Hex(finishedBlob.Hash), s.ui.Size(finishedBlob.Compressed), s.ui.Path(destination))
progressCallback := s.makeUploadProgressCallback(ctx, finishedBlob, startTime) progressCallback := s.makeUploadProgressCallback(ctx, finishedBlob, startTime)
if err := s.storage.PutWithProgress(ctx, blobPath, blobWithReader.Reader, finishedBlob.Compressed, progressCallback); err != nil { err := s.storage.PutWithProgress(ctx, blobPath, blobWithReader.Reader, finishedBlob.Compressed, progressCallback)
if err != nil {
log.Error("Failed to upload blob", "hash", finishedBlob.Hash, "error", err) log.Error("Failed to upload blob", "hash", finishedBlob.Hash, "error", err)
return false, fmt.Errorf("uploading blob to storage: %w", err) return false, fmt.Errorf("uploading blob to storage: %w", err)
} }
uploadDuration := time.Since(startTime) uploadDuration := time.Since(startTime)
uploadSpeedBps := float64(finishedBlob.Compressed) / uploadDuration.Seconds() uploadSpeedBps := float64(finishedBlob.Compressed) / uploadDuration.Seconds()
s.ui.Complete("Uploaded blob %s (%s) in %s at %s.", s.ui.Completef("Uploaded blob %s (%s) in %s at %s.",
s.ui.Hex(finishedBlob.Hash), s.ui.Hex(finishedBlob.Hash),
s.ui.Size(finishedBlob.Compressed), s.ui.Size(finishedBlob.Compressed),
s.ui.Duration(uploadDuration), s.ui.Duration(uploadDuration),
@@ -1228,17 +1340,21 @@ func (s *Scanner) makeUploadProgressCallback(ctx context.Context, finishedBlob *
lastProgressTime := time.Now() lastProgressTime := time.Now()
lastProgressBytes := int64(0) lastProgressBytes := int64(0)
lastStdoutTime := time.Now() lastStdoutTime := time.Now()
const stdoutInterval = 15 * time.Second const stdoutInterval = 15 * time.Second
return func(uploaded int64) error { return func(uploaded int64) error {
now := time.Now() now := time.Now()
elapsed := now.Sub(lastProgressTime).Seconds() elapsed := now.Sub(lastProgressTime).Seconds()
if elapsed > 0.5 { if elapsed > 0.5 {
bytesSinceLastUpdate := uploaded - lastProgressBytes bytesSinceLastUpdate := uploaded - lastProgressBytes
speed := float64(bytesSinceLastUpdate) / elapsed speed := float64(bytesSinceLastUpdate) / elapsed
if s.progress != nil { if s.progress != nil {
s.progress.ReportUploadProgress(finishedBlob.Hash, uploaded, finishedBlob.Compressed, speed) s.progress.ReportUploadProgress(finishedBlob.Hash, uploaded, finishedBlob.Compressed, speed)
} }
lastProgressTime = now lastProgressTime = now
lastProgressBytes = uploaded lastProgressBytes = uploaded
} }
@@ -1248,11 +1364,13 @@ func (s *Scanner) makeUploadProgressCallback(ctx context.Context, finishedBlob *
totalElapsed := now.Sub(uploadStart) totalElapsed := now.Sub(uploadStart)
pct := float64(uploaded) / float64(finishedBlob.Compressed) * 100 pct := float64(uploaded) / float64(finishedBlob.Compressed) * 100
avgSpeed := float64(uploaded) / totalElapsed.Seconds() avgSpeed := float64(uploaded) / totalElapsed.Seconds()
var eta time.Duration var eta time.Duration
if avgSpeed > 0 { if avgSpeed > 0 {
eta = time.Duration(float64(finishedBlob.Compressed-uploaded)/avgSpeed) * time.Second eta = time.Duration(float64(finishedBlob.Compressed-uploaded)/avgSpeed) * time.Second
} }
s.ui.Progress("Blob upload %s: %s / %s (%s) at %s, blob upload elapsed: %s, blob upload ETA: %s (est remain %s).",
s.ui.Progressf("Blob upload %s: %s / %s (%s) at %s, blob upload elapsed: %s, blob upload ETA: %s (est remain %s).",
s.ui.Hex(finishedBlob.Hash), s.ui.Hex(finishedBlob.Hash),
s.ui.Size(uploaded), s.ui.Size(uploaded),
s.ui.Size(finishedBlob.Compressed), s.ui.Size(finishedBlob.Compressed),
@@ -1283,11 +1401,13 @@ func (s *Scanner) recordBlobMetadata(ctx context.Context, finishedBlob *blob.Fin
uploadDuration := time.Since(startTime) uploadDuration := time.Since(startTime)
return s.repos.WithTx(ctx, func(txCtx context.Context, tx *sql.Tx) error { return s.repos.WithTx(ctx, func(txCtx context.Context, tx *sql.Tx) error {
if err := s.repos.Blobs.UpdateUploaded(txCtx, tx, finishedBlob.ID); err != nil { err := s.repos.Blobs.UpdateUploaded(txCtx, tx, finishedBlob.ID)
if err != nil {
return fmt.Errorf("updating blob upload timestamp: %w", err) return fmt.Errorf("updating blob upload timestamp: %w", err)
} }
if err := s.repos.Snapshots.AddBlob(txCtx, tx, s.snapshotID, finishedBlobID, types.BlobHash(finishedBlob.Hash)); err != nil { err = s.repos.Snapshots.AddBlob(txCtx, tx, s.snapshotID, finishedBlobID, types.BlobHash(finishedBlob.Hash))
if err != nil {
return fmt.Errorf("adding blob to snapshot: %w", err) return fmt.Errorf("adding blob to snapshot: %w", err)
} }
@@ -1299,7 +1419,9 @@ func (s *Scanner) recordBlobMetadata(ctx context.Context, finishedBlob *blob.Fin
Size: finishedBlob.Compressed, Size: finishedBlob.Compressed,
DurationMs: uploadDuration.Milliseconds(), DurationMs: uploadDuration.Milliseconds(),
} }
if err := s.repos.Uploads.Create(txCtx, tx, upload); err != nil {
err := s.repos.Uploads.Create(txCtx, tx, upload)
if err != nil {
return fmt.Errorf("recording upload metrics: %w", err) return fmt.Errorf("recording upload metrics: %w", err)
} }
} }
@@ -1312,10 +1434,14 @@ func (s *Scanner) recordBlobMetadata(ctx context.Context, finishedBlob *blob.Fin
func (s *Scanner) cleanupBlobTempFile(blobWithReader *blob.BlobWithReader) { func (s *Scanner) cleanupBlobTempFile(blobWithReader *blob.BlobWithReader) {
if blobWithReader.TempFile != nil { if blobWithReader.TempFile != nil {
tempName := blobWithReader.TempFile.Name() tempName := blobWithReader.TempFile.Name()
if err := blobWithReader.TempFile.Close(); err != nil {
err := blobWithReader.TempFile.Close()
if err != nil {
log.Fatal("Failed to close temp file", "file", tempName, "error", err) log.Fatal("Failed to close temp file", "file", tempName, "error", err)
} }
if err := s.fs.Remove(tempName); err != nil {
err = s.fs.Remove(tempName)
if err != nil {
log.Fatal("Failed to remove temp file", "file", tempName, "error", err) log.Fatal("Failed to remove temp file", "file", tempName, "error", err)
} }
} }
@@ -1343,6 +1469,7 @@ func (s *Scanner) processFileStreaming(ctx context.Context, fileToProcess *FileT
defer func() { _ = file.Close() }() defer func() { _ = file.Close() }()
var chunks []streamingChunkInfo var chunks []streamingChunkInfo
chunkIndex := 0 chunkIndex := 0
fileHash, err := s.chunker.ChunkReaderStreaming(file, func(chunk chunker.Chunk) error { fileHash, err := s.chunker.ChunkReaderStreaming(file, func(chunk chunker.Chunk) error {
@@ -1372,16 +1499,17 @@ func (s *Scanner) processFileStreaming(ctx context.Context, fileToProcess *FileT
s.updateChunkStats(chunkExists, chunk.Size, result) s.updateChunkStats(chunkExists, chunk.Size, result)
if !chunkExists { if !chunkExists {
if err := s.addChunkToPacker(chunk); err != nil { err := s.addChunkToPacker(chunk)
if err != nil {
return err return err
} }
} }
chunk.Data = nil chunk.Data = nil
chunkIndex++ chunkIndex++
return nil return nil
}) })
if err != nil { if err != nil {
return fmt.Errorf("chunking file: %w", err) return fmt.Errorf("chunking file: %w", err)
} }
@@ -1390,6 +1518,7 @@ func (s *Scanner) processFileStreaming(ctx context.Context, fileToProcess *FileT
"path", fileToProcess.Path, "file_hash", fileHash, "chunks", len(chunks)) "path", fileToProcess.Path, "file_hash", fileHash, "chunks", len(chunks))
s.queueFileForBatchInsert(ctx, fileToProcess, chunks) s.queueFileForBatchInsert(ctx, fileToProcess, chunks)
return nil return nil
} }
@@ -1397,6 +1526,7 @@ func (s *Scanner) processFileStreaming(ctx context.Context, fileToProcess *FileT
func (s *Scanner) updateChunkStats(chunkExists bool, chunkSize int64, result *ScanResult) { func (s *Scanner) updateChunkStats(chunkExists bool, chunkSize int64, result *ScanResult) {
if chunkExists { if chunkExists {
result.FilesSkipped++ result.FilesSkipped++
result.BytesSkipped += chunkSize result.BytesSkipped += chunkSize
if s.progress != nil { if s.progress != nil {
s.progress.GetStats().BytesSkipped.Add(chunkSize) s.progress.GetStats().BytesSkipped.Add(chunkSize)
@@ -1404,6 +1534,7 @@ func (s *Scanner) updateChunkStats(chunkExists bool, chunkSize int64, result *Sc
} else { } else {
result.ChunksCreated++ result.ChunksCreated++
result.BytesScanned += chunkSize result.BytesScanned += chunkSize
if s.progress != nil { if s.progress != nil {
s.progress.GetStats().ChunksCreated.Add(1) s.progress.GetStats().ChunksCreated.Add(1)
s.progress.GetStats().BytesProcessed.Add(chunkSize) s.progress.GetStats().BytesProcessed.Add(chunkSize)
@@ -1415,27 +1546,36 @@ func (s *Scanner) updateChunkStats(chunkExists bool, chunkSize int64, result *Sc
// addChunkToPacker adds a chunk to the blob packer, finalizing the current blob if needed // addChunkToPacker adds a chunk to the blob packer, finalizing the current blob if needed
func (s *Scanner) addChunkToPacker(chunk chunker.Chunk) error { func (s *Scanner) addChunkToPacker(chunk chunker.Chunk) error {
s.packerMu.Lock() s.packerMu.Lock()
err := s.packer.AddChunk(&blob.ChunkRef{Hash: chunk.Hash, Data: chunk.Data}) err := s.packer.AddChunk(&blob.ChunkRef{Hash: chunk.Hash, Data: chunk.Data})
if err == blob.ErrBlobSizeLimitExceeded { if errors.Is(err, blob.ErrBlobSizeLimitExceeded) {
if err := s.packer.FinalizeBlob(); err != nil { err := s.packer.FinalizeBlob()
if err != nil {
s.packerMu.Unlock() s.packerMu.Unlock()
return fmt.Errorf("finalizing blob: %w", err) return fmt.Errorf("finalizing blob: %w", err)
} }
if err := s.packer.AddChunk(&blob.ChunkRef{Hash: chunk.Hash, Data: chunk.Data}); err != nil {
err = s.packer.AddChunk(&blob.ChunkRef{Hash: chunk.Hash, Data: chunk.Data})
if err != nil {
s.packerMu.Unlock() s.packerMu.Unlock()
return fmt.Errorf("adding chunk after finalize: %w", err) return fmt.Errorf("adding chunk after finalize: %w", err)
} }
} else if err != nil { } else if err != nil {
s.packerMu.Unlock() s.packerMu.Unlock()
return fmt.Errorf("adding chunk to packer: %w", err) return fmt.Errorf("adding chunk to packer: %w", err)
} }
s.packerMu.Unlock() s.packerMu.Unlock()
return nil return nil
} }
// queueFileForBatchInsert builds file/chunk associations and queues the file for batch DB insert // queueFileForBatchInsert builds file/chunk associations and queues the file for batch DB insert
func (s *Scanner) queueFileForBatchInsert(ctx context.Context, fileToProcess *FileToProcess, chunks []streamingChunkInfo) { func (s *Scanner) queueFileForBatchInsert(ctx context.Context, fileToProcess *FileToProcess, chunks []streamingChunkInfo) {
fileChunks := make([]database.FileChunk, len(chunks)) fileChunks := make([]database.FileChunk, len(chunks))
chunkFiles := make([]database.ChunkFile, len(chunks)) chunkFiles := make([]database.ChunkFile, len(chunks))
for i, ci := range chunks { for i, ci := range chunks {
fileChunks[i] = database.FileChunk{ fileChunks[i] = database.FileChunk{
@@ -1489,7 +1629,7 @@ func (s *Scanner) detectDeletedFilesFromMap(ctx context.Context, knownFiles map[
} }
if result.FilesDeleted > 0 { if result.FilesDeleted > 0 {
s.ui.Info("Snapshot source files enumeration detected %s deleted files.", s.ui.Count(result.FilesDeleted)) s.ui.Infof("Snapshot source files enumeration detected %s deleted files.", s.ui.Count(result.FilesDeleted))
} }
return nil return nil
@@ -1503,6 +1643,7 @@ func wrapPermissionError(path string, err error) error {
if !errors.Is(err, os.ErrPermission) { if !errors.Is(err, os.ErrPermission) {
return err return err
} }
if runtime.GOOS == "darwin" { if runtime.GOOS == "darwin" {
return fmt.Errorf("cannot read %s: %w\n\n"+ return fmt.Errorf("cannot read %s: %w\n\n"+
"macOS is blocking access to this path. Grant Full Disk Access to your\n"+ "macOS is blocking access to this path. Grant Full Disk Access to your\n"+
@@ -1510,12 +1651,14 @@ func wrapPermissionError(path string, err error) error {
" System Settings → Privacy & Security → Full Disk Access\n\n"+ " System Settings → Privacy & Security → Full Disk Access\n\n"+
"then quit and reopen the terminal and re-run the backup", path, err) "then quit and reopen the terminal and re-run the backup", path, err)
} }
return fmt.Errorf("cannot read %s: %w (check file permissions, or run with --skip-errors to continue past unreadable files)", path, err) return fmt.Errorf("cannot read %s: %w (check file permissions, or run with --skip-errors to continue past unreadable files)", path, err)
} }
// compileExcludePatterns compiles the exclude patterns into glob matchers // compileExcludePatterns compiles the exclude patterns into glob matchers
func compileExcludePatterns(patterns []string) []compiledPattern { func compileExcludePatterns(patterns []string) []compiledPattern {
var compiled []compiledPattern var compiled []compiledPattern
for _, p := range patterns { for _, p := range patterns {
if p == "" { if p == "" {
continue continue
@@ -1523,6 +1666,7 @@ func compileExcludePatterns(patterns []string) []compiledPattern {
// Check if pattern is anchored (starts with /) // Check if pattern is anchored (starts with /)
anchored := strings.HasPrefix(p, "/") anchored := strings.HasPrefix(p, "/")
pattern := p pattern := p
if anchored { if anchored {
pattern = p[1:] // Remove leading / pattern = p[1:] // Remove leading /
@@ -1537,6 +1681,7 @@ func compileExcludePatterns(patterns []string) []compiledPattern {
g, err := glob.Compile(pattern, '/') g, err := glob.Compile(pattern, '/')
if err != nil { if err != nil {
log.Warn("Invalid exclude pattern, skipping", "pattern", p, "error", err) log.Warn("Invalid exclude pattern, skipping", "pattern", p, "error", err)
continue continue
} }
@@ -1546,6 +1691,7 @@ func compileExcludePatterns(patterns []string) []compiledPattern {
original: p, original: p,
}) })
} }
return compiled return compiled
} }

View File

@@ -33,16 +33,22 @@ func TestScannerSimpleDirectory(t *testing.T) {
// Create files with specific times // Create files with specific times
testTime := time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC) testTime := time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC)
for path, content := range testFiles { for path, content := range testFiles {
dir := filepath.Dir(path) dir := filepath.Dir(path)
if err := fs.MkdirAll(dir, 0755); err != nil {
err := fs.MkdirAll(dir, 0755)
if err != nil {
t.Fatalf("failed to create directory %s: %v", dir, err) t.Fatalf("failed to create directory %s: %v", dir, err)
} }
if err := afero.WriteFile(fs, path, []byte(content), 0644); err != nil {
err = afero.WriteFile(fs, path, []byte(content), 0644)
if err != nil {
t.Fatalf("failed to write file %s: %v", path, err) t.Fatalf("failed to write file %s: %v", path, err)
} }
// Set times // Set times
if err := fs.Chtimes(path, testTime, testTime); err != nil { err = fs.Chtimes(path, testTime, testTime)
if err != nil {
t.Fatalf("failed to set times for %s: %v", path, err) t.Fatalf("failed to set times for %s: %v", path, err)
} }
} }
@@ -53,7 +59,8 @@ func TestScannerSimpleDirectory(t *testing.T) {
t.Fatalf("failed to create test database: %v", err) t.Fatalf("failed to create test database: %v", err)
} }
defer func() { defer func() {
if err := db.Close(); err != nil { err := db.Close()
if err != nil {
t.Errorf("failed to close database: %v", err) t.Errorf("failed to close database: %v", err)
} }
}() }()
@@ -73,6 +80,7 @@ func TestScannerSimpleDirectory(t *testing.T) {
// Create a snapshot record for testing // Create a snapshot record for testing
ctx := context.Background() ctx := context.Background()
snapshotID := "test-snapshot-001" snapshotID := "test-snapshot-001"
err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
snapshot := &database.Snapshot{ snapshot := &database.Snapshot{
ID: types.SnapshotID(snapshotID), ID: types.SnapshotID(snapshotID),
@@ -87,6 +95,7 @@ func TestScannerSimpleDirectory(t *testing.T) {
BlobSize: 0, BlobSize: 0,
CompressionRatio: 1.0, CompressionRatio: 1.0,
} }
return repos.Snapshots.Create(ctx, tx, snapshot) return repos.Snapshots.Create(ctx, tx, snapshot)
}) })
if err != nil { if err != nil {
@@ -95,6 +104,7 @@ func TestScannerSimpleDirectory(t *testing.T) {
// Scan the directory // Scan the directory
var result *snapshot.ScanResult var result *snapshot.ScanResult
result, err = scanner.Scan(ctx, "/source", snapshotID) result, err = scanner.Scan(ctx, "/source", snapshotID)
if err != nil { if err != nil {
t.Fatalf("scan failed: %v", err) t.Fatalf("scan failed: %v", err)
@@ -170,7 +180,7 @@ func TestScannerLargeFile(t *testing.T) {
// Use random content to ensure good chunk boundaries // Use random content to ensure good chunk boundaries
largeContent := make([]byte, 1024*1024) // 1MB largeContent := make([]byte, 1024*1024) // 1MB
// Fill with pseudo-random data to ensure chunk boundaries // Fill with pseudo-random data to ensure chunk boundaries
for i := 0; i < len(largeContent); i++ { for i := range largeContent {
// Simple pseudo-random generator for deterministic tests // Simple pseudo-random generator for deterministic tests
largeContent[i] = byte((i * 7919) ^ (i >> 3)) largeContent[i] = byte((i * 7919) ^ (i >> 3))
} }
@@ -178,6 +188,7 @@ func TestScannerLargeFile(t *testing.T) {
if err := fs.MkdirAll("/source", 0755); err != nil { if err := fs.MkdirAll("/source", 0755); err != nil {
t.Fatal(err) t.Fatal(err)
} }
if err := afero.WriteFile(fs, "/source/large.bin", largeContent, 0644); err != nil { if err := afero.WriteFile(fs, "/source/large.bin", largeContent, 0644); err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -188,7 +199,8 @@ func TestScannerLargeFile(t *testing.T) {
t.Fatalf("failed to create test database: %v", err) t.Fatalf("failed to create test database: %v", err)
} }
defer func() { defer func() {
if err := db.Close(); err != nil { err := db.Close()
if err != nil {
t.Errorf("failed to close database: %v", err) t.Errorf("failed to close database: %v", err)
} }
}() }()
@@ -208,6 +220,7 @@ func TestScannerLargeFile(t *testing.T) {
// Create a snapshot record for testing // Create a snapshot record for testing
ctx := context.Background() ctx := context.Background()
snapshotID := "test-snapshot-001" snapshotID := "test-snapshot-001"
err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
snapshot := &database.Snapshot{ snapshot := &database.Snapshot{
ID: types.SnapshotID(snapshotID), ID: types.SnapshotID(snapshotID),
@@ -222,6 +235,7 @@ func TestScannerLargeFile(t *testing.T) {
BlobSize: 0, BlobSize: 0,
CompressionRatio: 1.0, CompressionRatio: 1.0,
} }
return repos.Snapshots.Create(ctx, tx, snapshot) return repos.Snapshots.Create(ctx, tx, snapshot)
}) })
if err != nil { if err != nil {
@@ -230,6 +244,7 @@ func TestScannerLargeFile(t *testing.T) {
// Scan the directory // Scan the directory
var result *snapshot.ScanResult var result *snapshot.ScanResult
result, err = scanner.Scan(ctx, "/source", snapshotID) result, err = scanner.Scan(ctx, "/source", snapshotID)
if err != nil { if err != nil {
t.Fatalf("scan failed: %v", err) t.Fatalf("scan failed: %v", err)

View File

@@ -37,6 +37,8 @@ import (
"bytes" "bytes"
"context" "context"
"database/sql" "database/sql"
"encoding/hex"
"errors"
"fmt" "fmt"
"io" "io"
"os/exec" "os/exec"
@@ -97,12 +99,13 @@ func (sm *SnapshotManager) CreateSnapshot(ctx context.Context, hostname, version
func (sm *SnapshotManager) CreateSnapshotWithName(ctx context.Context, hostname, name, version, gitRevision string) (string, error) { func (sm *SnapshotManager) CreateSnapshotWithName(ctx context.Context, hostname, name, version, gitRevision string) (string, error) {
// Use short hostname (strip domain if present) // Use short hostname (strip domain if present)
shortHostname := hostname shortHostname := hostname
if idx := strings.Index(hostname, "."); idx != -1 { if before, _, ok := strings.Cut(hostname, "."); ok {
shortHostname = hostname[:idx] shortHostname = before
} }
// Build snapshot ID with optional name // Build snapshot ID with optional name
timestamp := time.Now().UTC().Format("2006-01-02T15:04:05Z") timestamp := time.Now().UTC().Format("2006-01-02T15:04:05Z")
var snapshotID string var snapshotID string
if name != "" { if name != "" {
snapshotID = fmt.Sprintf("%s_%s_%s", shortHostname, name, timestamp) snapshotID = fmt.Sprintf("%s_%s_%s", shortHostname, name, timestamp)
@@ -128,12 +131,12 @@ func (sm *SnapshotManager) CreateSnapshotWithName(ctx context.Context, hostname,
err := sm.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { err := sm.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
return sm.repos.Snapshots.Create(ctx, tx, snapshot) return sm.repos.Snapshots.Create(ctx, tx, snapshot)
}) })
if err != nil { if err != nil {
return "", fmt.Errorf("creating snapshot: %w", err) return "", fmt.Errorf("creating snapshot: %w", err)
} }
log.Info("Created snapshot", "snapshot_id", snapshotID) log.Info("Created snapshot", "snapshot_id", snapshotID)
return snapshotID, nil return snapshotID, nil
} }
@@ -148,7 +151,6 @@ func (sm *SnapshotManager) UpdateSnapshotStats(ctx context.Context, snapshotID s
stats.BytesUploaded, stats.BytesUploaded,
) )
}) })
if err != nil { if err != nil {
return fmt.Errorf("updating snapshot stats: %w", err) return fmt.Errorf("updating snapshot stats: %w", err)
} }
@@ -161,13 +163,14 @@ func (sm *SnapshotManager) UpdateSnapshotStats(ctx context.Context, snapshotID s
func (sm *SnapshotManager) UpdateSnapshotStatsExtended(ctx context.Context, snapshotID string, stats ExtendedBackupStats) error { func (sm *SnapshotManager) UpdateSnapshotStatsExtended(ctx context.Context, snapshotID string, stats ExtendedBackupStats) error {
return sm.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { return sm.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
// First update basic stats // First update basic stats
if err := sm.repos.Snapshots.UpdateCounts(ctx, tx, snapshotID, err := sm.repos.Snapshots.UpdateCounts(ctx, tx, snapshotID,
int64(stats.FilesScanned), int64(stats.FilesScanned),
int64(stats.ChunksCreated), int64(stats.ChunksCreated),
int64(stats.BlobsCreated), int64(stats.BlobsCreated),
stats.BytesScanned, stats.BytesScanned,
stats.BytesUploaded, stats.BytesUploaded,
); err != nil { )
if err != nil {
return err return err
} }
@@ -190,18 +193,20 @@ func (sm *SnapshotManager) CompleteSnapshot(ctx context.Context, snapshotID stri
if err != nil { if err != nil {
return err return err
} }
if added > 0 { if added > 0 {
log.Info("Populated snapshot_blobs with dedup-referenced blobs", log.Info("Populated snapshot_blobs with dedup-referenced blobs",
"snapshot_id", snapshotID, "added", added) "snapshot_id", snapshotID, "added", added)
} }
return sm.repos.Snapshots.MarkComplete(ctx, tx, snapshotID) return sm.repos.Snapshots.MarkComplete(ctx, tx, snapshotID)
}) })
if err != nil { if err != nil {
return fmt.Errorf("marking snapshot complete: %w", err) return fmt.Errorf("marking snapshot complete: %w", err)
} }
log.Info("Completed snapshot", "snapshot_id", snapshotID) log.Info("Completed snapshot", "snapshot_id", snapshotID)
return nil return nil
} }
@@ -229,10 +234,13 @@ func (sm *SnapshotManager) ExportSnapshotMetadata(ctx context.Context, dbPath st
if err != nil { if err != nil {
return fmt.Errorf("creating temp dir: %w", err) return fmt.Errorf("creating temp dir: %w", err)
} }
log.Debug("Created temporary directory", "path", tempDir) log.Debug("Created temporary directory", "path", tempDir)
defer func() { defer func() {
log.Debug("Cleaning up temporary directory", "path", tempDir) log.Debug("Cleaning up temporary directory", "path", tempDir)
if err := sm.fs.RemoveAll(tempDir); err != nil {
err := sm.fs.RemoveAll(tempDir)
if err != nil {
log.Debug("Failed to remove temp dir", "path", tempDir, "error", err) log.Debug("Failed to remove temp dir", "path", tempDir, "error", err)
} }
}() }()
@@ -258,6 +266,7 @@ func (sm *SnapshotManager) ExportSnapshotMetadata(ctx context.Context, dbPath st
"snapshot_id", snapshotID, "snapshot_id", snapshotID,
"db_size", len(finalData), "db_size", len(finalData),
"manifest_size", len(blobManifest)) "manifest_size", len(blobManifest))
return nil return nil
} }
@@ -268,17 +277,21 @@ func (sm *SnapshotManager) prepareExportDB(ctx context.Context, dbPath, snapshot
// The main database should be closed at this point // The main database should be closed at this point
tempDBPath := filepath.Join(tempDir, "snapshot.db") tempDBPath := filepath.Join(tempDir, "snapshot.db")
log.Debug("Copying database to temporary location", "source", dbPath, "destination", tempDBPath) log.Debug("Copying database to temporary location", "source", dbPath, "destination", tempDBPath)
if err := sm.copyFile(dbPath, tempDBPath); err != nil { if err := sm.copyFile(dbPath, tempDBPath); err != nil {
return nil, "", fmt.Errorf("copying database: %w", err) return nil, "", fmt.Errorf("copying database: %w", err)
} }
log.Debug("Database copy complete", "size", sm.getFileSize(tempDBPath)) log.Debug("Database copy complete", "size", sm.getFileSize(tempDBPath))
// Step 2: Clean the temp database to only contain current snapshot data // Step 2: Clean the temp database to only contain current snapshot data
log.Debug("Cleaning temporary database", "snapshot_id", snapshotID) log.Debug("Cleaning temporary database", "snapshot_id", snapshotID)
stats, err := sm.cleanSnapshotDB(ctx, tempDBPath, snapshotID) stats, err := sm.cleanSnapshotDB(ctx, tempDBPath, snapshotID)
if err != nil { if err != nil {
return nil, "", fmt.Errorf("cleaning snapshot database: %w", err) return nil, "", fmt.Errorf("cleaning snapshot database: %w", err)
} }
log.Info("Temporary database cleanup complete", log.Info("Temporary database cleanup complete",
"db_path", tempDBPath, "db_path", tempDBPath,
"size_after_clean", humanize.Bytes(uint64(sm.getFileSize(tempDBPath))), "size_after_clean", humanize.Bytes(uint64(sm.getFileSize(tempDBPath))),
@@ -294,6 +307,7 @@ func (sm *SnapshotManager) prepareExportDB(ctx context.Context, dbPath, snapshot
if err := sm.vacuumDatabase(tempDBPath); err != nil { if err := sm.vacuumDatabase(tempDBPath); err != nil {
return nil, "", fmt.Errorf("vacuuming database: %w", err) return nil, "", fmt.Errorf("vacuuming database: %w", err)
} }
log.Debug("Database vacuumed", "size", humanize.Bytes(uint64(sm.getFileSize(tempDBPath)))) log.Debug("Database vacuumed", "size", humanize.Bytes(uint64(sm.getFileSize(tempDBPath))))
// Step 4: Compress and encrypt the binary database file // Step 4: Compress and encrypt the binary database file
@@ -301,6 +315,7 @@ func (sm *SnapshotManager) prepareExportDB(ctx context.Context, dbPath, snapshot
if err := sm.compressFile(tempDBPath, compressedPath); err != nil { if err := sm.compressFile(tempDBPath, compressedPath); err != nil {
return nil, "", fmt.Errorf("compressing database: %w", err) return nil, "", fmt.Errorf("compressing database: %w", err)
} }
log.Debug("Compression complete", log.Debug("Compression complete",
"original_size", humanize.Bytes(uint64(sm.getFileSize(tempDBPath))), "original_size", humanize.Bytes(uint64(sm.getFileSize(tempDBPath))),
"compressed_size", humanize.Bytes(uint64(sm.getFileSize(compressedPath)))) "compressed_size", humanize.Bytes(uint64(sm.getFileSize(compressedPath))))
@@ -327,9 +342,12 @@ func (sm *SnapshotManager) uploadSnapshotArtifacts(ctx context.Context, snapshot
dbKey := fmt.Sprintf("metadata/%s/db.zst.age", remoteKey) dbKey := fmt.Sprintf("metadata/%s/db.zst.age", remoteKey)
dbUploadStart := time.Now() dbUploadStart := time.Now()
if err := sm.storage.Put(ctx, dbKey, bytes.NewReader(dbData)); err != nil {
err := sm.storage.Put(ctx, dbKey, bytes.NewReader(dbData))
if err != nil {
return fmt.Errorf("uploading snapshot database: %w", err) return fmt.Errorf("uploading snapshot database: %w", err)
} }
dbUploadDuration := time.Since(dbUploadStart) dbUploadDuration := time.Since(dbUploadStart)
dbUploadSpeed := float64(len(dbData)) * 8 / dbUploadDuration.Seconds() // bits per second dbUploadSpeed := float64(len(dbData)) * 8 / dbUploadDuration.Seconds() // bits per second
log.Info("Uploaded snapshot database", log.Info("Uploaded snapshot database",
@@ -341,9 +359,12 @@ func (sm *SnapshotManager) uploadSnapshotArtifacts(ctx context.Context, snapshot
// Upload blob manifest (compressed only, not encrypted) // Upload blob manifest (compressed only, not encrypted)
manifestKey := fmt.Sprintf("metadata/%s/manifest.json.zst", remoteKey) manifestKey := fmt.Sprintf("metadata/%s/manifest.json.zst", remoteKey)
manifestUploadStart := time.Now() manifestUploadStart := time.Now()
if err := sm.storage.Put(ctx, manifestKey, bytes.NewReader(manifestData)); err != nil {
err = sm.storage.Put(ctx, manifestKey, bytes.NewReader(manifestData))
if err != nil {
return fmt.Errorf("uploading blob manifest: %w", err) return fmt.Errorf("uploading blob manifest: %w", err)
} }
manifestUploadDuration := time.Since(manifestUploadStart) manifestUploadDuration := time.Since(manifestUploadStart)
manifestUploadSpeed := float64(len(manifestData)) * 8 / manifestUploadDuration.Seconds() // bits per second manifestUploadSpeed := float64(len(manifestData)) * 8 / manifestUploadDuration.Seconds() // bits per second
log.Info("Uploaded blob manifest", log.Info("Uploaded blob manifest",
@@ -383,7 +404,8 @@ func (sm *SnapshotManager) cleanSnapshotDB(ctx context.Context, dbPath string, s
return nil, fmt.Errorf("opening temp database: %w", err) return nil, fmt.Errorf("opening temp database: %w", err)
} }
defer func() { defer func() {
if err := db.Close(); err != nil { err := db.Close()
if err != nil {
log.Debug("Failed to close temp database", "error", err) log.Debug("Failed to close temp database", "error", err)
} }
}() }()
@@ -394,7 +416,8 @@ func (sm *SnapshotManager) cleanSnapshotDB(ctx context.Context, dbPath string, s
return nil, fmt.Errorf("beginning transaction: %w", err) return nil, fmt.Errorf("beginning transaction: %w", err)
} }
defer func() { defer func() {
if rbErr := tx.Rollback(); rbErr != nil && rbErr != sql.ErrTxDone { rbErr := tx.Rollback()
if rbErr != nil && !errors.Is(rbErr, sql.ErrTxDone) {
log.Debug("Failed to rollback transaction", "error", rbErr) log.Debug("Failed to rollback transaction", "error", rbErr)
} }
}() }()
@@ -430,6 +453,7 @@ func (sm *SnapshotManager) cleanSnapshotDB(ctx context.Context, dbPath string, s
// Commit transaction // Commit transaction
log.Debug("[Temp DB Cleanup] Committing cleanup transaction") log.Debug("[Temp DB Cleanup] Committing cleanup transaction")
if err := tx.Commit(); err != nil { if err := tx.Commit(); err != nil {
return nil, fmt.Errorf("committing transaction: %w", err) return nil, fmt.Errorf("committing transaction: %w", err)
} }
@@ -439,23 +463,30 @@ func (sm *SnapshotManager) cleanSnapshotDB(ctx context.Context, dbPath string, s
// Count files // Count files
var fileCount int var fileCount int
err = db.QueryRowWithLog(ctx, "SELECT COUNT(*) FROM files").Scan(&fileCount) err = db.QueryRowWithLog(ctx, "SELECT COUNT(*) FROM files").Scan(&fileCount)
if err != nil { if err != nil {
return nil, fmt.Errorf("counting files: %w", err) return nil, fmt.Errorf("counting files: %w", err)
} }
stats.FileCount = fileCount stats.FileCount = fileCount
// Count chunks // Count chunks
var chunkCount int var chunkCount int
err = db.QueryRowWithLog(ctx, "SELECT COUNT(*) FROM chunks").Scan(&chunkCount) err = db.QueryRowWithLog(ctx, "SELECT COUNT(*) FROM chunks").Scan(&chunkCount)
if err != nil { if err != nil {
return nil, fmt.Errorf("counting chunks: %w", err) return nil, fmt.Errorf("counting chunks: %w", err)
} }
stats.ChunkCount = chunkCount stats.ChunkCount = chunkCount
// Count blobs and get sizes // Count blobs and get sizes
var blobCount int var (
var compressedSize, uncompressedSize sql.NullInt64 blobCount int
compressedSize, uncompressedSize sql.NullInt64
)
err = db.QueryRowWithLog(ctx, ` err = db.QueryRowWithLog(ctx, `
SELECT COUNT(*), COALESCE(SUM(compressed_size), 0), COALESCE(SUM(uncompressed_size), 0) SELECT COUNT(*), COALESCE(SUM(compressed_size), 0), COALESCE(SUM(uncompressed_size), 0)
FROM blobs FROM blobs
@@ -464,6 +495,7 @@ func (sm *SnapshotManager) cleanSnapshotDB(ctx context.Context, dbPath string, s
if err != nil { if err != nil {
return nil, fmt.Errorf("counting blobs and sizes: %w", err) return nil, fmt.Errorf("counting blobs and sizes: %w", err)
} }
stats.BlobCount = blobCount stats.BlobCount = blobCount
stats.CompressedSize = compressedSize.Int64 stats.CompressedSize = compressedSize.Int64
stats.UncompressedSize = uncompressedSize.Int64 stats.UncompressedSize = uncompressedSize.Int64
@@ -491,7 +523,8 @@ func (sm *SnapshotManager) compressFile(inputPath, outputPath string) error {
return fmt.Errorf("opening input file: %w", err) return fmt.Errorf("opening input file: %w", err)
} }
defer func() { defer func() {
if err := input.Close(); err != nil { err := input.Close()
if err != nil {
log.Debug("Failed to close input file", "path", inputPath, "error", err) log.Debug("Failed to close input file", "path", inputPath, "error", err)
} }
}() }()
@@ -501,13 +534,15 @@ func (sm *SnapshotManager) compressFile(inputPath, outputPath string) error {
return fmt.Errorf("creating output file: %w", err) return fmt.Errorf("creating output file: %w", err)
} }
defer func() { defer func() {
if err := output.Close(); err != nil { err := output.Close()
if err != nil {
log.Debug("Failed to close output file", "path", outputPath, "error", err) log.Debug("Failed to close output file", "path", outputPath, "error", err)
} }
}() }()
// Use blobgen for compression and encryption // Use blobgen for compression and encryption
log.Debug("Compressing and encrypting data") log.Debug("Compressing and encrypting data")
writer, err := blobgen.NewWriter(output, sm.config.CompressionLevel, sm.config.AgeRecipients) writer, err := blobgen.NewWriter(output, sm.config.CompressionLevel, sm.config.AgeRecipients)
if err != nil { if err != nil {
return fmt.Errorf("creating blobgen writer: %w", err) return fmt.Errorf("creating blobgen writer: %w", err)
@@ -517,7 +552,8 @@ func (sm *SnapshotManager) compressFile(inputPath, outputPath string) error {
writerClosed := false writerClosed := false
defer func() { defer func() {
if !writerClosed { if !writerClosed {
if err := writer.Close(); err != nil { err := writer.Close()
if err != nil {
log.Debug("Failed to close writer", "error", err) log.Debug("Failed to close writer", "error", err)
} }
} }
@@ -531,9 +567,10 @@ func (sm *SnapshotManager) compressFile(inputPath, outputPath string) error {
if err := writer.Close(); err != nil { if err := writer.Close(); err != nil {
return fmt.Errorf("closing writer: %w", err) return fmt.Errorf("closing writer: %w", err)
} }
writerClosed = true writerClosed = true
log.Debug("Compression complete", "hash", fmt.Sprintf("%x", writer.Sum256())) log.Debug("Compression complete", "hash", hex.EncodeToString(writer.Sum256()))
return nil return nil
} }
@@ -541,34 +578,42 @@ func (sm *SnapshotManager) compressFile(inputPath, outputPath string) error {
// copyFile copies a file from src to dst // copyFile copies a file from src to dst
func (sm *SnapshotManager) copyFile(src, dst string) error { func (sm *SnapshotManager) copyFile(src, dst string) error {
log.Debug("Opening source file for copy", "path", src) log.Debug("Opening source file for copy", "path", src)
sourceFile, err := sm.fs.Open(src) sourceFile, err := sm.fs.Open(src)
if err != nil { if err != nil {
return err return err
} }
defer func() { defer func() {
log.Debug("Closing source file", "path", src) log.Debug("Closing source file", "path", src)
if err := sourceFile.Close(); err != nil {
err := sourceFile.Close()
if err != nil {
log.Debug("Failed to close source file", "path", src, "error", err) log.Debug("Failed to close source file", "path", src, "error", err)
} }
}() }()
log.Debug("Creating destination file", "path", dst) log.Debug("Creating destination file", "path", dst)
destFile, err := sm.fs.Create(dst) destFile, err := sm.fs.Create(dst)
if err != nil { if err != nil {
return err return err
} }
defer func() { defer func() {
log.Debug("Closing destination file", "path", dst) log.Debug("Closing destination file", "path", dst)
if err := destFile.Close(); err != nil {
err := destFile.Close()
if err != nil {
log.Debug("Failed to close destination file", "path", dst, "error", err) log.Debug("Failed to close destination file", "path", dst, "error", err)
} }
}() }()
log.Debug("Copying file data") log.Debug("Copying file data")
n, err := io.Copy(destFile, sourceFile) n, err := io.Copy(destFile, sourceFile)
if err != nil { if err != nil {
return err return err
} }
log.Debug("File copy complete", "bytes_copied", n) log.Debug("File copy complete", "bytes_copied", n)
return nil return nil
@@ -576,7 +621,6 @@ func (sm *SnapshotManager) copyFile(src, dst string) error {
// generateBlobManifest creates a compressed JSON list of all blobs in the snapshot // generateBlobManifest creates a compressed JSON list of all blobs in the snapshot
func (sm *SnapshotManager) generateBlobManifest(ctx context.Context, dbPath string, snapshotID string) ([]byte, error) { func (sm *SnapshotManager) generateBlobManifest(ctx context.Context, dbPath string, snapshotID string) ([]byte, error) {
// Open the cleaned database using the database package // Open the cleaned database using the database package
db, err := database.New(ctx, dbPath) db, err := database.New(ctx, dbPath)
if err != nil { if err != nil {
@@ -589,10 +633,12 @@ func (sm *SnapshotManager) generateBlobManifest(ctx context.Context, dbPath stri
// Get all blobs for this snapshot // Get all blobs for this snapshot
log.Debug("Querying blobs for snapshot", "snapshot_id", snapshotID) log.Debug("Querying blobs for snapshot", "snapshot_id", snapshotID)
blobHashes, err := repos.Snapshots.GetBlobHashes(ctx, snapshotID) blobHashes, err := repos.Snapshots.GetBlobHashes(ctx, snapshotID)
if err != nil { if err != nil {
return nil, fmt.Errorf("getting snapshot blobs: %w", err) return nil, fmt.Errorf("getting snapshot blobs: %w", err)
} }
log.Debug("Found blobs", "count", len(blobHashes)) log.Debug("Found blobs", "count", len(blobHashes))
// Get blob details including sizes // Get blob details including sizes
@@ -603,8 +649,10 @@ func (sm *SnapshotManager) generateBlobManifest(ctx context.Context, dbPath stri
blob, err := repos.Blobs.GetByHash(ctx, hash) blob, err := repos.Blobs.GetByHash(ctx, hash)
if err != nil { if err != nil {
log.Warn("Failed to get blob details", "hash", hash, "error", err) log.Warn("Failed to get blob details", "hash", hash, "error", err)
continue continue
} }
if blob != nil { if blob != nil {
blobs = append(blobs, BlobInfo{ blobs = append(blobs, BlobInfo{
Hash: hash, Hash: hash,
@@ -648,6 +696,7 @@ func (sm *SnapshotManager) getFileSize(path string) int64 {
if err != nil { if err != nil {
return -1 return -1
} }
return info.Size() return info.Size()
} }
@@ -663,6 +712,7 @@ type BackupStats struct {
// ExtendedBackupStats contains additional statistics for comprehensive tracking // ExtendedBackupStats contains additional statistics for comprehensive tracking
type ExtendedBackupStats struct { type ExtendedBackupStats struct {
BackupStats BackupStats
BlobUncompressedSize int64 // Total uncompressed size of all referenced blobs BlobUncompressedSize int64 // Total uncompressed size of all referenced blobs
CompressionLevel int // Compression level used for this snapshot CompressionLevel int // Compression level used for this snapshot
UploadDurationMs int64 // Total milliseconds spent uploading to S3 UploadDurationMs int64 // Total milliseconds spent uploading to S3
@@ -682,6 +732,7 @@ func (sm *SnapshotManager) CleanupIncompleteSnapshots(ctx context.Context, hostn
if len(incompleteSnapshots) == 0 { if len(incompleteSnapshots) == 0 {
log.Debug("No incomplete snapshots found") log.Debug("No incomplete snapshots found")
return nil return nil
} }
@@ -692,14 +743,15 @@ func (sm *SnapshotManager) CleanupIncompleteSnapshots(ctx context.Context, hostn
// Check if metadata exists in storage (paths use the hashed // Check if metadata exists in storage (paths use the hashed
// remote key so we don't leak host info to the listing). // remote key so we don't leak host info to the listing).
metadataKey := fmt.Sprintf("metadata/%s/db.zst", RemoteSnapshotKey(snapshot.ID.String())) metadataKey := fmt.Sprintf("metadata/%s/db.zst", RemoteSnapshotKey(snapshot.ID.String()))
_, err := sm.storage.Stat(ctx, metadataKey)
_, err := sm.storage.Stat(ctx, metadataKey)
if err != nil { if err != nil {
// Metadata doesn't exist in S3 - this is an incomplete snapshot // Metadata doesn't exist in S3 - this is an incomplete snapshot
log.Info("Cleaning up incomplete snapshot record", "snapshot_id", snapshot.ID, "started_at", snapshot.StartedAt) log.Info("Cleaning up incomplete snapshot record", "snapshot_id", snapshot.ID, "started_at", snapshot.StartedAt)
// Delete the snapshot and all its associations // Delete the snapshot and all its associations
if err := sm.deleteSnapshot(ctx, snapshot.ID.String()); err != nil { err := sm.deleteSnapshot(ctx, snapshot.ID.String())
if err != nil {
return fmt.Errorf("deleting incomplete snapshot %s: %w", snapshot.ID, err) return fmt.Errorf("deleting incomplete snapshot %s: %w", snapshot.ID, err)
} }
@@ -708,7 +760,9 @@ func (sm *SnapshotManager) CleanupIncompleteSnapshots(ctx context.Context, hostn
// Metadata exists - this snapshot was completed but database wasn't updated // Metadata exists - this snapshot was completed but database wasn't updated
// This shouldn't happen in normal operation, but mark it complete // This shouldn't happen in normal operation, but mark it complete
log.Warn("Found snapshot with remote metadata but incomplete in database", "snapshot_id", snapshot.ID) log.Warn("Found snapshot with remote metadata but incomplete in database", "snapshot_id", snapshot.ID)
if err := sm.repos.Snapshots.MarkComplete(ctx, nil, snapshot.ID.String()); err != nil {
err := sm.repos.Snapshots.MarkComplete(ctx, nil, snapshot.ID.String())
if err != nil {
log.Error("Failed to mark snapshot as complete in database", "snapshot_id", snapshot.ID, "error", err) log.Error("Failed to mark snapshot as complete in database", "snapshot_id", snapshot.ID, "error", err)
} }
} }
@@ -720,28 +774,34 @@ func (sm *SnapshotManager) CleanupIncompleteSnapshots(ctx context.Context, hostn
// deleteSnapshot removes a snapshot and all its associations from the database // deleteSnapshot removes a snapshot and all its associations from the database
func (sm *SnapshotManager) deleteSnapshot(ctx context.Context, snapshotID string) error { func (sm *SnapshotManager) deleteSnapshot(ctx context.Context, snapshotID string) error {
// Delete snapshot_files entries // Delete snapshot_files entries
if err := sm.repos.Snapshots.DeleteSnapshotFiles(ctx, snapshotID); err != nil { err := sm.repos.Snapshots.DeleteSnapshotFiles(ctx, snapshotID)
if err != nil {
return fmt.Errorf("deleting snapshot files: %w", err) return fmt.Errorf("deleting snapshot files: %w", err)
} }
// Delete snapshot_blobs entries // Delete snapshot_blobs entries
if err := sm.repos.Snapshots.DeleteSnapshotBlobs(ctx, snapshotID); err != nil { err = sm.repos.Snapshots.DeleteSnapshotBlobs(ctx, snapshotID)
if err != nil {
return fmt.Errorf("deleting snapshot blobs: %w", err) return fmt.Errorf("deleting snapshot blobs: %w", err)
} }
// Delete uploads entries (has foreign key to snapshots without CASCADE) // Delete uploads entries (has foreign key to snapshots without CASCADE)
if err := sm.repos.Snapshots.DeleteSnapshotUploads(ctx, snapshotID); err != nil { err = sm.repos.Snapshots.DeleteSnapshotUploads(ctx, snapshotID)
if err != nil {
return fmt.Errorf("deleting snapshot uploads: %w", err) return fmt.Errorf("deleting snapshot uploads: %w", err)
} }
// Delete the snapshot itself // Delete the snapshot itself
if err := sm.repos.Snapshots.Delete(ctx, snapshotID); err != nil { err = sm.repos.Snapshots.Delete(ctx, snapshotID)
if err != nil {
return fmt.Errorf("deleting snapshot: %w", err) return fmt.Errorf("deleting snapshot: %w", err)
} }
// Clean up orphaned data // Clean up orphaned data
log.Debug("Cleaning up orphaned records in main database") log.Debug("Cleaning up orphaned records in main database")
if err := sm.CleanupOrphanedData(ctx); err != nil {
err = sm.CleanupOrphanedData(ctx)
if err != nil {
return fmt.Errorf("cleaning up orphaned data: %w", err) return fmt.Errorf("cleaning up orphaned data: %w", err)
} }
@@ -759,28 +819,36 @@ func (sm *SnapshotManager) CleanupOrphanedData(ctx context.Context) error {
// Delete orphaned files (files not in any snapshot) // Delete orphaned files (files not in any snapshot)
log.Debug("Deleting orphaned file records from database") log.Debug("Deleting orphaned file records from database")
if err := sm.repos.Files.DeleteOrphaned(ctx); err != nil {
err := sm.repos.Files.DeleteOrphaned(ctx)
if err != nil {
return fmt.Errorf("deleting orphaned files: %w", err) return fmt.Errorf("deleting orphaned files: %w", err)
} }
// Delete orphaned blobs (blobs not in any snapshot) // Delete orphaned blobs (blobs not in any snapshot)
// This will cascade delete blob_chunks for deleted blobs // This will cascade delete blob_chunks for deleted blobs
log.Debug("Deleting orphaned blob records from database") log.Debug("Deleting orphaned blob records from database")
if err := sm.repos.Blobs.DeleteOrphaned(ctx); err != nil {
err = sm.repos.Blobs.DeleteOrphaned(ctx)
if err != nil {
return fmt.Errorf("deleting orphaned blobs: %w", err) return fmt.Errorf("deleting orphaned blobs: %w", err)
} }
// Delete orphaned blob_chunks entries // Delete orphaned blob_chunks entries
// This handles cases where the blob still exists but chunks were deleted // This handles cases where the blob still exists but chunks were deleted
log.Debug("Deleting orphaned blob_chunks associations from database") log.Debug("Deleting orphaned blob_chunks associations from database")
if err := sm.repos.BlobChunks.DeleteOrphaned(ctx); err != nil {
err = sm.repos.BlobChunks.DeleteOrphaned(ctx)
if err != nil {
return fmt.Errorf("deleting orphaned blob_chunks: %w", err) return fmt.Errorf("deleting orphaned blob_chunks: %w", err)
} }
// Delete orphaned chunks (chunks not referenced by any file) // Delete orphaned chunks (chunks not referenced by any file)
// This must come after cleaning up blob_chunks to avoid foreign key violations // This must come after cleaning up blob_chunks to avoid foreign key violations
log.Debug("Deleting orphaned chunk records from database") log.Debug("Deleting orphaned chunk records from database")
if err := sm.repos.Chunks.DeleteOrphaned(ctx); err != nil {
err = sm.repos.Chunks.DeleteOrphaned(ctx)
if err != nil {
return fmt.Errorf("deleting orphaned chunks: %w", err) return fmt.Errorf("deleting orphaned chunks: %w", err)
} }
@@ -793,21 +861,26 @@ func (sm *SnapshotManager) deleteOtherSnapshots(ctx context.Context, tx *sql.Tx,
// First delete uploads that reference other snapshots (no CASCADE DELETE on this FK) // First delete uploads that reference other snapshots (no CASCADE DELETE on this FK)
database.LogSQL("Execute", "DELETE FROM uploads WHERE snapshot_id != ?", currentSnapshotID) database.LogSQL("Execute", "DELETE FROM uploads WHERE snapshot_id != ?", currentSnapshotID)
uploadResult, err := tx.ExecContext(ctx, "DELETE FROM uploads WHERE snapshot_id != ?", currentSnapshotID) uploadResult, err := tx.ExecContext(ctx, "DELETE FROM uploads WHERE snapshot_id != ?", currentSnapshotID)
if err != nil { if err != nil {
return fmt.Errorf("deleting uploads for other snapshots: %w", err) return fmt.Errorf("deleting uploads for other snapshots: %w", err)
} }
uploadsDeleted, _ := uploadResult.RowsAffected() uploadsDeleted, _ := uploadResult.RowsAffected()
log.Debug("[Temp DB Cleanup] Deleted upload records", "count", uploadsDeleted) log.Debug("[Temp DB Cleanup] Deleted upload records", "count", uploadsDeleted)
// Now we can safely delete the snapshots // Now we can safely delete the snapshots
database.LogSQL("Execute", "DELETE FROM snapshots WHERE id != ?", currentSnapshotID) database.LogSQL("Execute", "DELETE FROM snapshots WHERE id != ?", currentSnapshotID)
result, err := tx.ExecContext(ctx, "DELETE FROM snapshots WHERE id != ?", currentSnapshotID) result, err := tx.ExecContext(ctx, "DELETE FROM snapshots WHERE id != ?", currentSnapshotID)
if err != nil { if err != nil {
return fmt.Errorf("deleting other snapshots: %w", err) return fmt.Errorf("deleting other snapshots: %w", err)
} }
rowsAffected, _ := result.RowsAffected() rowsAffected, _ := result.RowsAffected()
log.Debug("[Temp DB Cleanup] Deleted snapshot records from database", "count", rowsAffected) log.Debug("[Temp DB Cleanup] Deleted snapshot records from database", "count", rowsAffected)
return nil return nil
} }
@@ -816,22 +889,27 @@ func (sm *SnapshotManager) deleteOrphanedSnapshotAssociations(ctx context.Contex
// Delete orphaned snapshot_files // Delete orphaned snapshot_files
log.Debug("[Temp DB Cleanup] Deleting orphaned snapshot_files associations") log.Debug("[Temp DB Cleanup] Deleting orphaned snapshot_files associations")
database.LogSQL("Execute", "DELETE FROM snapshot_files WHERE snapshot_id != ?", currentSnapshotID) database.LogSQL("Execute", "DELETE FROM snapshot_files WHERE snapshot_id != ?", currentSnapshotID)
result, err := tx.ExecContext(ctx, "DELETE FROM snapshot_files WHERE snapshot_id != ?", currentSnapshotID) result, err := tx.ExecContext(ctx, "DELETE FROM snapshot_files WHERE snapshot_id != ?", currentSnapshotID)
if err != nil { if err != nil {
return fmt.Errorf("deleting orphaned snapshot_files: %w", err) return fmt.Errorf("deleting orphaned snapshot_files: %w", err)
} }
rowsAffected, _ := result.RowsAffected() rowsAffected, _ := result.RowsAffected()
log.Debug("[Temp DB Cleanup] Deleted snapshot_files associations", "count", rowsAffected) log.Debug("[Temp DB Cleanup] Deleted snapshot_files associations", "count", rowsAffected)
// Delete orphaned snapshot_blobs // Delete orphaned snapshot_blobs
log.Debug("[Temp DB Cleanup] Deleting orphaned snapshot_blobs associations") log.Debug("[Temp DB Cleanup] Deleting orphaned snapshot_blobs associations")
database.LogSQL("Execute", "DELETE FROM snapshot_blobs WHERE snapshot_id != ?", currentSnapshotID) database.LogSQL("Execute", "DELETE FROM snapshot_blobs WHERE snapshot_id != ?", currentSnapshotID)
result, err = tx.ExecContext(ctx, "DELETE FROM snapshot_blobs WHERE snapshot_id != ?", currentSnapshotID) result, err = tx.ExecContext(ctx, "DELETE FROM snapshot_blobs WHERE snapshot_id != ?", currentSnapshotID)
if err != nil { if err != nil {
return fmt.Errorf("deleting orphaned snapshot_blobs: %w", err) return fmt.Errorf("deleting orphaned snapshot_blobs: %w", err)
} }
rowsAffected, _ = result.RowsAffected() rowsAffected, _ = result.RowsAffected()
log.Debug("[Temp DB Cleanup] Deleted snapshot_blobs associations", "count", rowsAffected) log.Debug("[Temp DB Cleanup] Deleted snapshot_blobs associations", "count", rowsAffected)
return nil return nil
} }
@@ -839,6 +917,7 @@ func (sm *SnapshotManager) deleteOrphanedSnapshotAssociations(ctx context.Contex
func (sm *SnapshotManager) deleteOrphanedFiles(ctx context.Context, tx *sql.Tx, currentSnapshotID string) error { func (sm *SnapshotManager) deleteOrphanedFiles(ctx context.Context, tx *sql.Tx, currentSnapshotID string) error {
log.Debug("[Temp DB Cleanup] Deleting file records not referenced by current snapshot") log.Debug("[Temp DB Cleanup] Deleting file records not referenced by current snapshot")
database.LogSQL("Execute", `DELETE FROM files WHERE NOT EXISTS (SELECT 1 FROM snapshot_files WHERE snapshot_files.file_id = files.id AND snapshot_files.snapshot_id = ?)`, currentSnapshotID) database.LogSQL("Execute", `DELETE FROM files WHERE NOT EXISTS (SELECT 1 FROM snapshot_files WHERE snapshot_files.file_id = files.id AND snapshot_files.snapshot_id = ?)`, currentSnapshotID)
result, err := tx.ExecContext(ctx, ` result, err := tx.ExecContext(ctx, `
DELETE FROM files DELETE FROM files
WHERE NOT EXISTS ( WHERE NOT EXISTS (
@@ -849,11 +928,13 @@ func (sm *SnapshotManager) deleteOrphanedFiles(ctx context.Context, tx *sql.Tx,
if err != nil { if err != nil {
return fmt.Errorf("deleting orphaned files: %w", err) return fmt.Errorf("deleting orphaned files: %w", err)
} }
rowsAffected, _ := result.RowsAffected() rowsAffected, _ := result.RowsAffected()
log.Debug("[Temp DB Cleanup] Deleted file records from database", "count", rowsAffected) log.Debug("[Temp DB Cleanup] Deleted file records from database", "count", rowsAffected)
// Note: file_chunks will be deleted via CASCADE // Note: file_chunks will be deleted via CASCADE
log.Debug("[Temp DB Cleanup] file_chunks associations deleted via CASCADE") log.Debug("[Temp DB Cleanup] file_chunks associations deleted via CASCADE")
return nil return nil
} }
@@ -861,6 +942,7 @@ func (sm *SnapshotManager) deleteOrphanedFiles(ctx context.Context, tx *sql.Tx,
func (sm *SnapshotManager) deleteOrphanedChunkToFileMappings(ctx context.Context, tx *sql.Tx) error { func (sm *SnapshotManager) deleteOrphanedChunkToFileMappings(ctx context.Context, tx *sql.Tx) error {
log.Debug("[Temp DB Cleanup] Deleting orphaned chunk_files associations") log.Debug("[Temp DB Cleanup] Deleting orphaned chunk_files associations")
database.LogSQL("Execute", `DELETE FROM chunk_files WHERE NOT EXISTS (SELECT 1 FROM files WHERE files.id = chunk_files.file_id)`) database.LogSQL("Execute", `DELETE FROM chunk_files WHERE NOT EXISTS (SELECT 1 FROM files WHERE files.id = chunk_files.file_id)`)
result, err := tx.ExecContext(ctx, ` result, err := tx.ExecContext(ctx, `
DELETE FROM chunk_files DELETE FROM chunk_files
WHERE NOT EXISTS ( WHERE NOT EXISTS (
@@ -870,8 +952,10 @@ func (sm *SnapshotManager) deleteOrphanedChunkToFileMappings(ctx context.Context
if err != nil { if err != nil {
return fmt.Errorf("deleting orphaned chunk_files: %w", err) return fmt.Errorf("deleting orphaned chunk_files: %w", err)
} }
rowsAffected, _ := result.RowsAffected() rowsAffected, _ := result.RowsAffected()
log.Debug("[Temp DB Cleanup] Deleted chunk_files associations", "count", rowsAffected) log.Debug("[Temp DB Cleanup] Deleted chunk_files associations", "count", rowsAffected)
return nil return nil
} }
@@ -879,6 +963,7 @@ func (sm *SnapshotManager) deleteOrphanedChunkToFileMappings(ctx context.Context
func (sm *SnapshotManager) deleteOrphanedBlobs(ctx context.Context, tx *sql.Tx, currentSnapshotID string) error { func (sm *SnapshotManager) deleteOrphanedBlobs(ctx context.Context, tx *sql.Tx, currentSnapshotID string) error {
log.Debug("[Temp DB Cleanup] Deleting blob records not referenced by current snapshot") log.Debug("[Temp DB Cleanup] Deleting blob records not referenced by current snapshot")
database.LogSQL("Execute", `DELETE FROM blobs WHERE NOT EXISTS (SELECT 1 FROM snapshot_blobs WHERE snapshot_blobs.blob_hash = blobs.blob_hash AND snapshot_blobs.snapshot_id = ?)`, currentSnapshotID) database.LogSQL("Execute", `DELETE FROM blobs WHERE NOT EXISTS (SELECT 1 FROM snapshot_blobs WHERE snapshot_blobs.blob_hash = blobs.blob_hash AND snapshot_blobs.snapshot_id = ?)`, currentSnapshotID)
result, err := tx.ExecContext(ctx, ` result, err := tx.ExecContext(ctx, `
DELETE FROM blobs DELETE FROM blobs
WHERE NOT EXISTS ( WHERE NOT EXISTS (
@@ -889,8 +974,10 @@ func (sm *SnapshotManager) deleteOrphanedBlobs(ctx context.Context, tx *sql.Tx,
if err != nil { if err != nil {
return fmt.Errorf("deleting orphaned blobs: %w", err) return fmt.Errorf("deleting orphaned blobs: %w", err)
} }
rowsAffected, _ := result.RowsAffected() rowsAffected, _ := result.RowsAffected()
log.Debug("[Temp DB Cleanup] Deleted blob records from database", "count", rowsAffected) log.Debug("[Temp DB Cleanup] Deleted blob records from database", "count", rowsAffected)
return nil return nil
} }
@@ -898,6 +985,7 @@ func (sm *SnapshotManager) deleteOrphanedBlobs(ctx context.Context, tx *sql.Tx,
func (sm *SnapshotManager) deleteOrphanedBlobToChunkMappings(ctx context.Context, tx *sql.Tx) error { func (sm *SnapshotManager) deleteOrphanedBlobToChunkMappings(ctx context.Context, tx *sql.Tx) error {
log.Debug("[Temp DB Cleanup] Deleting orphaned blob_chunks associations") log.Debug("[Temp DB Cleanup] Deleting orphaned blob_chunks associations")
database.LogSQL("Execute", `DELETE FROM blob_chunks WHERE NOT EXISTS (SELECT 1 FROM blobs WHERE blobs.id = blob_chunks.blob_id)`) database.LogSQL("Execute", `DELETE FROM blob_chunks WHERE NOT EXISTS (SELECT 1 FROM blobs WHERE blobs.id = blob_chunks.blob_id)`)
result, err := tx.ExecContext(ctx, ` result, err := tx.ExecContext(ctx, `
DELETE FROM blob_chunks DELETE FROM blob_chunks
WHERE NOT EXISTS ( WHERE NOT EXISTS (
@@ -907,14 +995,17 @@ func (sm *SnapshotManager) deleteOrphanedBlobToChunkMappings(ctx context.Context
if err != nil { if err != nil {
return fmt.Errorf("deleting orphaned blob_chunks: %w", err) return fmt.Errorf("deleting orphaned blob_chunks: %w", err)
} }
rowsAffected, _ := result.RowsAffected() rowsAffected, _ := result.RowsAffected()
log.Debug("[Temp DB Cleanup] Deleted blob_chunks associations", "count", rowsAffected) log.Debug("[Temp DB Cleanup] Deleted blob_chunks associations", "count", rowsAffected)
return nil return nil
} }
// deleteOrphanedChunks deletes chunks not referenced by any file or blob // deleteOrphanedChunks deletes chunks not referenced by any file or blob
func (sm *SnapshotManager) deleteOrphanedChunks(ctx context.Context, tx *sql.Tx) error { func (sm *SnapshotManager) deleteOrphanedChunks(ctx context.Context, tx *sql.Tx) error {
log.Debug("[Temp DB Cleanup] Deleting orphaned chunk records") log.Debug("[Temp DB Cleanup] Deleting orphaned chunk records")
query := ` query := `
DELETE FROM chunks DELETE FROM chunks
WHERE NOT EXISTS ( WHERE NOT EXISTS (
@@ -926,11 +1017,14 @@ func (sm *SnapshotManager) deleteOrphanedChunks(ctx context.Context, tx *sql.Tx)
WHERE blob_chunks.chunk_hash = chunks.chunk_hash WHERE blob_chunks.chunk_hash = chunks.chunk_hash
)` )`
database.LogSQL("Execute", query) database.LogSQL("Execute", query)
result, err := tx.ExecContext(ctx, query) result, err := tx.ExecContext(ctx, query)
if err != nil { if err != nil {
return fmt.Errorf("deleting orphaned chunks: %w", err) return fmt.Errorf("deleting orphaned chunks: %w", err)
} }
rowsAffected, _ := result.RowsAffected() rowsAffected, _ := result.RowsAffected()
log.Debug("[Temp DB Cleanup] Deleted chunk records from database", "count", rowsAffected) log.Debug("[Temp DB Cleanup] Deleted chunk records from database", "count", rowsAffected)
return nil return nil
} }

View File

@@ -33,6 +33,7 @@ func copyFile(fs afero.Fs, src, dst string) error {
defer func() { _ = destFile.Close() }() defer func() { _ = destFile.Close() }()
_, err = io.Copy(destFile, sourceFile) _, err = io.Copy(destFile, sourceFile)
return err return err
} }
@@ -46,6 +47,7 @@ func TestCleanSnapshotDBEmptySnapshot(t *testing.T) {
// Create a test database // Create a test database
tempDir := t.TempDir() tempDir := t.TempDir()
dbPath := filepath.Join(tempDir, "test.db") dbPath := filepath.Join(tempDir, "test.db")
db, err := database.New(ctx, dbPath) db, err := database.New(ctx, dbPath)
if err != nil { if err != nil {
t.Fatalf("failed to create database: %v", err) t.Fatalf("failed to create database: %v", err)
@@ -71,9 +73,11 @@ func TestCleanSnapshotDBEmptySnapshot(t *testing.T) {
chunk := &database.Chunk{ChunkHash: "orphan-chunk", Size: 500} chunk := &database.Chunk{ChunkHash: "orphan-chunk", Size: 500}
err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
if err := repos.Files.Create(ctx, tx, file); err != nil { err := repos.Files.Create(ctx, tx, file)
if err != nil {
return err return err
} }
return repos.Chunks.Create(ctx, tx, chunk) return repos.Chunks.Create(ctx, tx, chunk)
}) })
if err != nil { if err != nil {
@@ -111,7 +115,8 @@ func TestCleanSnapshotDBEmptySnapshot(t *testing.T) {
t.Fatalf("failed to open cleaned database: %v", err) t.Fatalf("failed to open cleaned database: %v", err)
} }
defer func() { defer func() {
if err := cleanedDB.Close(); err != nil { err := cleanedDB.Close()
if err != nil {
t.Errorf("failed to close database: %v", err) t.Errorf("failed to close database: %v", err)
} }
}() }()
@@ -123,6 +128,7 @@ func TestCleanSnapshotDBEmptySnapshot(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("failed to get snapshot: %v", err) t.Fatalf("failed to get snapshot: %v", err)
} }
if verifySnapshot == nil { if verifySnapshot == nil {
t.Error("snapshot should exist") t.Error("snapshot should exist")
} }
@@ -132,6 +138,7 @@ func TestCleanSnapshotDBEmptySnapshot(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("failed to check file: %v", err) t.Fatalf("failed to check file: %v", err)
} }
if f != nil { if f != nil {
t.Error("orphan file should not exist") t.Error("orphan file should not exist")
} }
@@ -141,6 +148,7 @@ func TestCleanSnapshotDBEmptySnapshot(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("failed to check chunk: %v", err) t.Fatalf("failed to check chunk: %v", err)
} }
if c != nil { if c != nil {
t.Error("orphan chunk should not exist") t.Error("orphan chunk should not exist")
} }
@@ -156,6 +164,7 @@ func TestCleanSnapshotDBNonExistentSnapshot(t *testing.T) {
// Create a test database // Create a test database
tempDir := t.TempDir() tempDir := t.TempDir()
dbPath := filepath.Join(tempDir, "test.db") dbPath := filepath.Join(tempDir, "test.db")
db, err := database.New(ctx, dbPath) db, err := database.New(ctx, dbPath)
if err != nil { if err != nil {
t.Fatalf("failed to create database: %v", err) t.Fatalf("failed to create database: %v", err)

View File

@@ -102,26 +102,32 @@ func (f *FileStorer) PutWithProgress(ctx context.Context, key string, data io.Re
// Get retrieves data from the specified key. // Get retrieves data from the specified key.
func (f *FileStorer) Get(ctx context.Context, key string) (io.ReadCloser, error) { func (f *FileStorer) Get(ctx context.Context, key string) (io.ReadCloser, error) {
path := f.fullPath(key) path := f.fullPath(key)
file, err := f.fs.Open(path) file, err := f.fs.Open(path)
if err != nil { if err != nil {
if os.IsNotExist(err) { if os.IsNotExist(err) {
return nil, ErrNotFound return nil, ErrNotFound
} }
return nil, fmt.Errorf("opening file: %w", err) return nil, fmt.Errorf("opening file: %w", err)
} }
return file, nil return file, nil
} }
// Stat returns metadata about an object without retrieving its contents. // Stat returns metadata about an object without retrieving its contents.
func (f *FileStorer) Stat(ctx context.Context, key string) (*ObjectInfo, error) { func (f *FileStorer) Stat(ctx context.Context, key string) (*ObjectInfo, error) {
path := f.fullPath(key) path := f.fullPath(key)
info, err := f.fs.Stat(path) info, err := f.fs.Stat(path)
if err != nil { if err != nil {
if os.IsNotExist(err) { if os.IsNotExist(err) {
return nil, ErrNotFound return nil, ErrNotFound
} }
return nil, fmt.Errorf("stat file: %w", err) return nil, fmt.Errorf("stat file: %w", err)
} }
return &ObjectInfo{ return &ObjectInfo{
Key: key, Key: key,
Size: info.Size(), Size: info.Size(),
@@ -131,19 +137,23 @@ func (f *FileStorer) Stat(ctx context.Context, key string) (*ObjectInfo, error)
// Delete removes an object. // Delete removes an object.
func (f *FileStorer) Delete(ctx context.Context, key string) error { func (f *FileStorer) Delete(ctx context.Context, key string) error {
path := f.fullPath(key) path := f.fullPath(key)
err := f.fs.Remove(path) err := f.fs.Remove(path)
if os.IsNotExist(err) { if os.IsNotExist(err) {
return nil // Match S3 behavior: no error if doesn't exist return nil // Match S3 behavior: no error if doesn't exist
} }
if err != nil { if err != nil {
return fmt.Errorf("removing file: %w", err) return fmt.Errorf("removing file: %w", err)
} }
return nil return nil
} }
// List returns all keys with the given prefix. // List returns all keys with the given prefix.
func (f *FileStorer) List(ctx context.Context, prefix string) ([]string, error) { func (f *FileStorer) List(ctx context.Context, prefix string) ([]string, error) {
var keys []string var keys []string
basePath := f.fullPath(prefix) basePath := f.fullPath(prefix)
// Check if base path exists // Check if base path exists
@@ -151,6 +161,7 @@ func (f *FileStorer) List(ctx context.Context, prefix string) ([]string, error)
if err != nil { if err != nil {
return nil, fmt.Errorf("checking path: %w", err) return nil, fmt.Errorf("checking path: %w", err)
} }
if !exists { if !exists {
return keys, nil // Empty list for non-existent prefix return keys, nil // Empty list for non-existent prefix
} }
@@ -177,9 +188,9 @@ func (f *FileStorer) List(ctx context.Context, prefix string) ([]string, error)
relPath = strings.ReplaceAll(relPath, string(filepath.Separator), "/") relPath = strings.ReplaceAll(relPath, string(filepath.Separator), "/")
keys = append(keys, relPath) keys = append(keys, relPath)
} }
return nil return nil
}) })
if err != nil { if err != nil {
return nil, fmt.Errorf("walking directory: %w", err) return nil, fmt.Errorf("walking directory: %w", err)
} }
@@ -192,14 +203,17 @@ func (f *FileStorer) ListStream(ctx context.Context, prefix string) <-chan Objec
ch := make(chan ObjectInfo) ch := make(chan ObjectInfo)
go func() { go func() {
defer close(ch) defer close(ch)
basePath := f.fullPath(prefix) basePath := f.fullPath(prefix)
// Check if base path exists // Check if base path exists
exists, err := afero.Exists(f.fs, basePath) exists, err := afero.Exists(f.fs, basePath)
if err != nil { if err != nil {
ch <- ObjectInfo{Err: fmt.Errorf("checking path: %w", err)} ch <- ObjectInfo{Err: fmt.Errorf("checking path: %w", err)}
return return
} }
if !exists { if !exists {
return // Empty channel for non-existent prefix return // Empty channel for non-existent prefix
} }
@@ -209,12 +223,14 @@ func (f *FileStorer) ListStream(ctx context.Context, prefix string) <-chan Objec
select { select {
case <-ctx.Done(): case <-ctx.Done():
ch <- ObjectInfo{Err: ctx.Err()} ch <- ObjectInfo{Err: ctx.Err()}
return ctx.Err() return ctx.Err()
default: default:
} }
if err != nil { if err != nil {
ch <- ObjectInfo{Err: err} ch <- ObjectInfo{Err: err}
return nil // Continue walking despite errors return nil // Continue walking despite errors
} }
@@ -222,6 +238,7 @@ func (f *FileStorer) ListStream(ctx context.Context, prefix string) <-chan Objec
relPath, err := filepath.Rel(f.basePath, path) relPath, err := filepath.Rel(f.basePath, path)
if err != nil { if err != nil {
ch <- ObjectInfo{Err: fmt.Errorf("computing relative path: %w", err)} ch <- ObjectInfo{Err: fmt.Errorf("computing relative path: %w", err)}
return nil return nil
} }
// Normalize path separators // Normalize path separators
@@ -231,9 +248,11 @@ func (f *FileStorer) ListStream(ctx context.Context, prefix string) <-chan Objec
Size: info.Size(), Size: info.Size(),
} }
} }
return nil return nil
}) })
}() }()
return ch return ch
} }
@@ -257,10 +276,12 @@ func (pw *progressWriter) Write(p []byte) (int, error) {
if n > 0 { if n > 0 {
pw.written += int64(n) pw.written += int64(n)
if pw.callback != nil { if pw.callback != nil {
if callbackErr := pw.callback(pw.written); callbackErr != nil { callbackErr := pw.callback(pw.written)
if callbackErr != nil {
return n, callbackErr return n, callbackErr
} }
} }
} }
return n, err return n, err
} }

View File

@@ -24,6 +24,7 @@ func NewStorer(cfg *config.Config) (Storer, error) {
if cfg.StorageURL != "" { if cfg.StorageURL != "" {
return storerFromURL(cfg.StorageURL, cfg) return storerFromURL(cfg.StorageURL, cfg)
} }
return storerFromLegacyS3Config(cfg) return storerFromLegacyS3Config(cfg)
} }
@@ -71,6 +72,7 @@ func storerFromURL(rawURL string, cfg *config.Config) (Storer, error) {
if err != nil { if err != nil {
return nil, fmt.Errorf("creating S3 client: %w", err) return nil, fmt.Errorf("creating S3 client: %w", err)
} }
return NewS3Storer(client), nil return NewS3Storer(client), nil
case "rclone": case "rclone":
@@ -109,5 +111,6 @@ func storerFromLegacyS3Config(cfg *config.Config) (Storer, error) {
if err != nil { if err != nil {
return nil, fmt.Errorf("creating S3 client: %w", err) return nil, fmt.Errorf("creating S3 client: %w", err)
} }
return NewS3Storer(client), nil return NewS3Storer(client), nil
} }

View File

@@ -49,6 +49,7 @@ func NewRcloneStorer(ctx context.Context, remote, path string) (*RcloneStorer, e
strings.Contains(err.Error(), "failed to find remote") { strings.Contains(err.Error(), "failed to find remote") {
return nil, fmt.Errorf("%w: %s", ErrRemoteNotFound, remote) return nil, fmt.Errorf("%w: %s", ErrRemoteNotFound, remote)
} }
return nil, fmt.Errorf("creating rclone filesystem: %w", err) return nil, fmt.Errorf("creating rclone filesystem: %w", err)
} }
@@ -101,9 +102,11 @@ func (r *RcloneStorer) Get(ctx context.Context, key string) (io.ReadCloser, erro
if errors.Is(err, fs.ErrorObjectNotFound) { if errors.Is(err, fs.ErrorObjectNotFound) {
return nil, ErrNotFound return nil, ErrNotFound
} }
if errors.Is(err, fs.ErrorDirNotFound) { if errors.Is(err, fs.ErrorDirNotFound) {
return nil, ErrNotFound return nil, ErrNotFound
} }
return nil, fmt.Errorf("getting object: %w", err) return nil, fmt.Errorf("getting object: %w", err)
} }
@@ -123,9 +126,11 @@ func (r *RcloneStorer) Stat(ctx context.Context, key string) (*ObjectInfo, error
if errors.Is(err, fs.ErrorObjectNotFound) { if errors.Is(err, fs.ErrorObjectNotFound) {
return nil, ErrNotFound return nil, ErrNotFound
} }
if errors.Is(err, fs.ErrorDirNotFound) { if errors.Is(err, fs.ErrorDirNotFound) {
return nil, ErrNotFound return nil, ErrNotFound
} }
return nil, fmt.Errorf("getting object: %w", err) return nil, fmt.Errorf("getting object: %w", err)
} }
@@ -142,9 +147,11 @@ func (r *RcloneStorer) Delete(ctx context.Context, key string) error {
if errors.Is(err, fs.ErrorObjectNotFound) { if errors.Is(err, fs.ErrorObjectNotFound) {
return nil // Match S3 behavior: no error if doesn't exist return nil // Match S3 behavior: no error if doesn't exist
} }
if errors.Is(err, fs.ErrorDirNotFound) { if errors.Is(err, fs.ErrorDirNotFound) {
return nil return nil
} }
return fmt.Errorf("getting object: %w", err) return fmt.Errorf("getting object: %w", err)
} }
@@ -209,6 +216,7 @@ func (r *RcloneStorer) Info() StorageInfo {
if r.path != "" { if r.path != "" {
location += ":" + r.path location += ":" + r.path
} }
return StorageInfo{ return StorageInfo{
Type: "rclone", Type: "rclone",
Location: location, Location: location,
@@ -227,10 +235,12 @@ func (pr *progressReader) Read(p []byte) (int, error) {
if n > 0 { if n > 0 {
pr.read += int64(n) pr.read += int64(n)
if pr.callback != nil { if pr.callback != nil {
if callbackErr := pr.callback(pr.read); callbackErr != nil { callbackErr := pr.callback(pr.read)
if callbackErr != nil {
return n, callbackErr return n, callbackErr
} }
} }
} }
return n, err return n, err
} }

View File

@@ -30,6 +30,7 @@ func (s *S3Storer) PutWithProgress(ctx context.Context, key string, data io.Read
if progress != nil { if progress != nil {
s3Progress = s3.ProgressCallback(progress) s3Progress = s3.ProgressCallback(progress)
} }
return s.client.PutObjectWithProgress(ctx, key, data, size, s3Progress) return s.client.PutObjectWithProgress(ctx, key, data, size, s3Progress)
} }
@@ -44,6 +45,7 @@ func (s *S3Storer) Stat(ctx context.Context, key string) (*ObjectInfo, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
return &ObjectInfo{ return &ObjectInfo{
Key: info.Key, Key: info.Key,
Size: info.Size, Size: info.Size,
@@ -65,6 +67,7 @@ func (s *S3Storer) ListStream(ctx context.Context, prefix string) <-chan ObjectI
ch := make(chan ObjectInfo) ch := make(chan ObjectInfo)
go func() { go func() {
defer close(ch) defer close(ch)
for info := range s.client.ListObjectsStream(ctx, prefix, false) { for info := range s.client.ListObjectsStream(ctx, prefix, false) {
ch <- ObjectInfo{ ch <- ObjectInfo{
Key: info.Key, Key: info.Key,
@@ -73,6 +76,7 @@ func (s *S3Storer) ListStream(ctx context.Context, prefix string) <-chan ObjectI
} }
} }
}() }()
return ch return ch
} }

View File

@@ -1,6 +1,7 @@
package storage package storage
import ( import (
"errors"
"fmt" "fmt"
"net/url" "net/url"
"strings" "strings"
@@ -24,15 +25,16 @@ type StorageURL struct {
// - rclone://remote/path/to/backups // - rclone://remote/path/to/backups
func ParseStorageURL(rawURL string) (*StorageURL, error) { func ParseStorageURL(rawURL string) (*StorageURL, error) {
if rawURL == "" { if rawURL == "" {
return nil, fmt.Errorf("storage URL is empty") return nil, errors.New("storage URL is empty")
} }
// Handle file:// URLs // Handle file:// URLs
if strings.HasPrefix(rawURL, "file://") { if after, ok := strings.CutPrefix(rawURL, "file://"); ok {
path := strings.TrimPrefix(rawURL, "file://") path := after
if path == "" { if path == "" {
return nil, fmt.Errorf("file URL path is empty") return nil, errors.New("file URL path is empty")
} }
return &StorageURL{ return &StorageURL{
Scheme: "file", Scheme: "file",
Prefix: path, Prefix: path,
@@ -48,12 +50,13 @@ func ParseStorageURL(rawURL string) (*StorageURL, error) {
bucket := u.Host bucket := u.Host
if bucket == "" { if bucket == "" {
return nil, fmt.Errorf("s3 URL missing bucket name") return nil, errors.New("s3 URL missing bucket name")
} }
prefix := strings.TrimPrefix(u.Path, "/") prefix := strings.TrimPrefix(u.Path, "/")
query := u.Query() query := u.Query()
useSSL := true useSSL := true
if query.Get("ssl") == "false" { if query.Get("ssl") == "false" {
useSSL = false useSSL = false
@@ -78,7 +81,7 @@ func ParseStorageURL(rawURL string) (*StorageURL, error) {
remote := u.Host remote := u.Host
if remote == "" { if remote == "" {
return nil, fmt.Errorf("rclone URL missing remote name") return nil, errors.New("rclone URL missing remote name")
} }
path := strings.TrimPrefix(u.Path, "/") path := strings.TrimPrefix(u.Path, "/")
@@ -90,29 +93,32 @@ func ParseStorageURL(rawURL string) (*StorageURL, error) {
}, nil }, nil
} }
return nil, fmt.Errorf("unsupported URL scheme: must start with s3://, file://, or rclone://") return nil, errors.New("unsupported URL scheme: must start with s3://, file://, or rclone://")
} }
// String returns a human-readable representation of the storage URL. // String returns a human-readable representation of the storage URL.
func (u *StorageURL) String() string { func (u *StorageURL) String() string {
switch u.Scheme { switch u.Scheme {
case "file": case "file":
return fmt.Sprintf("file://%s", u.Prefix) return "file://" + u.Prefix
case "s3": case "s3":
endpoint := u.Endpoint endpoint := u.Endpoint
if endpoint == "" { if endpoint == "" {
endpoint = "s3.amazonaws.com" endpoint = "s3.amazonaws.com"
} }
if u.Prefix != "" { if u.Prefix != "" {
return fmt.Sprintf("s3://%s/%s (endpoint: %s)", u.Bucket, u.Prefix, endpoint) return fmt.Sprintf("s3://%s/%s (endpoint: %s)", u.Bucket, u.Prefix, endpoint)
} }
return fmt.Sprintf("s3://%s (endpoint: %s)", u.Bucket, endpoint) return fmt.Sprintf("s3://%s (endpoint: %s)", u.Bucket, endpoint)
case "rclone": case "rclone":
if u.Prefix != "" { if u.Prefix != "" {
return fmt.Sprintf("rclone://%s/%s", u.RcloneRemote, u.Prefix) return fmt.Sprintf("rclone://%s/%s", u.RcloneRemote, u.Prefix)
} }
return fmt.Sprintf("rclone://%s", u.RcloneRemote)
return "rclone://" + u.RcloneRemote
default: default:
return fmt.Sprintf("%s://?", u.Scheme) return u.Scheme + "://?"
} }
} }

View File

@@ -24,6 +24,7 @@ func ParseFileID(s string) (FileID, error) {
if err != nil { if err != nil {
return FileID{}, err return FileID{}, err
} }
return FileID(id), nil return FileID(id), nil
} }
@@ -38,13 +39,15 @@ func (id FileID) Value() (driver.Value, error) {
} }
// Scan implements sql.Scanner for database deserialization. // Scan implements sql.Scanner for database deserialization.
func (id *FileID) Scan(src interface{}) error { func (id *FileID) Scan(src any) error {
if src == nil { if src == nil {
*id = FileID{} *id = FileID{}
return nil return nil
} }
var s string var s string
switch v := src.(type) { switch v := src.(type) {
case string: case string:
s = v s = v
@@ -58,7 +61,9 @@ func (id *FileID) Scan(src interface{}) error {
if err != nil { if err != nil {
return fmt.Errorf("invalid FileID: %w", err) return fmt.Errorf("invalid FileID: %w", err)
} }
*id = FileID(parsed) *id = FileID(parsed)
return nil return nil
} }
@@ -77,6 +82,7 @@ func ParseBlobID(s string) (BlobID, error) {
if err != nil { if err != nil {
return BlobID{}, err return BlobID{}, err
} }
return BlobID(id), nil return BlobID(id), nil
} }
@@ -91,13 +97,15 @@ func (id BlobID) Value() (driver.Value, error) {
} }
// Scan implements sql.Scanner for database deserialization. // Scan implements sql.Scanner for database deserialization.
func (id *BlobID) Scan(src interface{}) error { func (id *BlobID) Scan(src any) error {
if src == nil { if src == nil {
*id = BlobID{} *id = BlobID{}
return nil return nil
} }
var s string var s string
switch v := src.(type) { switch v := src.(type) {
case string: case string:
s = v s = v
@@ -111,7 +119,9 @@ func (id *BlobID) Scan(src interface{}) error {
if err != nil { if err != nil {
return fmt.Errorf("invalid BlobID: %w", err) return fmt.Errorf("invalid BlobID: %w", err)
} }
*id = BlobID(parsed) *id = BlobID(parsed)
return nil return nil
} }

View File

@@ -94,10 +94,12 @@ func shouldColor(w io.Writer) bool {
if os.Getenv("NO_COLOR") != "" { if os.Getenv("NO_COLOR") != "" {
return false return false
} }
f, ok := w.(*os.File) f, ok := w.(*os.File)
if !ok { if !ok {
return false return false
} }
return term.IsTerminal(int(f.Fd())) return term.IsTerminal(int(f.Fd()))
} }
@@ -106,67 +108,73 @@ func (w *Writer) paint(color, s string) string {
if !w.color { if !w.color {
return s return s
} }
return color + s + ansiReset return color + s + ansiReset
} }
// ───────────────────────── message methods ───────────────────────── // ───────────────────────── message methods ─────────────────────────
// Begin prints an operation-start line, left-aligned with a white marker. // Beginf prints an operation-start line, left-aligned with a white marker.
func (w *Writer) Begin(format string, args ...any) { func (w *Writer) Beginf(format string, args ...any) {
if w.quiet { if w.quiet {
return return
} }
w.emit(ansiWhite, Marker, "", format, args) w.emit(ansiWhite, Marker, "", format, args)
} }
// Complete prints an operation-completion line in green, left-aligned. // Completef prints an operation-completion line in green, left-aligned.
func (w *Writer) Complete(format string, args ...any) { func (w *Writer) Completef(format string, args ...any) {
if w.quiet { if w.quiet {
return return
} }
w.emit(ansiGreen, Marker, ansiGreen, format, args) w.emit(ansiGreen, Marker, ansiGreen, format, args)
} }
// Info prints a neutral status line, left-aligned with a white marker. // Infof prints a neutral status line, left-aligned with a white marker.
func (w *Writer) Info(format string, args ...any) { func (w *Writer) Infof(format string, args ...any) {
if w.quiet { if w.quiet {
return return
} }
w.emit(ansiWhite, Marker, "", format, args) w.emit(ansiWhite, Marker, "", format, args)
} }
// Notice prints an attention-worthy informational line, marker in cyan. // Noticef prints an attention-worthy informational line, marker in cyan.
func (w *Writer) Notice(format string, args ...any) { func (w *Writer) Noticef(format string, args ...any) {
if w.quiet { if w.quiet {
return return
} }
w.emit(ansiCyan, Marker, "", format, args) w.emit(ansiCyan, Marker, "", format, args)
} }
// Warning prints "⚠️ Warning: " in orange/yellow followed by the message. // Warningf prints "⚠️ Warning: " in orange/yellow followed by the message.
func (w *Writer) Warning(format string, args ...any) { func (w *Writer) Warningf(format string, args ...any) {
w.warnings++ w.warnings++
prefix := "⚠️ " + w.paint(ansiYellow+ansiBold, "Warning: ") prefix := "⚠️ " + w.paint(ansiYellow+ansiBold, "Warning: ")
_, _ = fmt.Fprintln(w.out, prefix+fmt.Sprintf(format, args...)) _, _ = fmt.Fprintln(w.out, prefix+fmt.Sprintf(format, args...))
} }
// Error prints "🛑 ERROR: " in red followed by the message. Goes to the // Errorf prints "🛑 ERROR: " in red followed by the message. Goes to the
// same writer as everything else; callers that want stderr should // same writer as everything else; callers that want stderr should
// construct a separate Writer for it. // construct a separate Writer for it.
func (w *Writer) Error(format string, args ...any) { func (w *Writer) Errorf(format string, args ...any) {
w.errors++ w.errors++
prefix := "🛑 " + w.paint(ansiRed+ansiBold, "ERROR: ") prefix := "🛑 " + w.paint(ansiRed+ansiBold, "ERROR: ")
_, _ = fmt.Fprintln(w.out, prefix+fmt.Sprintf(format, args...)) _, _ = fmt.Fprintln(w.out, prefix+fmt.Sprintf(format, args...))
} }
// Detail prints an indented continuation line under a preceding Complete // Detailf prints an indented continuation line under a preceding Completef
// (or other top-level message). Marker " 》" (white) at column 2. // (or other top-level message). Marker " 》" (white) at column 2.
// Distinct from Progress (semantically a "heartbeat") in usage but // Distinct from Progressf (semantically a "heartbeat") in usage but
// visually identical. // visually identical.
func (w *Writer) Detail(format string, args ...any) { func (w *Writer) Detailf(format string, args ...any) {
if w.quiet { if w.quiet {
return return
} }
w.emit(ansiWhite, " "+Marker, "", format, args) w.emit(ansiWhite, " "+Marker, "", format, args)
} }
@@ -176,24 +184,27 @@ func (w *Writer) WarningCount() int { return w.warnings }
// ErrorCount returns the number of Error() calls this writer has emitted. // ErrorCount returns the number of Error() calls this writer has emitted.
func (w *Writer) ErrorCount() int { return w.errors } func (w *Writer) ErrorCount() int { return w.errors }
// Progress prints an indented heartbeat / per-item update, marker in white. // Progressf prints an indented heartbeat / per-item update, marker in white.
func (w *Writer) Progress(format string, args ...any) { func (w *Writer) Progressf(format string, args ...any) {
if w.quiet { if w.quiet {
return return
} }
w.emit(ansiWhite, " "+Marker, "", format, args) w.emit(ansiWhite, " "+Marker, "", format, args)
} }
// Banner prints a line with no marker, left-aligned. Bold when color // Bannerf prints a line with no marker, left-aligned. Bold when color
// is enabled. Used for the application startup banner only. // is enabled. Used for the application startup banner only.
func (w *Writer) Banner(format string, args ...any) { func (w *Writer) Bannerf(format string, args ...any) {
if w.quiet { if w.quiet {
return return
} }
body := fmt.Sprintf(format, args...) body := fmt.Sprintf(format, args...)
if w.color { if w.color {
body = ansiBold + body + ansiReset body = ansiBold + body + ansiReset
} }
_, _ = fmt.Fprintln(w.out, body) _, _ = fmt.Fprintln(w.out, body)
} }
@@ -204,6 +215,7 @@ func (w *Writer) emit(prefixColor, prefix, bodyColor, format string, args []any)
if bodyColor != "" { if bodyColor != "" {
body = w.paint(bodyColor, body) body = w.paint(bodyColor, body)
} }
_, _ = fmt.Fprintln(w.out, w.paint(prefixColor, prefix)+" "+body) _, _ = fmt.Fprintln(w.out, w.paint(prefixColor, prefix)+" "+body)
} }
@@ -219,6 +231,7 @@ func (w *Writer) Hex(s string) string {
if len(s) > 12 { if len(s) > 12 {
short = s[:12] + "..." short = s[:12] + "..."
} }
return w.paint(ansiCyan, short) return w.paint(ansiCyan, short)
} }
@@ -244,8 +257,11 @@ func (w *Writer) Speed(bytesPerSec float64) string {
if bytesPerSec <= 0 { if bytesPerSec <= 0 {
return w.paint(ansiMagenta, "N/A") return w.paint(ansiMagenta, "N/A")
} }
bitsPerSec := bytesPerSec * 8 bitsPerSec := bytesPerSec * 8
var s string var s string
switch { switch {
case bitsPerSec >= 1e9: case bitsPerSec >= 1e9:
s = fmt.Sprintf("%.1f Gbit/sec", bitsPerSec/1e9) s = fmt.Sprintf("%.1f Gbit/sec", bitsPerSec/1e9)
@@ -256,6 +272,7 @@ func (w *Writer) Speed(bytesPerSec float64) string {
default: default:
s = fmt.Sprintf("%.0f bit/sec", bitsPerSec) s = fmt.Sprintf("%.0f bit/sec", bitsPerSec)
} }
return w.paint(ansiMagenta, s) return w.paint(ansiMagenta, s)
} }
@@ -270,10 +287,12 @@ func (w *Writer) Duration(d time.Duration) string {
// displayed in the process's local zone. // displayed in the process's local zone.
func (w *Writer) Time(t time.Time) string { func (w *Writer) Time(t time.Time) string {
t = t.Local() t = t.Local()
now := time.Now() now := time.Now()
if t.Year() == now.Year() && t.YearDay() == now.YearDay() { if t.Year() == now.Year() && t.YearDay() == now.YearDay() {
return w.paint(ansiYellow, t.Format("15:04:05")) return w.paint(ansiYellow, t.Format("15:04:05"))
} }
return w.paint(ansiYellow, t.Format("2006-01-02 15:04:05")) return w.paint(ansiYellow, t.Format("2006-01-02 15:04:05"))
} }

View File

@@ -9,6 +9,7 @@ import (
func newTestWriter(color bool) (*Writer, *bytes.Buffer) { func newTestWriter(color bool) (*Writer, *bytes.Buffer) {
buf := &bytes.Buffer{} buf := &bytes.Buffer{}
return NewWithColor(buf, color), buf return NewWithColor(buf, color), buf
} }
@@ -18,21 +19,22 @@ func TestMessageMethodsPlain(t *testing.T) {
fn func(*Writer) fn func(*Writer)
want string want string
}{ }{
{"Begin", func(w *Writer) { w.Begin("starting %s", "thing") }, "》 starting thing\n"}, {"Begin", func(w *Writer) { w.Beginf("starting %s", "thing") }, "》 starting thing\n"},
{"Complete", func(w *Writer) { w.Complete("done %s", "thing") }, "》 done thing\n"}, {"Complete", func(w *Writer) { w.Completef("done %s", "thing") }, "》 done thing\n"},
{"Info", func(w *Writer) { w.Info("status") }, "》 status\n"}, {"Info", func(w *Writer) { w.Infof("status") }, "》 status\n"},
{"Notice", func(w *Writer) { w.Notice("note") }, "》 note\n"}, {"Notice", func(w *Writer) { w.Noticef("note") }, "》 note\n"},
{"Warning", func(w *Writer) { w.Warning("oops") }, "⚠️ Warning: oops\n"}, {"Warning", func(w *Writer) { w.Warningf("oops") }, "⚠️ Warning: oops\n"},
{"Error", func(w *Writer) { w.Error("boom") }, "🛑 ERROR: boom\n"}, {"Error", func(w *Writer) { w.Errorf("boom") }, "🛑 ERROR: boom\n"},
{"Progress", func(w *Writer) { w.Progress("p") }, " 》 p\n"}, {"Progress", func(w *Writer) { w.Progressf("p") }, " 》 p\n"},
{"Detail", func(w *Writer) { w.Detail("d") }, " 》 d\n"}, {"Detail", func(w *Writer) { w.Detailf("d") }, " 》 d\n"},
{"Banner", func(w *Writer) { w.Banner("hello") }, "hello\n"}, // plain mode, no bold {"Banner", func(w *Writer) { w.Bannerf("hello") }, "hello\n"}, // plain mode, no bold
} }
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.method, func(t *testing.T) { t.Run(tt.method, func(t *testing.T) {
w, buf := newTestWriter(false) w, buf := newTestWriter(false)
tt.fn(w) tt.fn(w)
if got := buf.String(); got != tt.want { if got := buf.String(); got != tt.want {
t.Errorf("got %q, want %q", got, tt.want) t.Errorf("got %q, want %q", got, tt.want)
} }
@@ -45,13 +47,16 @@ func TestWarningErrorCounters(t *testing.T) {
if w.WarningCount() != 0 || w.ErrorCount() != 0 { if w.WarningCount() != 0 || w.ErrorCount() != 0 {
t.Fatalf("expected fresh writer to have zero counts") t.Fatalf("expected fresh writer to have zero counts")
} }
w.Info("normal")
w.Warning("first warn") w.Infof("normal")
w.Warning("second warn") w.Warningf("first warn")
w.Error("only error") w.Warningf("second warn")
w.Errorf("only error")
if got, want := w.WarningCount(), 2; got != want { if got, want := w.WarningCount(), 2; got != want {
t.Errorf("WarningCount: got %d, want %d", got, want) t.Errorf("WarningCount: got %d, want %d", got, want)
} }
if got, want := w.ErrorCount(), 1; got != want { if got, want := w.ErrorCount(), 1; got != want {
t.Errorf("ErrorCount: got %d, want %d", got, want) t.Errorf("ErrorCount: got %d, want %d", got, want)
} }
@@ -59,11 +64,13 @@ func TestWarningErrorCounters(t *testing.T) {
func TestColorOutputContainsANSI(t *testing.T) { func TestColorOutputContainsANSI(t *testing.T) {
w, buf := newTestWriter(true) w, buf := newTestWriter(true)
w.Error("boom") w.Errorf("boom")
out := buf.String() out := buf.String()
if !strings.Contains(out, "\033[") { if !strings.Contains(out, "\033[") {
t.Errorf("expected ANSI escapes in color output, got %q", out) t.Errorf("expected ANSI escapes in color output, got %q", out)
} }
if !strings.Contains(out, "ERROR: ") { if !strings.Contains(out, "ERROR: ") {
t.Errorf("expected 'ERROR: ' text in output, got %q", out) t.Errorf("expected 'ERROR: ' text in output, got %q", out)
} }
@@ -71,7 +78,8 @@ func TestColorOutputContainsANSI(t *testing.T) {
func TestBannerBoldWhenColor(t *testing.T) { func TestBannerBoldWhenColor(t *testing.T) {
w, buf := newTestWriter(true) w, buf := newTestWriter(true)
w.Banner("hello") w.Bannerf("hello")
out := buf.String() out := buf.String()
if !strings.Contains(out, "\033[1m") { if !strings.Contains(out, "\033[1m") {
t.Errorf("expected bold ANSI escape in colored Banner output, got %q", out) t.Errorf("expected bold ANSI escape in colored Banner output, got %q", out)
@@ -84,18 +92,23 @@ func TestValueFormattersPlain(t *testing.T) {
if got := w.Hex("0123456789abcdef0123"); got != "0123456789ab..." { if got := w.Hex("0123456789abcdef0123"); got != "0123456789ab..." {
t.Errorf("Hex long: got %q", got) t.Errorf("Hex long: got %q", got)
} }
if got := w.Hex("short"); got != "short" { if got := w.Hex("short"); got != "short" {
t.Errorf("Hex short: got %q", got) t.Errorf("Hex short: got %q", got)
} }
if got := w.Size(1024); got != "1.0 kB" { if got := w.Size(1024); got != "1.0 kB" {
t.Errorf("Size: got %q", got) t.Errorf("Size: got %q", got)
} }
if got := w.Duration(90 * time.Second); got != "1m30s" { if got := w.Duration(90 * time.Second); got != "1m30s" {
t.Errorf("Duration: got %q", got) t.Errorf("Duration: got %q", got)
} }
if got := w.Count(12345); got != "12,345" { if got := w.Count(12345); got != "12,345" {
t.Errorf("Count: got %q", got) t.Errorf("Count: got %q", got)
} }
if got := w.Percent(12.34); got != "12.3%" { if got := w.Percent(12.34); got != "12.3%" {
t.Errorf("Percent: got %q", got) t.Errorf("Percent: got %q", got)
} }
@@ -104,9 +117,11 @@ func TestValueFormattersPlain(t *testing.T) {
if got := w.Speed(0); got != "N/A" { if got := w.Speed(0); got != "N/A" {
t.Errorf("Speed(0): got %q, want N/A", got) t.Errorf("Speed(0): got %q, want N/A", got)
} }
if got := w.Speed(125_000_000); got != "1.0 Gbit/sec" { // 1 Gbit/s = 125 MB/s if got := w.Speed(125_000_000); got != "1.0 Gbit/sec" { // 1 Gbit/s = 125 MB/s
t.Errorf("Speed(125e6): got %q", got) t.Errorf("Speed(125e6): got %q", got)
} }
if got := w.Speed(125_000); got != "1 Mbit/sec" { if got := w.Speed(125_000); got != "1 Mbit/sec" {
t.Errorf("Speed(125e3): got %q", got) t.Errorf("Speed(125e3): got %q", got)
} }
@@ -116,6 +131,7 @@ func TestValueFormattersPlain(t *testing.T) {
if got := w.Time(today); got != "14:30:45" { if got := w.Time(today); got != "14:30:45" {
t.Errorf("Time today: got %q, want 14:30:45", got) t.Errorf("Time today: got %q, want 14:30:45", got)
} }
other := time.Date(2030, 1, 2, 3, 4, 5, 0, time.Local) other := time.Date(2030, 1, 2, 3, 4, 5, 0, time.Local)
if got := w.Time(other); got != "2030-01-02 03:04:05" { if got := w.Time(other); got != "2030-01-02 03:04:05" {
t.Errorf("Time other day: got %q", got) t.Errorf("Time other day: got %q", got)
@@ -124,10 +140,12 @@ func TestValueFormattersPlain(t *testing.T) {
func TestValueFormattersColored(t *testing.T) { func TestValueFormattersColored(t *testing.T) {
w, _ := newTestWriter(true) w, _ := newTestWriter(true)
hex := w.Hex("0123456789abcdef0123") hex := w.Hex("0123456789abcdef0123")
if !strings.Contains(hex, "\033[") { if !strings.Contains(hex, "\033[") {
t.Errorf("expected ANSI in colored Hex output, got %q", hex) t.Errorf("expected ANSI in colored Hex output, got %q", hex)
} }
if !strings.Contains(hex, "0123456789ab") { if !strings.Contains(hex, "0123456789ab") {
t.Errorf("expected hex content in output, got %q", hex) t.Errorf("expected hex content in output, got %q", hex)
} }

View File

@@ -4,6 +4,7 @@ import (
"context" "context"
"crypto/sha256" "crypto/sha256"
"encoding/hex" "encoding/hex"
"errors"
"fmt" "fmt"
"io" "io"
"time" "time"
@@ -26,9 +27,10 @@ type hashVerifyReader struct {
func (h *hashVerifyReader) Read(p []byte) (int, error) { func (h *hashVerifyReader) Read(p []byte) (int, error) {
n, err := h.reader.Read(p) n, err := h.reader.Read(p)
if err == io.EOF { if errors.Is(err, io.EOF) {
h.done = true h.done = true
} }
return n, err return n, err
} }
@@ -41,6 +43,7 @@ func (h *hashVerifyReader) Close() error {
firstHash := h.reader.Sum256() firstHash := h.reader.Sum256()
secondHasher := sha256.New() secondHasher := sha256.New()
secondHasher.Write(firstHash) secondHasher.Write(firstHash)
actualHashHex := hex.EncodeToString(secondHasher.Sum(nil)) actualHashHex := hex.EncodeToString(secondHasher.Sum(nil))
if actualHashHex != h.blobHash { if actualHashHex != h.blobHash {
return fmt.Errorf("blob hash mismatch: expected %s, got %s", h.blobHash[:16], actualHashHex[:16]) return fmt.Errorf("blob hash mismatch: expected %s, got %s", h.blobHash[:16], actualHashHex[:16])
@@ -50,6 +53,7 @@ func (h *hashVerifyReader) Close() error {
if readerErr != nil { if readerErr != nil {
return readerErr return readerErr
} }
return fetcherErr return fetcherErr
} }
@@ -66,6 +70,7 @@ func (v *Vaultik) FetchAndDecryptBlob(ctx context.Context, blobHash string, expe
reader, err := blobgen.NewReader(rc, identity) reader, err := blobgen.NewReader(rc, identity)
if err != nil { if err != nil {
_ = rc.Close() _ = rc.Close()
return nil, fmt.Errorf("creating blob reader: %w", err) return nil, fmt.Errorf("creating blob reader: %w", err)
} }
@@ -86,6 +91,7 @@ func (v *Vaultik) FetchBlob(ctx context.Context, blobHash string, expectedSize i
t0 := time.Now() t0 := time.Now()
rc, err := v.Storage.Get(ctx, blobPath) rc, err := v.Storage.Get(ctx, blobPath)
getDur := time.Since(t0) getDur := time.Since(t0)
if err != nil { if err != nil {
return nil, 0, fmt.Errorf("downloading blob %s: %w", blobHash[:16], err) return nil, 0, fmt.Errorf("downloading blob %s: %w", blobHash[:16], err)
} }
@@ -93,8 +99,10 @@ func (v *Vaultik) FetchBlob(ctx context.Context, blobHash string, expectedSize i
t0 = time.Now() t0 = time.Now()
info, err := v.Storage.Stat(ctx, blobPath) info, err := v.Storage.Stat(ctx, blobPath)
statDur := time.Since(t0) statDur := time.Since(t0)
if err != nil { if err != nil {
_ = rc.Close() _ = rc.Close()
return nil, 0, fmt.Errorf("stat blob %s: %w", blobHash[:16], err) return nil, 0, fmt.Errorf("stat blob %s: %w", blobHash[:16], err)
} }

View File

@@ -24,17 +24,22 @@ func TestFetchAndDecryptBlobVerifiesHash(t *testing.T) {
// Create test data and encrypt it using blobgen.Writer // Create test data and encrypt it using blobgen.Writer
plaintext := []byte("hello world test data for blob hash verification") plaintext := []byte("hello world test data for blob hash verification")
var encBuf bytes.Buffer var encBuf bytes.Buffer
writer, err := blobgen.NewWriter(&encBuf, 1, []string{identity.Recipient().String()}) writer, err := blobgen.NewWriter(&encBuf, 1, []string{identity.Recipient().String()})
if err != nil { if err != nil {
t.Fatalf("creating blobgen writer: %v", err) t.Fatalf("creating blobgen writer: %v", err)
} }
if _, err := writer.Write(plaintext); err != nil { if _, err := writer.Write(plaintext); err != nil {
t.Fatalf("writing plaintext: %v", err) t.Fatalf("writing plaintext: %v", err)
} }
if err := writer.Close(); err != nil { if err := writer.Close(); err != nil {
t.Fatalf("closing writer: %v", err) t.Fatalf("closing writer: %v", err)
} }
encryptedData := encBuf.Bytes() encryptedData := encBuf.Bytes()
// Compute correct double-SHA-256 hash of the plaintext (matches blobgen.Writer.Sum256) // Compute correct double-SHA-256 hash of the plaintext (matches blobgen.Writer.Sum256)
@@ -51,6 +56,7 @@ func TestFetchAndDecryptBlobVerifiesHash(t *testing.T) {
// Set up mock storage with the blob at the correct path // Set up mock storage with the blob at the correct path
mockStorage := NewMockStorer() mockStorage := NewMockStorer()
blobPath := "blobs/" + correctHash[:2] + "/" + correctHash[2:4] + "/" + correctHash blobPath := "blobs/" + correctHash[:2] + "/" + correctHash[2:4] + "/" + correctHash
mockStorage.mu.Lock() mockStorage.mu.Lock()
mockStorage.data[blobPath] = encryptedData mockStorage.data[blobPath] = encryptedData
mockStorage.mu.Unlock() mockStorage.mu.Unlock()
@@ -63,13 +69,16 @@ func TestFetchAndDecryptBlobVerifiesHash(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("expected success, got error: %v", err) t.Fatalf("expected success, got error: %v", err)
} }
data, err := io.ReadAll(rc) data, err := io.ReadAll(rc)
if err != nil { if err != nil {
t.Fatalf("reading stream: %v", err) t.Fatalf("reading stream: %v", err)
} }
if err := rc.Close(); err != nil { if err := rc.Close(); err != nil {
t.Fatalf("close (hash verification) failed: %v", err) t.Fatalf("close (hash verification) failed: %v", err)
} }
if !bytes.Equal(data, plaintext) { if !bytes.Equal(data, plaintext) {
t.Fatalf("decrypted data mismatch: got %q, want %q", data, plaintext) t.Fatalf("decrypted data mismatch: got %q, want %q", data, plaintext)
} }
@@ -79,6 +88,7 @@ func TestFetchAndDecryptBlobVerifiesHash(t *testing.T) {
// Use a fake hash that doesn't match the actual plaintext // Use a fake hash that doesn't match the actual plaintext
fakeHash := strings.Repeat("ab", 32) // 64 hex chars fakeHash := strings.Repeat("ab", 32) // 64 hex chars
fakePath := "blobs/" + fakeHash[:2] + "/" + fakeHash[2:4] + "/" + fakeHash fakePath := "blobs/" + fakeHash[:2] + "/" + fakeHash[2:4] + "/" + fakeHash
mockStorage.mu.Lock() mockStorage.mu.Lock()
mockStorage.data[fakePath] = encryptedData mockStorage.data[fakePath] = encryptedData
mockStorage.mu.Unlock() mockStorage.mu.Unlock()
@@ -89,10 +99,12 @@ func TestFetchAndDecryptBlobVerifiesHash(t *testing.T) {
} }
// Read all data — hash is verified on Close // Read all data — hash is verified on Close
_, _ = io.ReadAll(rc) _, _ = io.ReadAll(rc)
err = rc.Close() err = rc.Close()
if err == nil { if err == nil {
t.Fatal("expected error for mismatched hash, got nil") t.Fatal("expected error for mismatched hash, got nil")
} }
if !strings.Contains(err.Error(), "hash mismatch") { if !strings.Contains(err.Error(), "hash mismatch") {
t.Fatalf("expected hash mismatch error, got: %v", err) t.Fatalf("expected hash mismatch error, got: %v", err)
} }

View File

@@ -53,6 +53,7 @@ func newBlobDiskCache(maxBytes int64) (*blobDiskCache, error) {
if err != nil { if err != nil {
return nil, fmt.Errorf("creating blob cache dir: %w", err) return nil, fmt.Errorf("creating blob cache dir: %w", err)
} }
return &blobDiskCache{ return &blobDiskCache{
dir: dir, dir: dir,
maxBytes: maxBytes, maxBytes: maxBytes,
@@ -70,21 +71,25 @@ func (c *blobDiskCache) unlink(e *blobDiskCacheEntry) {
} else { } else {
c.head = e.next c.head = e.next
} }
if e.next != nil { if e.next != nil {
e.next.prev = e.prev e.next.prev = e.prev
} else { } else {
c.tail = e.prev c.tail = e.prev
} }
e.prev = nil e.prev = nil
e.next = nil e.next = nil
} }
func (c *blobDiskCache) pushFront(e *blobDiskCacheEntry) { func (c *blobDiskCache) pushFront(e *blobDiskCacheEntry) {
e.prev = nil e.prev = nil
e.next = c.head e.next = c.head
if c.head != nil { if c.head != nil {
c.head.prev = e c.head.prev = e
} }
c.head = e c.head = e
if c.tail == nil { if c.tail == nil {
c.tail = e c.tail = e
@@ -95,6 +100,7 @@ func (c *blobDiskCache) evictLRU() {
if c.tail == nil { if c.tail == nil {
return return
} }
victim := c.tail victim := c.tail
c.unlink(victim) c.unlink(victim)
delete(c.items, victim.key) delete(c.items, victim.key)
@@ -121,7 +127,8 @@ func (c *blobDiskCache) Put(key string, data []byte) error {
delete(c.items, key) delete(c.items, key)
} }
if err := os.WriteFile(c.path(key), data, 0600); err != nil { err := os.WriteFile(c.path(key), data, 0600)
if err != nil {
return fmt.Errorf("writing blob to cache: %w", err) return fmt.Errorf("writing blob to cache: %w", err)
} }
@@ -163,14 +170,19 @@ func (c *blobDiskCache) PutFromReader(key string, r io.Reader) (int64, error) {
if err != nil { if err != nil {
return 0, fmt.Errorf("creating cache file: %w", err) return 0, fmt.Errorf("creating cache file: %w", err)
} }
written, copyErr := io.Copy(f, r) written, copyErr := io.Copy(f, r)
closeErr := f.Close() closeErr := f.Close()
if copyErr != nil { if copyErr != nil {
_ = os.Remove(c.path(key)) _ = os.Remove(c.path(key))
return written, fmt.Errorf("streaming to cache file: %w", copyErr) return written, fmt.Errorf("streaming to cache file: %w", copyErr)
} }
if closeErr != nil { if closeErr != nil {
_ = os.Remove(c.path(key)) _ = os.Remove(c.path(key))
return written, fmt.Errorf("closing cache file: %w", closeErr) return written, fmt.Errorf("closing cache file: %w", closeErr)
} }
@@ -182,6 +194,7 @@ func (c *blobDiskCache) PutFromReader(key string, r io.Reader) (int64, error) {
// so this branch is effectively unreachable there. // so this branch is effectively unreachable there.
if written > c.maxBytes { if written > c.maxBytes {
_ = os.Remove(c.path(key)) _ = os.Remove(c.path(key))
return written, nil return written, nil
} }
@@ -205,11 +218,14 @@ func (c *blobDiskCache) PutFromReader(key string, r io.Reader) (int64, error) {
func (c *blobDiskCache) Get(key string) ([]byte, bool) { func (c *blobDiskCache) Get(key string) ([]byte, bool) {
c.mu.Lock() c.mu.Lock()
c.getCalls++ c.getCalls++
e, ok := c.items[key] e, ok := c.items[key]
if !ok { if !ok {
c.mu.Unlock() c.mu.Unlock()
return nil, false return nil, false
} }
c.unlink(e) c.unlink(e)
c.pushFront(e) c.pushFront(e)
c.mu.Unlock() c.mu.Unlock()
@@ -223,8 +239,10 @@ func (c *blobDiskCache) Get(key string) ([]byte, bool) {
c.curBytes -= e.size c.curBytes -= e.size
} }
c.mu.Unlock() c.mu.Unlock()
return nil, false return nil, false
} }
return data, true return data, true
} }
@@ -232,15 +250,20 @@ func (c *blobDiskCache) Get(key string) ([]byte, bool) {
func (c *blobDiskCache) ReadAt(key string, offset, length int64) ([]byte, error) { func (c *blobDiskCache) ReadAt(key string, offset, length int64) ([]byte, error) {
c.mu.Lock() c.mu.Lock()
c.readAtCalls++ c.readAtCalls++
e, ok := c.items[key] e, ok := c.items[key]
if !ok { if !ok {
c.mu.Unlock() c.mu.Unlock()
return nil, fmt.Errorf("key %q not in cache", key) return nil, fmt.Errorf("key %q not in cache", key)
} }
if offset+length > e.size { if offset+length > e.size {
c.mu.Unlock() c.mu.Unlock()
return nil, fmt.Errorf("read beyond blob size: offset=%d length=%d size=%d", offset, length, e.size) return nil, fmt.Errorf("read beyond blob size: offset=%d length=%d size=%d", offset, length, e.size)
} }
c.unlink(e) c.unlink(e)
c.pushFront(e) c.pushFront(e)
c.mu.Unlock() c.mu.Unlock()
@@ -255,6 +278,7 @@ func (c *blobDiskCache) ReadAt(key string, offset, length int64) ([]byte, error)
if _, err := f.ReadAt(buf, offset); err != nil { if _, err := f.ReadAt(buf, offset); err != nil {
return nil, err return nil, err
} }
return buf, nil return buf, nil
} }
@@ -262,7 +286,9 @@ func (c *blobDiskCache) ReadAt(key string, offset, length int64) ([]byte, error)
func (c *blobDiskCache) Has(key string) bool { func (c *blobDiskCache) Has(key string) bool {
c.mu.Lock() c.mu.Lock()
defer c.mu.Unlock() defer c.mu.Unlock()
_, ok := c.items[key] _, ok := c.items[key]
return ok return ok
} }
@@ -272,10 +298,12 @@ func (c *blobDiskCache) Has(key string) bool {
func (c *blobDiskCache) Delete(key string) { func (c *blobDiskCache) Delete(key string) {
c.mu.Lock() c.mu.Lock()
defer c.mu.Unlock() defer c.mu.Unlock()
e, ok := c.items[key] e, ok := c.items[key]
if !ok { if !ok {
return return
} }
c.unlink(e) c.unlink(e)
delete(c.items, key) delete(c.items, key)
c.curBytes -= e.size c.curBytes -= e.size
@@ -287,10 +315,12 @@ func (c *blobDiskCache) Delete(key string) {
func (c *blobDiskCache) Keys() []string { func (c *blobDiskCache) Keys() []string {
c.mu.Lock() c.mu.Lock()
defer c.mu.Unlock() defer c.mu.Unlock()
keys := make([]string, 0, len(c.items)) keys := make([]string, 0, len(c.items))
for k := range c.items { for k := range c.items {
keys = append(keys, k) keys = append(keys, k)
} }
return keys return keys
} }
@@ -298,6 +328,7 @@ func (c *blobDiskCache) Keys() []string {
func (c *blobDiskCache) Size() int64 { func (c *blobDiskCache) Size() int64 {
c.mu.Lock() c.mu.Lock()
defer c.mu.Unlock() defer c.mu.Unlock()
return c.curBytes return c.curBytes
} }
@@ -305,6 +336,7 @@ func (c *blobDiskCache) Size() int64 {
func (c *blobDiskCache) Len() int { func (c *blobDiskCache) Len() int {
c.mu.Lock() c.mu.Lock()
defer c.mu.Unlock() defer c.mu.Unlock()
return len(c.items) return len(c.items)
} }
@@ -312,6 +344,7 @@ func (c *blobDiskCache) Len() int {
func (c *blobDiskCache) GetCalls() int { func (c *blobDiskCache) GetCalls() int {
c.mu.Lock() c.mu.Lock()
defer c.mu.Unlock() defer c.mu.Unlock()
return c.getCalls return c.getCalls
} }
@@ -319,6 +352,7 @@ func (c *blobDiskCache) GetCalls() int {
func (c *blobDiskCache) ReadAtCalls() int { func (c *blobDiskCache) ReadAtCalls() int {
c.mu.Lock() c.mu.Lock()
defer c.mu.Unlock() defer c.mu.Unlock()
return c.readAtCalls return c.readAtCalls
} }
@@ -327,6 +361,7 @@ func (c *blobDiskCache) ReadAtCalls() int {
func (c *blobDiskCache) PeakLen() int { func (c *blobDiskCache) PeakLen() int {
c.mu.Lock() c.mu.Lock()
defer c.mu.Unlock() defer c.mu.Unlock()
return c.peakLen return c.peakLen
} }
@@ -334,9 +369,11 @@ func (c *blobDiskCache) PeakLen() int {
func (c *blobDiskCache) Close() error { func (c *blobDiskCache) Close() error {
c.mu.Lock() c.mu.Lock()
defer c.mu.Unlock() defer c.mu.Unlock()
c.items = nil c.items = nil
c.head = nil c.head = nil
c.tail = nil c.tail = nil
c.curBytes = 0 c.curBytes = 0
return os.RemoveAll(c.dir) return os.RemoveAll(c.dir)
} }

View File

@@ -23,6 +23,7 @@ func TestBlobDiskCache_BasicGetPut(t *testing.T) {
if !ok { if !ok {
t.Fatal("expected cache hit") t.Fatal("expected cache hit")
} }
if !bytes.Equal(got, data) { if !bytes.Equal(got, data) {
t.Fatalf("got %q, want %q", got, data) t.Fatalf("got %q, want %q", got, data)
} }
@@ -35,15 +36,19 @@ func TestBlobDiskCache_BasicGetPut(t *testing.T) {
func TestBlobDiskCache_EvictionUnderPressure(t *testing.T) { func TestBlobDiskCache_EvictionUnderPressure(t *testing.T) {
maxBytes := int64(1000) maxBytes := int64(1000)
cache, err := newBlobDiskCache(maxBytes) cache, err := newBlobDiskCache(maxBytes)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
defer func() { _ = cache.Close() }() defer func() { _ = cache.Close() }()
for i := 0; i < 5; i++ { for i := range 5 {
data := make([]byte, 300) data := make([]byte, 300)
if err := cache.Put(fmt.Sprintf("key%d", i), data); err != nil {
err := cache.Put(fmt.Sprintf("key%d", i), data)
if err != nil {
t.Fatal(err) t.Fatal(err)
} }
} }
@@ -55,6 +60,7 @@ func TestBlobDiskCache_EvictionUnderPressure(t *testing.T) {
if !cache.Has("key4") { if !cache.Has("key4") {
t.Fatal("expected key4 to be cached") t.Fatal("expected key4 to be cached")
} }
if cache.Has("key0") { if cache.Has("key0") {
t.Fatal("expected key0 to be evicted") t.Fatal("expected key0 to be evicted")
} }
@@ -87,6 +93,7 @@ func TestBlobDiskCache_UpdateInPlace(t *testing.T) {
if err := cache.Put("key1", []byte("v1")); err != nil { if err := cache.Put("key1", []byte("v1")); err != nil {
t.Fatal(err) t.Fatal(err)
} }
if err := cache.Put("key1", []byte("version2")); err != nil { if err := cache.Put("key1", []byte("version2")); err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -95,12 +102,15 @@ func TestBlobDiskCache_UpdateInPlace(t *testing.T) {
if !ok { if !ok {
t.Fatal("expected hit") t.Fatal("expected hit")
} }
if string(got) != "version2" { if string(got) != "version2" {
t.Fatalf("got %q, want %q", got, "version2") t.Fatalf("got %q, want %q", got, "version2")
} }
if cache.Len() != 1 { if cache.Len() != 1 {
t.Fatalf("expected 1 entry, got %d", cache.Len()) t.Fatalf("expected 1 entry, got %d", cache.Len())
} }
if cache.Size() != int64(len("version2")) { if cache.Size() != int64(len("version2")) {
t.Fatalf("expected size %d, got %d", len("version2"), cache.Size()) t.Fatalf("expected size %d, got %d", len("version2"), cache.Size())
} }
@@ -117,6 +127,7 @@ func TestBlobDiskCache_ReadAt(t *testing.T) {
if _, err := rand.Read(data); err != nil { if _, err := rand.Read(data); err != nil {
t.Fatal(err) t.Fatal(err)
} }
if err := cache.Put("blob1", data); err != nil { if err := cache.Put("blob1", data); err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -125,6 +136,7 @@ func TestBlobDiskCache_ReadAt(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
if !bytes.Equal(chunk, data[100:300]) { if !bytes.Equal(chunk, data[100:300]) {
t.Fatal("ReadAt returned wrong data") t.Fatal("ReadAt returned wrong data")
} }
@@ -149,6 +161,7 @@ func TestBlobDiskCache_Close(t *testing.T) {
if err := cache.Put("key1", []byte("data")); err != nil { if err := cache.Put("key1", []byte("data")); err != nil {
t.Fatal(err) t.Fatal(err)
} }
if err := cache.Close(); err != nil { if err := cache.Close(); err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -165,6 +178,7 @@ func TestBlobDiskCache_LRUOrder(t *testing.T) {
if err := cache.Put("a", d); err != nil { if err := cache.Put("a", d); err != nil {
t.Fatal(err) t.Fatal(err)
} }
if err := cache.Put("b", d); err != nil { if err := cache.Put("b", d); err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -180,9 +194,11 @@ func TestBlobDiskCache_LRUOrder(t *testing.T) {
if !cache.Has("a") { if !cache.Has("a") {
t.Fatal("expected 'a' to survive") t.Fatal("expected 'a' to survive")
} }
if !cache.Has("c") { if !cache.Has("c") {
t.Fatal("expected 'c' to be present") t.Fatal("expected 'c' to be present")
} }
if cache.Has("b") { if cache.Has("b") {
t.Fatal("expected 'b' to be evicted") t.Fatal("expected 'b' to be evicted")
} }

View File

@@ -1,6 +1,7 @@
package vaultik package vaultik
import ( import (
"errors"
"fmt" "fmt"
"regexp" "regexp"
"strconv" "strconv"
@@ -29,11 +30,13 @@ func formatBytes(bytes int64) string {
if bytes < unit { if bytes < unit {
return fmt.Sprintf("%d B", bytes) return fmt.Sprintf("%d B", bytes)
} }
div, exp := int64(unit), 0 div, exp := int64(unit), 0
for n := bytes / unit; n >= unit; n /= unit { for n := bytes / unit; n >= unit; n /= unit {
div *= unit div *= unit
exp++ exp++
} }
return fmt.Sprintf("%.1f %cB", float64(bytes)/float64(div), "KMGTPE"[exp]) return fmt.Sprintf("%.1f %cB", float64(bytes)/float64(div), "KMGTPE"[exp])
} }
@@ -42,11 +45,12 @@ func formatBytes(bytes int64) string {
func parseSnapshotTimestamp(snapshotID string) (time.Time, error) { func parseSnapshotTimestamp(snapshotID string) (time.Time, error) {
parts := strings.Split(snapshotID, "_") parts := strings.Split(snapshotID, "_")
if len(parts) < 2 { if len(parts) < 2 {
return time.Time{}, fmt.Errorf("invalid snapshot ID format: expected hostname_snapshotname_timestamp") return time.Time{}, errors.New("invalid snapshot ID format: expected hostname_snapshotname_timestamp")
} }
// Last part is the RFC3339 timestamp // Last part is the RFC3339 timestamp
timestampStr := parts[len(parts)-1] timestampStr := parts[len(parts)-1]
timestamp, err := time.Parse(time.RFC3339, timestampStr) timestamp, err := time.Parse(time.RFC3339, timestampStr)
if err != nil { if err != nil {
return time.Time{}, fmt.Errorf("invalid timestamp: %w", err) return time.Time{}, fmt.Errorf("invalid timestamp: %w", err)
@@ -80,17 +84,20 @@ func parseDuration(s string) (time.Duration, error) {
} }
re := regexp.MustCompile(`(\d+)\s*([a-zA-Z]+)`) re := regexp.MustCompile(`(\d+)\s*([a-zA-Z]+)`)
matches := re.FindAllStringSubmatch(s, -1) matches := re.FindAllStringSubmatch(s, -1)
if len(matches) == 0 { if len(matches) == 0 {
return 0, fmt.Errorf("invalid duration: %q", s) return 0, fmt.Errorf("invalid duration: %q", s)
} }
var total time.Duration var total time.Duration
for _, match := range matches { for _, match := range matches {
n, err := strconv.Atoi(match[1]) n, err := strconv.Atoi(match[1])
if err != nil { if err != nil {
return 0, fmt.Errorf("invalid number %q: %w", match[1], err) return 0, fmt.Errorf("invalid number %q: %w", match[1], err)
} }
unit := strings.ToLower(match[2]) unit := strings.ToLower(match[2])
switch unit { switch unit {
case "d", "day", "days": case "d", "day", "days":
@@ -105,5 +112,6 @@ func parseDuration(s string) (time.Duration, error) {
return 0, fmt.Errorf("unknown time unit %q", unit) return 0, fmt.Errorf("unknown time unit %q", unit)
} }
} }
return total, nil return total, nil
} }

View File

@@ -61,11 +61,14 @@ func TestParseDuration(t *testing.T) {
if err == nil { if err == nil {
t.Fatalf("expected error for %q, got %v", tt.input, got) t.Fatalf("expected error for %q, got %v", tt.input, got)
} }
return return
} }
if err != nil { if err != nil {
t.Fatalf("unexpected error for %q: %v", tt.input, err) t.Fatalf("unexpected error for %q: %v", tt.input, err)
} }
if got != tt.want { if got != tt.want {
t.Errorf("parseDuration(%q) = %v, want %v", tt.input, got, tt.want) t.Errorf("parseDuration(%q) = %v, want %v", tt.input, got, tt.want)
} }

View File

@@ -15,102 +15,122 @@ import (
// ShowInfo displays system and configuration information // ShowInfo displays system and configuration information
func (v *Vaultik) ShowInfo() error { func (v *Vaultik) ShowInfo() error {
// System Information // System Information
v.printfStdout("=== System Information ===\n") v.stdoutf("=== System Information ===\n")
v.printfStdout("OS/Architecture: %s/%s\n", runtime.GOOS, runtime.GOARCH) v.stdoutf("OS/Architecture: %s/%s\n", runtime.GOOS, runtime.GOARCH)
v.printfStdout("Version: %s\n", v.Globals.Version) v.stdoutf("Version: %s\n", v.Globals.Version)
v.printfStdout("Commit: %s\n", v.Globals.Commit) v.stdoutf("Commit: %s\n", v.Globals.Commit)
v.printfStdout("Go Version: %s\n", runtime.Version()) v.stdoutf("Go Version: %s\n", runtime.Version())
v.printlnStdout() v.printlnStdout()
// Storage Configuration. The backend is selected by storage_url // Storage Configuration. The backend is selected by storage_url
// (s3://, file://, rclone://); the legacy s3.* fields are only // (s3://, file://, rclone://); the legacy s3.* fields are only
// printed when they're actually populated, since the URL scheme // printed when they're actually populated, since the URL scheme
// is the primary configuration. // is the primary configuration.
v.printfStdout("=== Storage Configuration ===\n") v.stdoutf("=== Storage Configuration ===\n")
storageInfo := v.Storage.Info() storageInfo := v.Storage.Info()
v.printfStdout("Type: %s\n", storageInfo.Type) v.stdoutf("Type: %s\n", storageInfo.Type)
v.printfStdout("Location: %s\n", storageInfo.Location) v.stdoutf("Location: %s\n", storageInfo.Location)
if v.Config.StorageURL != "" { if v.Config.StorageURL != "" {
v.printfStdout("Storage URL: %s\n", v.Config.StorageURL) v.stdoutf("Storage URL: %s\n", v.Config.StorageURL)
} }
if v.Config.S3.Bucket != "" { if v.Config.S3.Bucket != "" {
v.printfStdout("S3 Bucket: %s\n", v.Config.S3.Bucket) v.stdoutf("S3 Bucket: %s\n", v.Config.S3.Bucket)
} }
if v.Config.S3.Prefix != "" { if v.Config.S3.Prefix != "" {
v.printfStdout("S3 Prefix: %s\n", v.Config.S3.Prefix) v.stdoutf("S3 Prefix: %s\n", v.Config.S3.Prefix)
} }
if v.Config.S3.Endpoint != "" { if v.Config.S3.Endpoint != "" {
v.printfStdout("S3 Endpoint: %s\n", v.Config.S3.Endpoint) v.stdoutf("S3 Endpoint: %s\n", v.Config.S3.Endpoint)
} }
if v.Config.S3.Region != "" { if v.Config.S3.Region != "" {
v.printfStdout("S3 Region: %s\n", v.Config.S3.Region) v.stdoutf("S3 Region: %s\n", v.Config.S3.Region)
} }
v.printlnStdout() v.printlnStdout()
// Backup Settings // Backup Settings
v.printfStdout("=== Backup Settings ===\n") v.stdoutf("=== Backup Settings ===\n")
// Show configured snapshots // Show configured snapshots
v.printfStdout("Snapshots:\n") v.stdoutf("Snapshots:\n")
for _, name := range v.Config.SnapshotNames() { for _, name := range v.Config.SnapshotNames() {
snap := v.Config.Snapshots[name] snap := v.Config.Snapshots[name]
v.printfStdout(" %s:\n", name) v.stdoutf(" %s:\n", name)
for _, path := range snap.Paths { for _, path := range snap.Paths {
v.printfStdout(" - %s\n", path) v.stdoutf(" - %s\n", path)
} }
if len(snap.Exclude) > 0 { if len(snap.Exclude) > 0 {
v.printfStdout(" exclude: %s\n", strings.Join(snap.Exclude, ", ")) v.stdoutf(" exclude: %s\n", strings.Join(snap.Exclude, ", "))
} }
} }
// Global exclude patterns // Global exclude patterns
if len(v.Config.Exclude) > 0 { if len(v.Config.Exclude) > 0 {
v.printfStdout("Global Exclude: %s\n", strings.Join(v.Config.Exclude, ", ")) v.stdoutf("Global Exclude: %s\n", strings.Join(v.Config.Exclude, ", "))
} }
v.printfStdout("Compression: zstd level %d\n", v.Config.CompressionLevel) v.stdoutf("Compression: zstd level %d\n", v.Config.CompressionLevel)
v.printfStdout("Chunk Size: %s\n", humanize.Bytes(uint64(v.Config.ChunkSize))) v.stdoutf("Chunk Size: %s\n", humanize.Bytes(uint64(v.Config.ChunkSize)))
v.printfStdout("Blob Size Limit: %s\n", humanize.Bytes(uint64(v.Config.BlobSizeLimit))) v.stdoutf("Blob Size Limit: %s\n", humanize.Bytes(uint64(v.Config.BlobSizeLimit)))
v.printlnStdout() v.printlnStdout()
// Encryption Configuration // Encryption Configuration
v.printfStdout("=== Encryption Configuration ===\n") v.stdoutf("=== Encryption Configuration ===\n")
v.printfStdout("Recipients:\n") v.stdoutf("Recipients:\n")
for _, recipient := range v.Config.AgeRecipients { for _, recipient := range v.Config.AgeRecipients {
v.printfStdout(" - %s\n", recipient) v.stdoutf(" - %s\n", recipient)
} }
v.printlnStdout() v.printlnStdout()
// Local Database // Local Database
v.printfStdout("=== Local Database ===\n") v.stdoutf("=== Local Database ===\n")
v.printfStdout("Index Path: %s\n", v.Config.IndexPath) v.stdoutf("Index Path: %s\n", v.Config.IndexPath)
// Check if index file exists and get its size // Check if index file exists and get its size
if info, err := v.Fs.Stat(v.Config.IndexPath); err == nil { if info, err := v.Fs.Stat(v.Config.IndexPath); err == nil {
v.printfStdout("Index Size: %s\n", humanize.Bytes(uint64(info.Size()))) v.stdoutf("Index Size: %s\n", humanize.Bytes(uint64(info.Size())))
// Get snapshot count from database // Get snapshot count from database
query := `SELECT COUNT(*) FROM snapshots WHERE completed_at IS NOT NULL` query := `SELECT COUNT(*) FROM snapshots WHERE completed_at IS NOT NULL`
var snapshotCount int var snapshotCount int
if err := v.DB.Conn().QueryRowContext(v.ctx, query).Scan(&snapshotCount); err == nil {
v.printfStdout("Snapshots: %d\n", snapshotCount) err := v.DB.Conn().QueryRowContext(v.ctx, query).Scan(&snapshotCount)
if err == nil {
v.stdoutf("Snapshots: %d\n", snapshotCount)
} }
// Get blob count from database // Get blob count from database
query = `SELECT COUNT(*) FROM blobs` query = `SELECT COUNT(*) FROM blobs`
var blobCount int var blobCount int
if err := v.DB.Conn().QueryRowContext(v.ctx, query).Scan(&blobCount); err == nil {
v.printfStdout("Blobs: %d\n", blobCount) err = v.DB.Conn().QueryRowContext(v.ctx, query).Scan(&blobCount)
if err == nil {
v.stdoutf("Blobs: %d\n", blobCount)
} }
// Get file count from database // Get file count from database
query = `SELECT COUNT(*) FROM files` query = `SELECT COUNT(*) FROM files`
var fileCount int var fileCount int
if err := v.DB.Conn().QueryRowContext(v.ctx, query).Scan(&fileCount); err == nil {
v.printfStdout("Files: %d\n", fileCount) err = v.DB.Conn().QueryRowContext(v.ctx, query).Scan(&fileCount)
if err == nil {
v.stdoutf("Files: %d\n", fileCount)
} }
} else { } else {
v.printfStdout("Index Size: (not created)\n") v.stdoutf("Index Size: (not created)\n")
} }
return nil return nil
@@ -153,6 +173,7 @@ type RemoteInfoResult struct {
// RemoteInfo displays information about remote storage // RemoteInfo displays information about remote storage
func (v *Vaultik) RemoteInfo(jsonOutput bool) error { func (v *Vaultik) RemoteInfo(jsonOutput bool) error {
log.Info("Starting remote storage info gathering") log.Info("Starting remote storage info gathering")
result := &RemoteInfoResult{} result := &RemoteInfoResult{}
storageInfo := v.Storage.Info() storageInfo := v.Storage.Info()
@@ -160,11 +181,11 @@ func (v *Vaultik) RemoteInfo(jsonOutput bool) error {
result.StorageLocation = storageInfo.Location result.StorageLocation = storageInfo.Location
if !jsonOutput { if !jsonOutput {
v.printfStdout("=== Remote Storage ===\n") v.stdoutf("=== Remote Storage ===\n")
v.printfStdout("Type: %s\n", storageInfo.Type) v.stdoutf("Type: %s\n", storageInfo.Type)
v.printfStdout("Location: %s\n", storageInfo.Location) v.stdoutf("Location: %s\n", storageInfo.Location)
v.printlnStdout() v.printlnStdout()
v.printfStdout("Scanning snapshot metadata...\n") v.stdoutf("Scanning snapshot metadata...\n")
} }
snapshotMetadata, snapshotIDs, err := v.collectSnapshotMetadata() snapshotMetadata, snapshotIDs, err := v.collectSnapshotMetadata()
@@ -173,7 +194,7 @@ func (v *Vaultik) RemoteInfo(jsonOutput bool) error {
} }
if !jsonOutput { if !jsonOutput {
v.printfStdout("Downloading %d manifest(s)...\n", len(snapshotIDs)) v.stdoutf("Downloading %d manifest(s)...\n", len(snapshotIDs))
} }
referencedBlobs := v.collectReferencedBlobsFromManifests(snapshotIDs, snapshotMetadata) referencedBlobs := v.collectReferencedBlobsFromManifests(snapshotIDs, snapshotMetadata)
@@ -193,10 +214,12 @@ func (v *Vaultik) RemoteInfo(jsonOutput bool) error {
if jsonOutput { if jsonOutput {
enc := json.NewEncoder(v.Stdout) enc := json.NewEncoder(v.Stdout)
enc.SetIndent("", " ") enc.SetIndent("", " ")
return enc.Encode(result) return enc.Encode(result)
} }
v.printRemoteInfoTable(result) v.printRemoteInfoTable(result)
return nil return nil
} }
@@ -214,6 +237,7 @@ func (v *Vaultik) collectSnapshotMetadata() (map[string]*SnapshotMetadataInfo, [
if len(parts) < 3 { if len(parts) < 3 {
continue continue
} }
snapshotID := parts[1] snapshotID := parts[1]
if _, exists := snapshotMetadata[snapshotID]; !exists { if _, exists := snapshotMetadata[snapshotID]; !exists {
@@ -221,12 +245,14 @@ func (v *Vaultik) collectSnapshotMetadata() (map[string]*SnapshotMetadataInfo, [
} }
info := snapshotMetadata[snapshotID] info := snapshotMetadata[snapshotID]
filename := parts[2] filename := parts[2]
if strings.HasPrefix(filename, "manifest") { if strings.HasPrefix(filename, "manifest") {
info.ManifestSize = obj.Size info.ManifestSize = obj.Size
} else if strings.HasPrefix(filename, "db") { } else if strings.HasPrefix(filename, "db") {
info.DatabaseSize = obj.Size info.DatabaseSize = obj.Size
} }
info.TotalSize = info.ManifestSize + info.DatabaseSize info.TotalSize = info.ManifestSize + info.DatabaseSize
} }
@@ -234,6 +260,7 @@ func (v *Vaultik) collectSnapshotMetadata() (map[string]*SnapshotMetadataInfo, [
for id := range snapshotMetadata { for id := range snapshotMetadata {
snapshotIDs = append(snapshotIDs, id) snapshotIDs = append(snapshotIDs, id)
} }
sort.Strings(snapshotIDs) sort.Strings(snapshotIDs)
return snapshotMetadata, snapshotIDs, nil return snapshotMetadata, snapshotIDs, nil
@@ -245,26 +272,33 @@ func (v *Vaultik) collectReferencedBlobsFromManifests(snapshotIDs []string, snap
for _, snapshotID := range snapshotIDs { for _, snapshotID := range snapshotIDs {
manifestKey := fmt.Sprintf("metadata/%s/manifest.json.zst", snapshotID) manifestKey := fmt.Sprintf("metadata/%s/manifest.json.zst", snapshotID)
reader, err := v.Storage.Get(v.ctx, manifestKey) reader, err := v.Storage.Get(v.ctx, manifestKey)
if err != nil { if err != nil {
log.Warn("Failed to get manifest", "snapshot", snapshotID, "error", err) log.Warn("Failed to get manifest", "snapshot", snapshotID, "error", err)
continue continue
} }
manifest, err := snapshot.DecodeManifest(reader) manifest, err := snapshot.DecodeManifest(reader)
_ = reader.Close() _ = reader.Close()
if err != nil { if err != nil {
log.Warn("Failed to decode manifest", "snapshot", snapshotID, "error", err) log.Warn("Failed to decode manifest", "snapshot", snapshotID, "error", err)
continue continue
} }
info := snapshotMetadata[snapshotID] info := snapshotMetadata[snapshotID]
info.BlobCount = manifest.BlobCount info.BlobCount = manifest.BlobCount
var blobsSize int64 var blobsSize int64
for _, blob := range manifest.Blobs { for _, blob := range manifest.Blobs {
referencedBlobs[blob.Hash] = blob.CompressedSize referencedBlobs[blob.Hash] = blob.CompressedSize
blobsSize += blob.CompressedSize blobsSize += blob.CompressedSize
} }
info.BlobsSize = blobsSize info.BlobsSize = blobsSize
} }
@@ -274,11 +308,13 @@ func (v *Vaultik) collectReferencedBlobsFromManifests(snapshotIDs []string, snap
// populateRemoteInfoResult fills in the result's snapshot and referenced blob stats // populateRemoteInfoResult fills in the result's snapshot and referenced blob stats
func (v *Vaultik) populateRemoteInfoResult(result *RemoteInfoResult, snapshotMetadata map[string]*SnapshotMetadataInfo, snapshotIDs []string, referencedBlobs map[string]int64) { func (v *Vaultik) populateRemoteInfoResult(result *RemoteInfoResult, snapshotMetadata map[string]*SnapshotMetadataInfo, snapshotIDs []string, referencedBlobs map[string]int64) {
var totalMetadataSize int64 var totalMetadataSize int64
for _, id := range snapshotIDs { for _, id := range snapshotIDs {
info := snapshotMetadata[id] info := snapshotMetadata[id]
result.Snapshots = append(result.Snapshots, *info) result.Snapshots = append(result.Snapshots, *info)
totalMetadataSize += info.TotalSize totalMetadataSize += info.TotalSize
} }
result.TotalMetadataSize = totalMetadataSize result.TotalMetadataSize = totalMetadataSize
result.TotalMetadataCount = len(snapshotIDs) result.TotalMetadataCount = len(snapshotIDs)
@@ -291,7 +327,7 @@ func (v *Vaultik) populateRemoteInfoResult(result *RemoteInfoResult, snapshotMet
// scanRemoteBlobStorage lists all blobs on remote and computes orphan stats // scanRemoteBlobStorage lists all blobs on remote and computes orphan stats
func (v *Vaultik) scanRemoteBlobStorage(result *RemoteInfoResult, referencedBlobs map[string]int64, jsonOutput bool) error { func (v *Vaultik) scanRemoteBlobStorage(result *RemoteInfoResult, referencedBlobs map[string]int64, jsonOutput bool) error {
if !jsonOutput { if !jsonOutput {
v.printfStdout("Scanning blobs...\n") v.stdoutf("Scanning blobs...\n")
} }
blobCh := v.Storage.ListStream(v.ctx, "blobs/") blobCh := v.Storage.ListStream(v.ctx, "blobs/")
@@ -301,10 +337,12 @@ func (v *Vaultik) scanRemoteBlobStorage(result *RemoteInfoResult, referencedBlob
if obj.Err != nil { if obj.Err != nil {
return fmt.Errorf("listing blobs: %w", obj.Err) return fmt.Errorf("listing blobs: %w", obj.Err)
} }
parts := strings.Split(obj.Key, "/") parts := strings.Split(obj.Key, "/")
if len(parts) < 4 { if len(parts) < 4 {
continue continue
} }
hash := parts[3] hash := parts[3]
allBlobs[hash] = obj.Size allBlobs[hash] = obj.Size
result.TotalBlobCount++ result.TotalBlobCount++
@@ -323,14 +361,16 @@ func (v *Vaultik) scanRemoteBlobStorage(result *RemoteInfoResult, referencedBlob
// printRemoteInfoTable renders the human-readable remote info output // printRemoteInfoTable renders the human-readable remote info output
func (v *Vaultik) printRemoteInfoTable(result *RemoteInfoResult) { func (v *Vaultik) printRemoteInfoTable(result *RemoteInfoResult) {
v.printfStdout("\n=== Snapshot Metadata ===\n") v.stdoutf("\n=== Snapshot Metadata ===\n")
if len(result.Snapshots) == 0 { if len(result.Snapshots) == 0 {
v.printfStdout("No snapshots found\n") v.stdoutf("No snapshots found\n")
} else { } else {
v.printfStdout("%-45s %12s %12s %12s %10s %12s\n", "SNAPSHOT", "MANIFEST", "DATABASE", "TOTAL", "BLOBS", "BLOB SIZE") v.stdoutf("%-45s %12s %12s %12s %10s %12s\n", "SNAPSHOT", "MANIFEST", "DATABASE", "TOTAL", "BLOBS", "BLOB SIZE")
v.printfStdout("%-45s %12s %12s %12s %10s %12s\n", strings.Repeat("-", 45), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 10), strings.Repeat("-", 12)) v.stdoutf("%-45s %12s %12s %12s %10s %12s\n", strings.Repeat("-", 45), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 10), strings.Repeat("-", 12))
for _, info := range result.Snapshots { for _, info := range result.Snapshots {
v.printfStdout("%-45s %12s %12s %12s %10s %12s\n", v.stdoutf("%-45s %12s %12s %12s %10s %12s\n",
truncateString(info.SnapshotID, 45), truncateString(info.SnapshotID, 45),
humanize.Bytes(uint64(info.ManifestSize)), humanize.Bytes(uint64(info.ManifestSize)),
humanize.Bytes(uint64(info.DatabaseSize)), humanize.Bytes(uint64(info.DatabaseSize)),
@@ -339,20 +379,21 @@ func (v *Vaultik) printRemoteInfoTable(result *RemoteInfoResult) {
humanize.Bytes(uint64(info.BlobsSize)), humanize.Bytes(uint64(info.BlobsSize)),
) )
} }
v.printfStdout("%-45s %12s %12s %12s %10s %12s\n", strings.Repeat("-", 45), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 10), strings.Repeat("-", 12))
v.printfStdout("%-45s %12s %12s %12s\n", fmt.Sprintf("Total (%d snapshots)", result.TotalMetadataCount), "", "", humanize.Bytes(uint64(result.TotalMetadataSize))) v.stdoutf("%-45s %12s %12s %12s %10s %12s\n", strings.Repeat("-", 45), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 10), strings.Repeat("-", 12))
v.stdoutf("%-45s %12s %12s %12s\n", fmt.Sprintf("Total (%d snapshots)", result.TotalMetadataCount), "", "", humanize.Bytes(uint64(result.TotalMetadataSize)))
} }
v.printfStdout("\n=== Blob Storage ===\n") v.stdoutf("\n=== Blob Storage ===\n")
v.printfStdout("Total blobs on remote: %s (%s)\n", v.stdoutf("Total blobs on remote: %s (%s)\n",
humanize.Comma(int64(result.TotalBlobCount)), humanize.Bytes(uint64(result.TotalBlobSize))) humanize.Comma(int64(result.TotalBlobCount)), humanize.Bytes(uint64(result.TotalBlobSize)))
v.printfStdout("Referenced by snapshots: %s (%s)\n", v.stdoutf("Referenced by snapshots: %s (%s)\n",
humanize.Comma(int64(result.ReferencedBlobCount)), humanize.Bytes(uint64(result.ReferencedBlobSize))) humanize.Comma(int64(result.ReferencedBlobCount)), humanize.Bytes(uint64(result.ReferencedBlobSize)))
v.printfStdout("Orphaned (unreferenced): %s (%s)\n", v.stdoutf("Orphaned (unreferenced): %s (%s)\n",
humanize.Comma(int64(result.OrphanedBlobCount)), humanize.Bytes(uint64(result.OrphanedBlobSize))) humanize.Comma(int64(result.OrphanedBlobCount)), humanize.Bytes(uint64(result.OrphanedBlobSize)))
if result.OrphanedBlobCount > 0 { if result.OrphanedBlobCount > 0 {
v.printfStdout("\nRun 'vaultik prune' to remove orphaned blobs.\n") v.stdoutf("\nRun 'vaultik prune' to remove orphaned blobs.\n")
} }
} }
@@ -361,8 +402,10 @@ func truncateString(s string, maxLen int) string {
if len(s) <= maxLen { if len(s) <= maxLen {
return s return s
} }
if maxLen <= 3 { if maxLen <= 3 {
return s[:maxLen] return s[:maxLen]
} }
return s[:maxLen-3] + "..." return s[:maxLen-3] + "..."
} }

View File

@@ -43,11 +43,14 @@ func (m *MockStorer) Put(ctx context.Context, key string, reader io.Reader) erro
defer m.mu.Unlock() defer m.mu.Unlock()
m.calls = append(m.calls, "Put:"+key) m.calls = append(m.calls, "Put:"+key)
data, err := io.ReadAll(reader) data, err := io.ReadAll(reader)
if err != nil { if err != nil {
return err return err
} }
m.data[key] = data m.data[key] = data
return nil return nil
} }
@@ -60,10 +63,12 @@ func (m *MockStorer) Get(ctx context.Context, key string) (io.ReadCloser, error)
defer m.mu.Unlock() defer m.mu.Unlock()
m.calls = append(m.calls, "Get:"+key) m.calls = append(m.calls, "Get:"+key)
data, exists := m.data[key] data, exists := m.data[key]
if !exists { if !exists {
return nil, storage.ErrNotFound return nil, storage.ErrNotFound
} }
return io.NopCloser(bytes.NewReader(data)), nil return io.NopCloser(bytes.NewReader(data)), nil
} }
@@ -72,10 +77,12 @@ func (m *MockStorer) Stat(ctx context.Context, key string) (*storage.ObjectInfo,
defer m.mu.Unlock() defer m.mu.Unlock()
m.calls = append(m.calls, "Stat:"+key) m.calls = append(m.calls, "Stat:"+key)
data, exists := m.data[key] data, exists := m.data[key]
if !exists { if !exists {
return nil, storage.ErrNotFound return nil, storage.ErrNotFound
} }
return &storage.ObjectInfo{ return &storage.ObjectInfo{
Key: key, Key: key,
Size: int64(len(data)), Size: int64(len(data)),
@@ -88,6 +95,7 @@ func (m *MockStorer) Delete(ctx context.Context, key string) error {
m.calls = append(m.calls, "Delete:"+key) m.calls = append(m.calls, "Delete:"+key)
delete(m.data, key) delete(m.data, key)
return nil return nil
} }
@@ -96,12 +104,15 @@ func (m *MockStorer) List(ctx context.Context, prefix string) ([]string, error)
defer m.mu.Unlock() defer m.mu.Unlock()
m.calls = append(m.calls, "List:"+prefix) m.calls = append(m.calls, "List:"+prefix)
var keys []string var keys []string
for key := range m.data { for key := range m.data {
if len(prefix) == 0 || (len(key) >= len(prefix) && key[:len(prefix)] == prefix) { if len(prefix) == 0 || (len(key) >= len(prefix) && key[:len(prefix)] == prefix) {
keys = append(keys, key) keys = append(keys, key)
} }
} }
return keys, nil return keys, nil
} }
@@ -109,6 +120,7 @@ func (m *MockStorer) ListStream(ctx context.Context, prefix string) <-chan stora
ch := make(chan storage.ObjectInfo) ch := make(chan storage.ObjectInfo)
go func() { go func() {
defer close(ch) defer close(ch)
m.mu.Lock() m.mu.Lock()
defer m.mu.Unlock() defer m.mu.Unlock()
@@ -121,6 +133,7 @@ func (m *MockStorer) ListStream(ctx context.Context, prefix string) <-chan stora
} }
} }
}() }()
return ch return ch
} }
@@ -138,6 +151,7 @@ func (m *MockStorer) GetCalls() []string {
calls := make([]string, len(m.calls)) calls := make([]string, len(m.calls))
copy(calls, m.calls) copy(calls, m.calls)
return calls return calls
} }
@@ -172,14 +186,16 @@ func TestEndToEndBackup(t *testing.T) {
"/home/user/code", "/home/user/code",
} }
for _, dir := range dirs { for _, dir := range dirs {
if err := fs.MkdirAll(dir, 0755); err != nil { err := fs.MkdirAll(dir, 0755)
if err != nil {
t.Fatalf("failed to create directory %s: %v", dir, err) t.Fatalf("failed to create directory %s: %v", dir, err)
} }
} }
// Create test files // Create test files
for path, content := range testFiles { for path, content := range testFiles {
if err := afero.WriteFile(fs, path, []byte(content), 0644); err != nil { err := afero.WriteFile(fs, path, []byte(content), 0644)
if err != nil {
t.Fatalf("failed to create test file %s: %v", path, err) t.Fatalf("failed to create test file %s: %v", path, err)
} }
} }
@@ -216,9 +232,11 @@ func TestEndToEndBackup(t *testing.T) {
// Create in-memory database // Create in-memory database
db, err := database.New(ctx, ":memory:") db, err := database.New(ctx, ":memory:")
require.NoError(t, err) require.NoError(t, err)
defer func() { defer func() {
if err := db.Close(); err != nil { err := db.Close()
if err != nil {
t.Errorf("failed to close database: %v", err) t.Errorf("failed to close database: %v", err)
} }
}() }()
@@ -246,6 +264,7 @@ func TestEndToEndBackup(t *testing.T) {
VaultikVersion: "test-version", VaultikVersion: "test-version",
StartedAt: time.Now(), StartedAt: time.Now(),
} }
return repos.Snapshots.Create(ctx, tx, snapshot) return repos.Snapshots.Create(ctx, tx, snapshot)
}) })
require.NoError(t, err) require.NoError(t, err)
@@ -258,9 +277,9 @@ func TestEndToEndBackup(t *testing.T) {
// The scanner counts both files and directories, so we have: // The scanner counts both files and directories, so we have:
// 4 files + 4 directories (/home, /home/user, /home/user/documents, /home/user/pictures, /home/user/code) // 4 files + 4 directories (/home, /home/user, /home/user/documents, /home/user/pictures, /home/user/code)
assert.GreaterOrEqual(t, result.FilesScanned, 4, "Should scan at least 4 files") assert.GreaterOrEqual(t, result.FilesScanned, 4, "Should scan at least 4 files")
assert.Greater(t, result.BytesScanned, int64(0), "Should scan some bytes") assert.Positive(t, result.BytesScanned, "Should scan some bytes")
assert.Greater(t, result.ChunksCreated, 0, "Should create chunks") assert.Positive(t, result.ChunksCreated, "Should create chunks")
assert.Greater(t, result.BlobsCreated, 0, "Should create blobs") assert.Positive(t, result.BlobsCreated, "Should create blobs")
// Verify storage operations // Verify storage operations
calls := mockStorage.GetCalls() calls := mockStorage.GetCalls()
@@ -268,6 +287,7 @@ func TestEndToEndBackup(t *testing.T) {
// Should have uploaded at least one blob // Should have uploaded at least one blob
blobUploads := 0 blobUploads := 0
for _, call := range calls { for _, call := range calls {
if len(call) > 4 && call[:4] == "Put:" { if len(call) > 4 && call[:4] == "Put:" {
if len(call) > 10 && call[4:10] == "blobs/" { if len(call) > 10 && call[4:10] == "blobs/" {
@@ -275,27 +295,30 @@ func TestEndToEndBackup(t *testing.T) {
} }
} }
} }
assert.Greater(t, blobUploads, 0, "Should upload at least one blob")
assert.Positive(t, blobUploads, "Should upload at least one blob")
// Verify files in database // Verify files in database
files, err := repos.Files.ListByPrefix(ctx, "/home/user") files, err := repos.Files.ListByPrefix(ctx, "/home/user")
require.NoError(t, err) require.NoError(t, err)
// Count only regular files (not directories) // Count only regular files (not directories)
regularFiles := 0 regularFiles := 0
for _, f := range files { for _, f := range files {
if f.Mode&0x80000000 == 0 { // Check if regular file (not directory) if f.Mode&0x80000000 == 0 { // Check if regular file (not directory)
regularFiles++ regularFiles++
} }
} }
assert.Equal(t, 4, regularFiles, "Should have 4 regular files in database") assert.Equal(t, 4, regularFiles, "Should have 4 regular files in database")
// Verify chunks were created by checking a specific file // Verify chunks were created by checking a specific file
fileChunks, err := repos.FileChunks.GetByPath(ctx, "/home/user/documents/file1.txt") fileChunks, err := repos.FileChunks.GetByPath(ctx, "/home/user/documents/file1.txt")
require.NoError(t, err) require.NoError(t, err)
assert.Greater(t, len(fileChunks), 0, "Should have chunks for file1.txt") assert.NotEmpty(t, fileChunks, "Should have chunks for file1.txt")
// Verify blobs were uploaded to storage // Verify blobs were uploaded to storage
assert.Greater(t, mockStorage.GetStorageSize(), 0, "Should have blobs in storage") assert.Positive(t, mockStorage.GetStorageSize(), "Should have blobs in storage")
// Complete the snapshot - just verify we got results // Complete the snapshot - just verify we got results
// In a real integration test, we'd update the snapshot record // In a real integration test, we'd update the snapshot record
@@ -337,9 +360,11 @@ func TestBackupAndVerify(t *testing.T) {
// Create test database // Create test database
ctx := context.Background() ctx := context.Background()
db, err := database.New(ctx, ":memory:") db, err := database.New(ctx, ":memory:")
require.NoError(t, err) require.NoError(t, err)
defer func() { defer func() {
if err := db.Close(); err != nil { err := db.Close()
if err != nil {
t.Errorf("failed to close database: %v", err) t.Errorf("failed to close database: %v", err)
} }
}() }()
@@ -366,6 +391,7 @@ func TestBackupAndVerify(t *testing.T) {
VaultikVersion: "test-version", VaultikVersion: "test-version",
StartedAt: time.Now(), StartedAt: time.Now(),
} }
return repos.Snapshots.Create(ctx, tx, snapshot) return repos.Snapshots.Create(ctx, tx, snapshot)
}) })
require.NoError(t, err) require.NoError(t, err)
@@ -375,7 +401,7 @@ func TestBackupAndVerify(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
// Verify backup created blobs // Verify backup created blobs
assert.Greater(t, result.BlobsCreated, 0, "Should create at least one blob") assert.Positive(t, result.BlobsCreated, "Should create at least one blob")
assert.Equal(t, mockStorage.GetStorageSize(), result.BlobsCreated, "Storage should have the blobs") assert.Equal(t, mockStorage.GetStorageSize(), result.BlobsCreated, "Storage should have the blobs")
// Verify we can retrieve the blob from storage // Verify we can retrieve the blob from storage
@@ -391,18 +417,19 @@ func TestBackupAndVerify(t *testing.T) {
// Get blob info // Get blob info
blobInfo, err := mockStorage.Stat(ctx, blobKey) blobInfo, err := mockStorage.Stat(ctx, blobKey)
require.NoError(t, err) require.NoError(t, err)
assert.Greater(t, blobInfo.Size, int64(0), "Blob should have content") assert.Positive(t, blobInfo.Size, "Blob should have content")
// Get blob content // Get blob content
reader, err := mockStorage.Get(ctx, blobKey) reader, err := mockStorage.Get(ctx, blobKey)
require.NoError(t, err) require.NoError(t, err)
defer func() { _ = reader.Close() }() defer func() { _ = reader.Close() }()
// Verify blob data is encrypted (should not contain plaintext) // Verify blob data is encrypted (should not contain plaintext)
blobData, err := io.ReadAll(reader) blobData, err := io.ReadAll(reader)
require.NoError(t, err) require.NoError(t, err)
assert.NotContains(t, string(blobData), testContent, "Blob should be encrypted") assert.NotContains(t, string(blobData), testContent, "Blob should be encrypted")
assert.Greater(t, len(blobData), 0, "Blob should have data") assert.NotEmpty(t, blobData, "Blob should have data")
} }
t.Logf("Backup and verify test completed successfully") t.Logf("Backup and verify test completed successfully")
@@ -418,6 +445,7 @@ func TestBackupAndRestore(t *testing.T) {
// Create real temp directory for the database (SQLite needs real filesystem) // Create real temp directory for the database (SQLite needs real filesystem)
realTempDir, err := os.MkdirTemp("", "vaultik-test-") realTempDir, err := os.MkdirTemp("", "vaultik-test-")
require.NoError(t, err) require.NoError(t, err)
defer func() { _ = os.RemoveAll(realTempDir) }() defer func() { _ = os.RemoveAll(realTempDir) }()
// Use real OS filesystem for this test // Use real OS filesystem for this test
@@ -434,10 +462,14 @@ func TestBackupAndRestore(t *testing.T) {
// Create directories and files // Create directories and files
for path, content := range testFiles { for path, content := range testFiles {
dir := filepath.Dir(path) dir := filepath.Dir(path)
if err := fs.MkdirAll(dir, 0755); err != nil {
err := fs.MkdirAll(dir, 0755)
if err != nil {
t.Fatalf("failed to create directory %s: %v", dir, err) t.Fatalf("failed to create directory %s: %v", dir, err)
} }
if err := afero.WriteFile(fs, path, []byte(content), 0644); err != nil {
err = afero.WriteFile(fs, path, []byte(content), 0644)
if err != nil {
t.Fatalf("failed to create test file %s: %v", path, err) t.Fatalf("failed to create test file %s: %v", path, err)
} }
} }
@@ -455,6 +487,7 @@ func TestBackupAndRestore(t *testing.T) {
dbPath := filepath.Join(realTempDir, "test.db") dbPath := filepath.Join(realTempDir, "test.db")
db, err := database.New(ctx, dbPath) db, err := database.New(ctx, dbPath)
require.NoError(t, err) require.NoError(t, err)
defer func() { _ = db.Close() }() defer func() { _ = db.Close() }()
repos := database.NewRepositories(db) repos := database.NewRepositories(db)
@@ -558,6 +591,7 @@ func TestEndToEndFileStorage(t *testing.T) {
fs := afero.NewOsFs() fs := afero.NewOsFs()
tempDir, err := os.MkdirTemp("", "vaultik-e2e-") tempDir, err := os.MkdirTemp("", "vaultik-e2e-")
require.NoError(t, err) require.NoError(t, err)
defer func() { _ = os.RemoveAll(tempDir) }() defer func() { _ = os.RemoveAll(tempDir) }()
dataDir := filepath.Join(tempDir, "source") dataDir := filepath.Join(tempDir, "source")
@@ -618,6 +652,7 @@ func TestEndToEndFileStorage(t *testing.T) {
db, err := database.New(ctx, dbPath) db, err := database.New(ctx, dbPath)
require.NoError(t, err) require.NoError(t, err)
defer func() { _ = db.Close() }() defer func() { _ = db.Close() }()
repos := database.NewRepositories(db) repos := database.NewRepositories(db)
@@ -644,8 +679,8 @@ func TestEndToEndFileStorage(t *testing.T) {
scanResult, err := scanner.Scan(ctx, dataDir, snapshotID) scanResult, err := scanner.Scan(ctx, dataDir, snapshotID)
require.NoError(t, err) require.NoError(t, err)
require.Greater(t, scanResult.FilesScanned, 0) require.Positive(t, scanResult.FilesScanned)
require.Greater(t, scanResult.BlobsCreated, 0) require.Positive(t, scanResult.BlobsCreated)
require.NoError(t, sm.CompleteSnapshot(ctx, snapshotID)) require.NoError(t, sm.CompleteSnapshot(ctx, snapshotID))
require.NoError(t, sm.ExportSnapshotMetadata(ctx, dbPath, snapshotID)) require.NoError(t, sm.ExportSnapshotMetadata(ctx, dbPath, snapshotID))
@@ -656,6 +691,7 @@ func TestEndToEndFileStorage(t *testing.T) {
blobInfo, err := os.Stat(filepath.Join(storeDir, "blobs")) blobInfo, err := os.Stat(filepath.Join(storeDir, "blobs"))
require.NoError(t, err) require.NoError(t, err)
require.True(t, blobInfo.IsDir()) require.True(t, blobInfo.IsDir())
metaInfo, err := os.Stat(filepath.Join(storeDir, "metadata", snapshot.RemoteSnapshotKey(snapshotID))) metaInfo, err := os.Stat(filepath.Join(storeDir, "metadata", snapshot.RemoteSnapshotKey(snapshotID)))
require.NoError(t, err) require.NoError(t, err)
require.True(t, metaInfo.IsDir()) require.True(t, metaInfo.IsDir())
@@ -721,6 +757,7 @@ func TestDedupOnlySnapshotRestores(t *testing.T) {
fs := afero.NewOsFs() fs := afero.NewOsFs()
tempDir, err := os.MkdirTemp("", "vaultik-dedup-") tempDir, err := os.MkdirTemp("", "vaultik-dedup-")
require.NoError(t, err) require.NoError(t, err)
defer func() { _ = os.RemoveAll(tempDir) }() defer func() { _ = os.RemoveAll(tempDir) }()
dataDir := filepath.Join(tempDir, "source") dataDir := filepath.Join(tempDir, "source")
@@ -756,7 +793,9 @@ func TestDedupOnlySnapshotRestores(t *testing.T) {
ctx := context.Background() ctx := context.Background()
db, err := database.New(ctx, dbPath) db, err := database.New(ctx, dbPath)
require.NoError(t, err) require.NoError(t, err)
defer func() { _ = db.Close() }() defer func() { _ = db.Close() }()
repos := database.NewRepositories(db) repos := database.NewRepositories(db)
makeScanner := func() *snapshot.Scanner { makeScanner := func() *snapshot.Scanner {
@@ -780,13 +819,14 @@ func TestDedupOnlySnapshotRestores(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
r1, err := makeScanner().Scan(ctx, dataDir, id1) r1, err := makeScanner().Scan(ctx, dataDir, id1)
require.NoError(t, err) require.NoError(t, err)
require.Greater(t, r1.BlobsCreated, 0, "first snapshot should upload at least one blob") require.Positive(t, r1.BlobsCreated, "first snapshot should upload at least one blob")
require.NoError(t, sm.CompleteSnapshot(ctx, id1)) require.NoError(t, sm.CompleteSnapshot(ctx, id1))
require.NoError(t, sm.ExportSnapshotMetadata(ctx, dbPath, id1)) require.NoError(t, sm.ExportSnapshotMetadata(ctx, dbPath, id1))
// Second snapshot — same data, every chunk dedups. Sleep past the // Second snapshot — same data, every chunk dedups. Sleep past the
// second-precision timestamp so the snapshot IDs differ. // second-precision timestamp so the snapshot IDs differ.
time.Sleep(1100 * time.Millisecond) time.Sleep(1100 * time.Millisecond)
id2, err := sm.CreateSnapshotWithName(ctx, cfg.Hostname, "dedup", "v", "g") id2, err := sm.CreateSnapshotWithName(ctx, cfg.Hostname, "dedup", "v", "g")
require.NoError(t, err) require.NoError(t, err)
r2, err := makeScanner().Scan(ctx, dataDir, id2) r2, err := makeScanner().Scan(ctx, dataDir, id2)
@@ -833,5 +873,6 @@ func bytesPattern(tag string, n int) []byte {
for i := range out { for i := range out {
out[i] = byte(tag[i%len(tag)] ^ byte(i&0xff)) out[i] = byte(tag[i%len(tag)] ^ byte(i&0xff))
} }
return out return out
} }

View File

@@ -2,6 +2,7 @@ package vaultik
import ( import (
"encoding/json" "encoding/json"
"errors"
"fmt" "fmt"
"strings" "strings"
@@ -23,20 +24,24 @@ type PruneOptions struct {
// confirming with the user. // confirming with the user.
func (v *Vaultik) NukeRemote(force bool) error { func (v *Vaultik) NukeRemote(force bool) error {
if !force { if !force {
return fmt.Errorf("nuke requires --force (this deletes ALL remote snapshots and blobs)") return errors.New("nuke requires --force (this deletes ALL remote snapshots and blobs)")
} }
v.UI.Begin("Removing all snapshot metadata from backup destination store.") v.UI.Beginf("Removing all snapshot metadata from backup destination store.")
if _, err := v.RemoveAllSnapshots(&RemoveOptions{Force: true}); err != nil { if _, err := v.RemoveAllSnapshots(&RemoveOptions{Force: true}); err != nil {
return fmt.Errorf("removing all snapshots: %w", err) return fmt.Errorf("removing all snapshots: %w", err)
} }
v.UI.Begin("Removing any blobs still present in backup destination store.") v.UI.Beginf("Removing any blobs still present in backup destination store.")
if err := v.PruneBlobs(&PruneOptions{Force: true}); err != nil {
err := v.PruneBlobs(&PruneOptions{Force: true})
if err != nil {
return fmt.Errorf("pruning blobs: %w", err) return fmt.Errorf("pruning blobs: %w", err)
} }
v.UI.Complete("Backup destination store is now empty.") v.UI.Completef("Backup destination store is now empty.")
return nil return nil
} }
@@ -55,7 +60,8 @@ type PruneBlobsResult struct {
// prefer this method over PruneDatabase or PruneBlobs individually // prefer this method over PruneDatabase or PruneBlobs individually
// unless it specifically wants one half. // unless it specifically wants one half.
func (v *Vaultik) Prune(opts *PruneOptions) error { func (v *Vaultik) Prune(opts *PruneOptions) error {
if err := v.EnsureStorageBinding(); err != nil { err := v.EnsureStorageBinding()
if err != nil {
return err return err
} }
// First reconcile local snapshot records against remote metadata: // First reconcile local snapshot records against remote metadata:
@@ -63,12 +69,15 @@ func (v *Vaultik) Prune(opts *PruneOptions) error {
// store is treated as gone. This used to be the separate 'snapshot // store is treated as gone. This used to be the separate 'snapshot
// cleanup' command and is now folded in so a single 'vaultik prune' // cleanup' command and is now folded in so a single 'vaultik prune'
// gets the local index fully back in sync with the destination. // gets the local index fully back in sync with the destination.
if err := v.CleanupLocalSnapshots(); err != nil { err = v.CleanupLocalSnapshots()
if err != nil {
return fmt.Errorf("reconciling local snapshots with remote: %w", err) return fmt.Errorf("reconciling local snapshots with remote: %w", err)
} }
if _, err := v.PruneDatabase(); err != nil { if _, err := v.PruneDatabase(); err != nil {
return fmt.Errorf("pruning local database: %w", err) return fmt.Errorf("pruning local database: %w", err)
} }
return v.PruneBlobs(opts) return v.PruneBlobs(opts)
} }
@@ -92,27 +101,35 @@ func (v *Vaultik) PruneBlobs(opts *PruneOptions) error {
if len(unreferencedBlobs) == 0 { if len(unreferencedBlobs) == 0 {
log.Info("No unreferenced blobs found") log.Info("No unreferenced blobs found")
if opts.JSON { if opts.JSON {
return v.outputPruneBlobsJSON(result) return v.outputPruneBlobsJSON(result)
} }
v.printlnStdout("No unreferenced blobs to remove.") v.printlnStdout("No unreferenced blobs to remove.")
return nil return nil
} }
log.Info("Found unreferenced blobs", "count", len(unreferencedBlobs), "total_size", humanize.Bytes(uint64(totalSize))) log.Info("Found unreferenced blobs", "count", len(unreferencedBlobs), "total_size", humanize.Bytes(uint64(totalSize)))
if !opts.JSON { if !opts.JSON {
v.printfStdout("Found %d unreferenced blob(s) totaling %s\n", len(unreferencedBlobs), humanize.Bytes(uint64(totalSize))) v.stdoutf("Found %d unreferenced blob(s) totaling %s\n", len(unreferencedBlobs), humanize.Bytes(uint64(totalSize)))
} }
if !opts.Force && !opts.JSON { if !opts.Force && !opts.JSON {
v.printfStdout("\nDelete %d unreferenced blob(s)? [y/N] ", len(unreferencedBlobs)) v.stdoutf("\nDelete %d unreferenced blob(s)? [y/N] ", len(unreferencedBlobs))
var confirm string var confirm string
if _, err := v.scanStdin(&confirm); err != nil { if _, err := v.scanStdin(&confirm); err != nil {
v.printlnStdout("Cancelled") v.printlnStdout("Cancelled")
return nil return nil
} }
if strings.ToLower(confirm) != "y" { if strings.ToLower(confirm) != "y" {
v.printlnStdout("Cancelled") v.printlnStdout("Cancelled")
return nil return nil
} }
} }
@@ -123,9 +140,10 @@ func (v *Vaultik) PruneBlobs(opts *PruneOptions) error {
return v.outputPruneBlobsJSON(result) return v.outputPruneBlobsJSON(result)
} }
v.printfStdout("\nDeleted %d blob(s) totaling %s\n", result.BlobsDeleted, humanize.Bytes(uint64(result.BytesFreed))) v.stdoutf("\nDeleted %d blob(s) totaling %s\n", result.BlobsDeleted, humanize.Bytes(uint64(result.BytesFreed)))
if result.BlobsFailed > 0 { if result.BlobsFailed > 0 {
v.printfStdout("Failed to delete %d blob(s)\n", result.BlobsFailed) v.stdoutf("Failed to delete %d blob(s)\n", result.BlobsFailed)
} }
return nil return nil
@@ -140,6 +158,7 @@ func (v *Vaultik) collectReferencedBlobs() (map[string]bool, error) {
if err != nil { if err != nil {
return nil, fmt.Errorf("listing snapshot keys: %w", err) return nil, fmt.Errorf("listing snapshot keys: %w", err)
} }
log.Info("Found manifests in remote storage", "count", len(remoteKeys)) log.Info("Found manifests in remote storage", "count", len(remoteKeys))
allBlobsReferenced := make(map[string]bool) allBlobsReferenced := make(map[string]bool)
@@ -147,18 +166,23 @@ func (v *Vaultik) collectReferencedBlobs() (map[string]bool, error) {
for _, remoteKey := range remoteKeys { for _, remoteKey := range remoteKeys {
log.Debug("Processing manifest", "remote_key", remoteKey) log.Debug("Processing manifest", "remote_key", remoteKey)
manifest, err := v.downloadManifestByKey(remoteKey) manifest, err := v.downloadManifestByKey(remoteKey)
if err != nil { if err != nil {
log.Error("Failed to download manifest", "remote_key", remoteKey, "error", err) log.Error("Failed to download manifest", "remote_key", remoteKey, "error", err)
continue continue
} }
for _, blob := range manifest.Blobs { for _, blob := range manifest.Blobs {
allBlobsReferenced[blob.Hash] = true allBlobsReferenced[blob.Hash] = true
} }
manifestCount++ manifestCount++
} }
log.Info("Processed manifests", "count", manifestCount, "unique_blobs_referenced", len(allBlobsReferenced)) log.Info("Processed manifests", "count", manifestCount, "unique_blobs_referenced", len(allBlobsReferenced))
return allBlobsReferenced, nil return allBlobsReferenced, nil
} }
@@ -166,12 +190,14 @@ func (v *Vaultik) collectReferencedBlobs() (map[string]bool, error) {
func (v *Vaultik) listUniqueSnapshotIDs() ([]string, error) { func (v *Vaultik) listUniqueSnapshotIDs() ([]string, error) {
objectCh := v.Storage.ListStream(v.ctx, "metadata/") objectCh := v.Storage.ListStream(v.ctx, "metadata/")
seen := make(map[string]bool) seen := make(map[string]bool)
var snapshotIDs []string var snapshotIDs []string
for object := range objectCh { for object := range objectCh {
if object.Err != nil { if object.Err != nil {
return nil, fmt.Errorf("listing metadata objects: %w", object.Err) return nil, fmt.Errorf("listing metadata objects: %w", object.Err)
} }
parts := strings.Split(object.Key, "/") parts := strings.Split(object.Key, "/")
if len(parts) >= 2 && parts[0] == "metadata" && parts[1] != "" { if len(parts) >= 2 && parts[0] == "metadata" && parts[1] != "" {
if strings.HasSuffix(object.Key, "/") || strings.Contains(object.Key, "/manifest.json.zst") { if strings.HasSuffix(object.Key, "/") || strings.Contains(object.Key, "/manifest.json.zst") {
@@ -183,12 +209,14 @@ func (v *Vaultik) listUniqueSnapshotIDs() ([]string, error) {
} }
} }
} }
return snapshotIDs, nil return snapshotIDs, nil
} }
// listAllRemoteBlobs returns a map of all blob hashes to their sizes in remote storage // listAllRemoteBlobs returns a map of all blob hashes to their sizes in remote storage
func (v *Vaultik) listAllRemoteBlobs() (map[string]int64, error) { func (v *Vaultik) listAllRemoteBlobs() (map[string]int64, error) {
log.Info("Listing all blobs in storage") log.Info("Listing all blobs in storage")
allBlobs := make(map[string]int64) allBlobs := make(map[string]int64)
blobObjectCh := v.Storage.ListStream(v.ctx, "blobs/") blobObjectCh := v.Storage.ListStream(v.ctx, "blobs/")
@@ -196,6 +224,7 @@ func (v *Vaultik) listAllRemoteBlobs() (map[string]int64, error) {
if object.Err != nil { if object.Err != nil {
return nil, fmt.Errorf("listing blobs: %w", object.Err) return nil, fmt.Errorf("listing blobs: %w", object.Err)
} }
parts := strings.Split(object.Key, "/") parts := strings.Split(object.Key, "/")
if len(parts) == 4 && parts[0] == "blobs" { if len(parts) == 4 && parts[0] == "blobs" {
allBlobs[parts[3]] = object.Size allBlobs[parts[3]] = object.Size
@@ -203,19 +232,24 @@ func (v *Vaultik) listAllRemoteBlobs() (map[string]int64, error) {
} }
log.Info("Found blobs in storage", "count", len(allBlobs)) log.Info("Found blobs in storage", "count", len(allBlobs))
return allBlobs, nil return allBlobs, nil
} }
// findUnreferencedBlobs returns blob hashes not referenced by any manifest and their total size // findUnreferencedBlobs returns blob hashes not referenced by any manifest and their total size
func (v *Vaultik) findUnreferencedBlobs(allBlobs map[string]int64, referenced map[string]bool) ([]string, int64) { func (v *Vaultik) findUnreferencedBlobs(allBlobs map[string]int64, referenced map[string]bool) ([]string, int64) {
var unreferenced []string var (
var totalSize int64 unreferenced []string
totalSize int64
)
for hash, size := range allBlobs { for hash, size := range allBlobs {
if !referenced[hash] { if !referenced[hash] {
unreferenced = append(unreferenced, hash) unreferenced = append(unreferenced, hash)
totalSize += size totalSize += size
} }
} }
return unreferenced, totalSize return unreferenced, totalSize
} }
@@ -226,8 +260,10 @@ func (v *Vaultik) deleteUnreferencedBlobs(unreferencedBlobs []string, allBlobs m
for i, hash := range unreferencedBlobs { for i, hash := range unreferencedBlobs {
blobPath := fmt.Sprintf("blobs/%s/%s/%s", hash[:2], hash[2:4], hash) blobPath := fmt.Sprintf("blobs/%s/%s/%s", hash[:2], hash[2:4], hash)
if err := v.Storage.Delete(v.ctx, blobPath); err != nil { err := v.Storage.Delete(v.ctx, blobPath)
if err != nil {
log.Error("Failed to delete blob", "hash", hash, "error", err) log.Error("Failed to delete blob", "hash", hash, "error", err)
continue continue
} }
@@ -256,5 +292,6 @@ func (v *Vaultik) deleteUnreferencedBlobs(unreferencedBlobs []string, allBlobs m
func (v *Vaultik) outputPruneBlobsJSON(result *PruneBlobsResult) error { func (v *Vaultik) outputPruneBlobsJSON(result *PruneBlobsResult) error {
encoder := json.NewEncoder(v.Stdout) encoder := json.NewEncoder(v.Stdout)
encoder.SetIndent("", " ") encoder.SetIndent("", " ")
return encoder.Encode(result) return encoder.Encode(result)
} }

View File

@@ -79,16 +79,19 @@ func setupPurgeTest(t *testing.T, snapshotIDs []string) *vaultik.Vaultik {
// listRemainingSnapshots returns IDs of all completed snapshots in the database. // listRemainingSnapshots returns IDs of all completed snapshots in the database.
func listRemainingSnapshots(t *testing.T, v *vaultik.Vaultik) []string { func listRemainingSnapshots(t *testing.T, v *vaultik.Vaultik) []string {
t.Helper() t.Helper()
ctx := context.Background() ctx := context.Background()
dbSnaps, err := v.Repositories.Snapshots.ListRecent(ctx, 10000) dbSnaps, err := v.Repositories.Snapshots.ListRecent(ctx, 10000)
require.NoError(t, err) require.NoError(t, err)
var ids []string var ids []string
for _, s := range dbSnaps { for _, s := range dbSnaps {
if s.CompletedAt != nil { if s.CompletedAt != nil {
ids = append(ids, s.ID.String()) ids = append(ids, s.ID.String())
} }
} }
return ids return ids
} }

View File

@@ -37,7 +37,9 @@ func (s *testStorer) Put(ctx context.Context, key string, reader io.Reader) erro
if err != nil { if err != nil {
return err return err
} }
s.data[key] = data s.data[key] = data
return nil return nil
} }
@@ -53,6 +55,7 @@ func (s *testStorer) Get(ctx context.Context, key string) (io.ReadCloser, error)
if !exists { if !exists {
return nil, storage.ErrNotFound return nil, storage.ErrNotFound
} }
return io.NopCloser(bytes.NewReader(data)), nil return io.NopCloser(bytes.NewReader(data)), nil
} }
@@ -64,6 +67,7 @@ func (s *testStorer) Stat(ctx context.Context, key string) (*storage.ObjectInfo,
if !exists { if !exists {
return nil, storage.ErrNotFound return nil, storage.ErrNotFound
} }
return &storage.ObjectInfo{ return &storage.ObjectInfo{
Key: key, Key: key,
Size: int64(len(data)), Size: int64(len(data)),
@@ -75,6 +79,7 @@ func (s *testStorer) Delete(ctx context.Context, key string) error {
defer s.mu.Unlock() defer s.mu.Unlock()
delete(s.data, key) delete(s.data, key)
return nil return nil
} }
@@ -83,11 +88,13 @@ func (s *testStorer) List(ctx context.Context, prefix string) ([]string, error)
defer s.mu.Unlock() defer s.mu.Unlock()
var keys []string var keys []string
for key := range s.data { for key := range s.data {
if prefix == "" || strings.HasPrefix(key, prefix) { if prefix == "" || strings.HasPrefix(key, prefix) {
keys = append(keys, key) keys = append(keys, key)
} }
} }
return keys, nil return keys, nil
} }
@@ -96,6 +103,7 @@ func (s *testStorer) ListStream(ctx context.Context, prefix string) <-chan stora
go func() { go func() {
defer close(ch) defer close(ch)
s.mu.Lock() s.mu.Lock()
defer s.mu.Unlock() defer s.mu.Unlock()
@@ -115,13 +123,16 @@ func (s *testStorer) ListStream(ctx context.Context, prefix string) <-chan stora
func (s *testStorer) hasKey(key string) bool { func (s *testStorer) hasKey(key string) bool {
s.mu.Lock() s.mu.Lock()
defer s.mu.Unlock() defer s.mu.Unlock()
_, exists := s.data[key] _, exists := s.data[key]
return exists return exists
} }
func (s *testStorer) keyCount() int { func (s *testStorer) keyCount() int {
s.mu.Lock() s.mu.Lock()
defer s.mu.Unlock() defer s.mu.Unlock()
return len(s.data) return len(s.data)
} }
@@ -175,6 +186,7 @@ func addBlob(t *testing.T, store *testStorer, hash string) {
// Create zstd compressed data // Create zstd compressed data
var buf bytes.Buffer var buf bytes.Buffer
writer, _ := zstd.NewWriter(&buf) writer, _ := zstd.NewWriter(&buf)
_, _ = writer.Write([]byte("blob data")) _, _ = writer.Write([]byte("blob data"))
_ = writer.Close() _ = writer.Close()
@@ -366,7 +378,7 @@ func TestRemoveAllSnapshots_NoSnapshots(t *testing.T) {
result, err := tv.RemoveAllSnapshots(opts) result, err := tv.RemoveAllSnapshots(opts)
require.NoError(t, err) require.NoError(t, err)
assert.Len(t, result.SnapshotsRemoved, 0) assert.Empty(t, result.SnapshotsRemoved)
// Verify output // Verify output
assert.Contains(t, tv.Stdout.String(), "No snapshots found") assert.Contains(t, tv.Stdout.String(), "No snapshots found")

Some files were not shown because too many files have changed in this diff Show More