sneak/vaultik
1
0
Fork 0
Code Issues 19 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Apply mechanical lint fixes for golangci-lint v2.12.2 rollout

Browse Source 
Auto-remediate style-only findings (wsl_v5, nlreturn, noinlineerr,
modernize, intrange, perfsprint, usetesting, unconvert, errorlint,
gocritic, testifylint) and rename printf-style helpers to f-suffixed
names (goprintffuncname): ui.Writer message methods, cli.ReportErrorf,
database.Fatalf, vaultik stdoutf.
This commit is contained in:
Jeffrey Paul
2026-08-07 17:01:52 +00:00
parent 23d22a0f19
commit 6cf9211407
110 changed files with 2566 additions and 722 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
.golangci.yml
View File

@@ -1,5 +1,9 @@
version: "2"
# Config schema uses the golangci-lint v2 layout (settings live under
# linters.settings, not top-level linters-settings) so that the
# thresholds below are actually applied by golangci-lint >= v2.
run:
timeout: 5m
modules-download-mode: readonly
@@ -14,8 +18,7 @@ linters:
- wsl # Deprecated, replaced by wsl_v5
- wrapcheck # Too verbose for internal packages
- varnamelen # Short names like db, id are idiomatic Go
linters-settings:
settings:
lll:
line-length: 88
funlen:
@@ -27,6 +30,5 @@ linters-settings:
threshold: 100
issues:
exclude-use-default: false
max-issues-per-linter: 0
max-same-issues: 0

4
Dockerfile
View File

@@ -1,6 +1,6 @@
# Lint stage
# golangci/golangci-lint:v2.11.3-alpine, 2026-03-17
FROM golangci/golangci-lint:v2.11.3-alpine@sha256:b1c3de5862ad0a95b4e45a993b0f00415835d687e4f12c845c7493b86c13414e AS lint
# golangci/golangci-lint:v2.12.2-alpine, 2026-08-07
FROM golangci/golangci-lint:v2.12.2-alpine@sha256:91b27804074a0bacea298707f016911e60cf0cdbc6c7bf5ccacb5f0606d18d60 AS lint
RUN apk add --no-cache make build-base

2
Makefile
View File

@@ -55,7 +55,7 @@ clean:
# Install dependencies.
deps:
go mod download
go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest
go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.12.2
# Run tests with coverage.
test-coverage:

4
cmd/vaultik/main.go
View File

@@ -16,9 +16,11 @@ func main() {
panic("could not create CPU profile: " + err.Error())
}
defer func() { _ = f.Close() }()
if err := pprof.StartCPUProfile(f); err != nil {
panic("could not start CPU profile: " + err.Error())
}
defer pprof.StopCPUProfile()
}
@@ -30,7 +32,9 @@ func main() {
panic("could not create memory profile: " + err.Error())
}
defer func() { _ = f.Close() }()
runtime.GC() // get up-to-date statistics
if err := pprof.WriteHeapProfile(f); err != nil {
panic("could not write memory profile: " + err.Error())
}

67
internal/blob/packer.go
View File

@@ -18,6 +18,7 @@ import (
"context"
"database/sql"
"encoding/hex"
"errors"
"fmt"
"io"
"sync"
@@ -124,6 +125,7 @@ type BlobChunkRef struct {
// BlobWithReader wraps a FinishedBlob with its data reader
type BlobWithReader struct {
*FinishedBlob
Reader io.ReadSeeker
TempFile afero.File // Optional, only set for disk-based blobs
InsertedChunkHashes []string // Chunk hashes that were inserted to DB with this blob
@@ -134,14 +136,17 @@ type BlobWithReader struct {
// Returns an error if required configuration fields are missing or invalid.
func NewPacker(cfg PackerConfig) (*Packer, error) {
if len(cfg.Recipients) == 0 {
return nil, fmt.Errorf("recipients are required - blobs must be encrypted")
return nil, errors.New("recipients are required - blobs must be encrypted")
}
if cfg.MaxBlobSize <= 0 {
return nil, fmt.Errorf("max blob size must be positive")
return nil, errors.New("max blob size must be positive")
}
if cfg.Fs == nil {
return nil, fmt.Errorf("filesystem is required")
return nil, errors.New("filesystem is required")
}
return &Packer{
maxBlobSize: cfg.MaxBlobSize,
compressionLevel: cfg.CompressionLevel,
@@ -160,6 +165,7 @@ func NewPacker(cfg PackerConfig) (*Packer, error) {
func (p *Packer) SetBlobHandler(handler BlobHandler) {
p.mu.Lock()
defer p.mu.Unlock()
p.blobHandler = handler
}
@@ -169,6 +175,7 @@ func (p *Packer) SetBlobHandler(handler BlobHandler) {
func (p *Packer) AddPendingChunk(hash string, size int64) {
p.mu.Lock()
defer p.mu.Unlock()
p.pendingChunks = append(p.pendingChunks, PendingChunk{Hash: hash, Size: size})
}
@@ -183,7 +190,8 @@ func (p *Packer) AddChunk(chunk *ChunkRef) error {
// Initialize new blob if needed
if p.currentBlob == nil {
if err := p.startNewBlob(); err != nil {
err := p.startNewBlob()
if err != nil {
return fmt.Errorf("starting new blob: %w", err)
}
}
@@ -202,7 +210,8 @@ func (p *Packer) AddChunk(chunk *ChunkRef) error {
}
// Add chunk to current blob
if err := p.addChunkToCurrentBlob(chunk); err != nil {
err := p.addChunkToCurrentBlob(chunk)
if err != nil {
return err
}
@@ -218,7 +227,8 @@ func (p *Packer) Flush() error {
defer p.mu.Unlock()
if p.currentBlob != nil && len(p.currentBlob.chunks) > 0 {
if err := p.finalizeCurrentBlob(); err != nil {
err := p.finalizeCurrentBlob()
if err != nil {
return fmt.Errorf("finalizing blob: %w", err)
}
}
@@ -253,6 +263,7 @@ func (p *Packer) GetFinishedBlobs() []*FinishedBlob {
blobs := p.finishedBlobs
p.finishedBlobs = make([]*FinishedBlob, 0)
return blobs
}
@@ -267,6 +278,7 @@ func (p *Packer) startNewBlob() error {
if err != nil {
return fmt.Errorf("parsing blob ID: %w", err)
}
blob := &database.Blob{
ID: blobIDTyped,
Hash: types.BlobHash("temp-placeholder-" + blobID), // Temporary placeholder until finalized
@@ -276,6 +288,7 @@ func (p *Packer) startNewBlob() error {
CompressedSize: 0,
UploadedTS: nil,
}
if err := p.repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error {
return p.repos.Blobs.Create(ctx, tx, blob)
}); err != nil {
@@ -294,6 +307,7 @@ func (p *Packer) startNewBlob() error {
if err != nil {
_ = tempFile.Close()
_ = p.fs.Remove(tempFile.Name())
return fmt.Errorf("creating blobgen writer: %w", err)
}
@@ -308,6 +322,7 @@ func (p *Packer) startNewBlob() error {
}
log.Debug("Created new blob container", "blob_id", blobID, "temp_file", tempFile.Name())
return nil
}
@@ -316,6 +331,7 @@ func (p *Packer) addChunkToCurrentBlob(chunk *ChunkRef) error {
// Skip if chunk already in current blob
if p.currentBlob.chunkSet[chunk.Hash] {
log.Debug("Skipping duplicate chunk already in current blob", "chunk_hash", chunk.Hash)
return nil
}
@@ -403,24 +419,31 @@ func (p *Packer) finalizeCurrentBlob() error {
func (p *Packer) closeBlobWriter() (string, int64, error) {
if err := p.currentBlob.writer.Close(); err != nil {
p.cleanupTempFile()
return "", 0, fmt.Errorf("closing blobgen writer: %w", err)
}
if err := p.currentBlob.tempFile.Sync(); err != nil {
p.cleanupTempFile()
return "", 0, fmt.Errorf("syncing temp file: %w", err)
}
finalSize, err := p.currentBlob.tempFile.Seek(0, io.SeekCurrent)
if err != nil {
p.cleanupTempFile()
return "", 0, fmt.Errorf("getting file size: %w", err)
}
if _, err := p.currentBlob.tempFile.Seek(0, io.SeekStart); err != nil {
p.cleanupTempFile()
return "", 0, fmt.Errorf("seeking to start: %w", err)
}
finalHash := p.currentBlob.writer.Sum256()
return hex.EncodeToString(finalHash), finalSize, nil
}
@@ -432,6 +455,7 @@ func (p *Packer) buildChunkRefs() []*BlobChunkRef {
ChunkHash: chunk.Hash, Offset: chunk.Offset, Length: chunk.Size,
})
}
return refs
}
@@ -444,13 +468,16 @@ func (p *Packer) commitBlobToDatabase(blobHash string, finalSize int64, chunksTo
blobIDTyped, parseErr := types.ParseBlobID(p.currentBlob.id)
if parseErr != nil {
p.cleanupTempFile()
return fmt.Errorf("parsing blob ID: %w", parseErr)
}
err := p.repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error {
for _, chunk := range chunksToInsert {
dbChunk := &database.Chunk{ChunkHash: types.ChunkHash(chunk.Hash), Size: chunk.Size}
if err := p.repos.Chunks.Create(ctx, tx, dbChunk); err != nil {
err := p.repos.Chunks.Create(ctx, tx, dbChunk)
if err != nil {
return fmt.Errorf("creating chunk: %w", err)
}
}
@@ -460,7 +487,9 @@ func (p *Packer) commitBlobToDatabase(blobHash string, finalSize int64, chunksTo
BlobID: blobIDTyped, ChunkHash: types.ChunkHash(chunk.Hash),
Offset: chunk.Offset, Length: chunk.Size,
}
if err := p.repos.BlobChunks.Create(ctx, tx, blobChunk); err != nil {
err := p.repos.BlobChunks.Create(ctx, tx, blobChunk)
if err != nil {
return fmt.Errorf("creating blob_chunk: %w", err)
}
}
@@ -469,11 +498,13 @@ func (p *Packer) commitBlobToDatabase(blobHash string, finalSize int64, chunksTo
})
if err != nil {
p.cleanupTempFile()
return fmt.Errorf("finalizing blob transaction: %w", err)
}
log.Debug("Committed blob transaction",
"chunks_inserted", len(chunksToInsert), "blob_chunks_inserted", len(p.currentBlob.chunks))
return nil
}
@@ -482,6 +513,7 @@ func (p *Packer) deliverFinishedBlob(finished *FinishedBlob, insertedChunkHashes
if p.blobHandler != nil {
if _, err := p.currentBlob.tempFile.Seek(0, io.SeekStart); err != nil {
p.cleanupTempFile()
return fmt.Errorf("seeking for handler: %w", err)
}
@@ -492,30 +524,39 @@ func (p *Packer) deliverFinishedBlob(finished *FinishedBlob, insertedChunkHashes
InsertedChunkHashes: insertedChunkHashes,
}
if err := p.blobHandler(blobWithReader); err != nil {
err := p.blobHandler(blobWithReader)
if err != nil {
p.cleanupTempFile()
return fmt.Errorf("blob handler failed: %w", err)
}
p.currentBlob = nil
return nil
}
// No handler - read data for legacy behavior
log.Debug("No blob handler callback configured", "blob_hash", finished.Hash[:8]+"...")
if _, err := p.currentBlob.tempFile.Seek(0, io.SeekStart); err != nil {
p.cleanupTempFile()
return fmt.Errorf("seeking to read data: %w", err)
}
data, err := io.ReadAll(p.currentBlob.tempFile)
if err != nil {
p.cleanupTempFile()
return fmt.Errorf("reading blob data: %w", err)
}
finished.Data = data
p.finishedBlobs = append(p.finishedBlobs, finished)
p.cleanupTempFile()
p.currentBlob = nil
return nil
}
@@ -532,13 +573,15 @@ func (p *Packer) cleanupTempFile() {
func (p *Packer) PackChunks(chunks []*ChunkRef) error {
for _, chunk := range chunks {
err := p.AddChunk(chunk)
if err == ErrBlobSizeLimitExceeded {
if errors.Is(err, ErrBlobSizeLimitExceeded) {
// Finalize current blob and retry
if err := p.FinalizeBlob(); err != nil {
err := p.FinalizeBlob()
if err != nil {
return fmt.Errorf("finalizing blob before retry: %w", err)
}
// Retry the chunk
if err := p.AddChunk(chunk); err != nil {
err = p.AddChunk(chunk)
if err != nil {
return fmt.Errorf("adding chunk %s after finalize: %w", chunk.Hash, err)
}
} else if err != nil {

31
internal/blob/packer_test.go
View File

@@ -6,6 +6,7 @@ import (
"crypto/sha256"
"database/sql"
"encoding/hex"
"errors"
"io"
"testing"
@@ -40,6 +41,7 @@ func TestPacker(t *testing.T) {
t.Fatalf("failed to create test db: %v", err)
}
defer func() { _ = db.Close() }()
repos := database.NewRepositories(db)
cfg := PackerConfig{
@@ -49,6 +51,7 @@ func TestPacker(t *testing.T) {
Repositories: repos,
Fs: afero.NewMemMapFs(),
}
packer, err := NewPacker(cfg)
if err != nil {
t.Fatalf("failed to create packer: %v", err)
@@ -64,6 +67,7 @@ func TestPacker(t *testing.T) {
ChunkHash: types.ChunkHash(hashStr),
Size: int64(len(data)),
}
err = repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error {
return repos.Chunks.Create(ctx, tx, dbChunk)
})
@@ -130,6 +134,7 @@ func TestPacker(t *testing.T) {
t.Fatalf("failed to create test db: %v", err)
}
defer func() { _ = db.Close() }()
repos := database.NewRepositories(db)
cfg := PackerConfig{
@@ -139,6 +144,7 @@ func TestPacker(t *testing.T) {
Repositories: repos,
Fs: afero.NewMemMapFs(),
}
packer, err := NewPacker(cfg)
if err != nil {
t.Fatalf("failed to create packer: %v", err)
@@ -146,7 +152,8 @@ func TestPacker(t *testing.T) {
// Create multiple small chunks
chunks := make([]*ChunkRef, 10)
for i := 0; i < 10; i++ {
for i := range 10 {
data := bytes.Repeat([]byte{byte(i)}, 1000)
hash := sha256.Sum256(data)
hashStr := hex.EncodeToString(hash[:])
@@ -156,6 +163,7 @@ func TestPacker(t *testing.T) {
ChunkHash: types.ChunkHash(hashStr),
Size: int64(len(data)),
}
err = repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error {
return repos.Chunks.Create(ctx, tx, dbChunk)
})
@@ -198,9 +206,11 @@ func TestPacker(t *testing.T) {
if chunkRef.Offset != expectedOffset {
t.Errorf("chunk %d: expected offset %d, got %d", i, expectedOffset, chunkRef.Offset)
}
if chunkRef.Length != 1000 {
t.Errorf("chunk %d: expected length 1000, got %d", i, chunkRef.Length)
}
expectedOffset += chunkRef.Length
}
})
@@ -212,6 +222,7 @@ func TestPacker(t *testing.T) {
t.Fatalf("failed to create test db: %v", err)
}
defer func() { _ = db.Close() }()
repos := database.NewRepositories(db)
// Small blob size limit to force multiple blobs
@@ -222,6 +233,7 @@ func TestPacker(t *testing.T) {
Repositories: repos,
Fs: afero.NewMemMapFs(),
}
packer, err := NewPacker(cfg)
if err != nil {
t.Fatalf("failed to create packer: %v", err)
@@ -229,7 +241,8 @@ func TestPacker(t *testing.T) {
// Create chunks that will exceed the limit
chunks := make([]*ChunkRef, 10)
for i := 0; i < 10; i++ {
for i := range 10 {
data := bytes.Repeat([]byte{byte(i)}, 1000) // 1KB each
hash := sha256.Sum256(data)
hashStr := hex.EncodeToString(hash[:])
@@ -239,6 +252,7 @@ func TestPacker(t *testing.T) {
ChunkHash: types.ChunkHash(hashStr),
Size: int64(len(data)),
}
err = repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error {
return repos.Chunks.Create(ctx, tx, dbChunk)
})
@@ -257,14 +271,17 @@ func TestPacker(t *testing.T) {
// Add chunks and handle size limit errors
for _, chunk := range chunks {
err := packer.AddChunk(chunk)
if err == ErrBlobSizeLimitExceeded {
if errors.Is(err, ErrBlobSizeLimitExceeded) {
// Finalize current blob
if err := packer.FinalizeBlob(); err != nil {
err := packer.FinalizeBlob()
if err != nil {
t.Fatalf("failed to finalize blob: %v", err)
}
blobCount++
// Retry adding the chunk
if err := packer.AddChunk(chunk); err != nil {
err = packer.AddChunk(chunk)
if err != nil {
t.Fatalf("failed to add chunk after finalize: %v", err)
}
} else if err != nil {
@@ -301,6 +318,7 @@ func TestPacker(t *testing.T) {
t.Fatalf("failed to create test db: %v", err)
}
defer func() { _ = db.Close() }()
repos := database.NewRepositories(db)
// Generate test identity (using the one from parent test)
@@ -311,6 +329,7 @@ func TestPacker(t *testing.T) {
Repositories: repos,
Fs: afero.NewMemMapFs(),
}
packer, err := NewPacker(cfg)
if err != nil {
t.Fatalf("failed to create packer: %v", err)
@@ -326,6 +345,7 @@ func TestPacker(t *testing.T) {
ChunkHash: types.ChunkHash(hashStr),
Size: int64(len(data)),
}
err = repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error {
return repos.Chunks.Create(ctx, tx, dbChunk)
})
@@ -342,6 +362,7 @@ func TestPacker(t *testing.T) {
if err := packer.AddChunk(chunk); err != nil {
t.Fatalf("failed to add chunk: %v", err)
}
if err := packer.Flush(); err != nil {
t.Fatalf("failed to flush: %v", err)
}

2
internal/blobgen/compress.go
View File

@@ -28,6 +28,7 @@ func CompressData(data []byte, compressionLevel int, recipients []string) (*Comp
// Write data
if _, err := w.Write(data); err != nil {
_ = w.Close()
return nil, fmt.Errorf("writing data: %w", err)
}
@@ -68,6 +69,7 @@ func CompressStream(dst io.Writer, src io.Reader, compressionLevel int, recipien
if err := w.Close(); err != nil {
return 0, "", fmt.Errorf("closing writer: %w", err)
}
closed = true
return w.BytesWritten(), hex.EncodeToString(w.Sum256()), nil

13
internal/blobgen/compress_test.go
View File

@@ -20,13 +20,14 @@ const testRecipient = "age1cplgrwj77ta54dnmydvvmzn64ltk83ankxl5sww04mrtmu62kv3s8
// cause a double close.
func TestCompressStreamNoDoubleClose(t *testing.T) {
input := []byte("regression test data for issue #28 double-close fix")
var buf bytes.Buffer
written, hash, err := CompressStream(&buf, bytes.NewReader(input), 3, []string{testRecipient})
require.NoError(t, err, "CompressStream should not return an error")
assert.True(t, written > 0, "expected bytes written > 0")
assert.Positive(t, written, "expected bytes written > 0")
assert.NotEmpty(t, hash, "expected non-empty hash")
assert.True(t, buf.Len() > 0, "expected non-empty output")
assert.Positive(t, buf.Len(), "expected non-empty output")
}
// TestCompressStreamLargeInput exercises CompressStream with a larger payload
@@ -37,9 +38,10 @@ func TestCompressStreamLargeInput(t *testing.T) {
require.NoError(t, err)
var buf bytes.Buffer
written, hash, err := CompressStream(&buf, bytes.NewReader(data), 3, []string{testRecipient})
require.NoError(t, err)
assert.True(t, written > 0)
assert.Positive(t, written)
assert.NotEmpty(t, hash)
}
@@ -47,6 +49,7 @@ func TestCompressStreamLargeInput(t *testing.T) {
// without double-close issues.
func TestCompressStreamEmptyInput(t *testing.T) {
var buf bytes.Buffer
_, hash, err := CompressStream(&buf, strings.NewReader(""), 3, []string{testRecipient})
require.NoError(t, err)
assert.NotEmpty(t, hash)
@@ -58,7 +61,7 @@ func TestCompressDataNoDoubleClose(t *testing.T) {
input := []byte("CompressData regression test for double-close")
result, err := CompressData(input, 3, []string{testRecipient})
require.NoError(t, err)
assert.True(t, result.CompressedSize > 0)
assert.True(t, result.UncompressedSize == int64(len(input)))
assert.Positive(t, result.CompressedSize)
assert.Equal(t, result.UncompressedSize, int64(len(input)))
assert.NotEmpty(t, result.SHA256)
}

2
internal/blobgen/reader.go
View File

@@ -53,12 +53,14 @@ func NewReader(r io.Reader, identity age.Identity) (*Reader, error) {
func (r *Reader) Read(p []byte) (n int, err error) {
n, err = r.teeReader.Read(p)
r.bytesRead += int64(n)
return n, err
}
// Close closes the decompressor
func (r *Reader) Close() error {
r.decompressor.Close()
return nil
}

17
internal/blobgen/writer.go
View File

@@ -36,11 +36,13 @@ func NewWriter(w io.Writer, compressionLevel int, recipients []string) (*Writer,
// Parse recipients
var ageRecipients []age.Recipient
for _, recipient := range recipients {
r, err := age.ParseX25519Recipient(recipient)
if err != nil {
return nil, fmt.Errorf("parsing recipient %s: %w", recipient, err)
}
ageRecipients = append(ageRecipients, r)
}
@@ -51,10 +53,7 @@ func NewWriter(w io.Writer, compressionLevel int, recipients []string) (*Writer,
}
// Calculate compression concurrency: CPUs - 2, minimum 1
concurrency := runtime.NumCPU() - 2
if concurrency < 1 {
concurrency = 1
}
concurrency := max(runtime.NumCPU()-2, 1)
// Create compression writer with encryption as destination
compressor, err := zstd.NewWriter(encWriter,
@@ -63,6 +62,7 @@ func NewWriter(w io.Writer, compressionLevel int, recipients []string) (*Writer,
)
if err != nil {
_ = encWriter.Close()
return nil, fmt.Errorf("creating compression writer: %w", err)
}
@@ -82,18 +82,21 @@ func NewWriter(w io.Writer, compressionLevel int, recipients []string) (*Writer,
func (w *Writer) Write(p []byte) (n int, err error) {
n, err = w.teeWriter.Write(p)
w.bytesWritten += int64(n)
return n, err
}
// Close closes all layers and returns any errors
func (w *Writer) Close() error {
// Close compressor first
if err := w.compressor.Close(); err != nil {
err := w.compressor.Close()
if err != nil {
return fmt.Errorf("closing compressor: %w", err)
}
// Then close encryptor
if err := w.encryptor.Close(); err != nil {
err = w.encryptor.Close()
if err != nil {
return fmt.Errorf("closing encryptor: %w", err)
}
@@ -109,6 +112,7 @@ func (w *Writer) Sum256() []byte {
firstHash := w.hasher.Sum(nil)
// Second hash: SHA256(firstHash) - this is the blob ID
secondHash := sha256.Sum256(firstHash)
return secondHash[:]
}
@@ -123,5 +127,6 @@ func validateCompressionLevel(level int) error {
if level < 1 || level > 19 {
return fmt.Errorf("invalid compression level %d: must be between 1 and 19", level)
}
return nil
}

11
internal/chunker/chunker.go
View File

@@ -3,6 +3,7 @@ package chunker
import (
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io"
"os"
@@ -50,13 +51,15 @@ func (c *Chunker) ChunkReader(r io.Reader) ([]Chunk, error) {
defer chunker.Release()
var chunks []Chunk
offset := int64(0)
for {
chunk, err := chunker.Next()
if err == io.EOF {
if errors.Is(err, io.EOF) {
break
}
if err != nil {
return nil, fmt.Errorf("reading chunk: %w", err)
}
@@ -104,9 +107,10 @@ func (c *Chunker) ChunkReaderStreaming(r io.Reader, callback ChunkCallback) (str
for {
chunk, err := chunker.Next()
if err == io.EOF {
if errors.Is(err, io.EOF) {
break
}
if err != nil {
return "", fmt.Errorf("reading chunk: %w", err)
}
@@ -143,7 +147,8 @@ func (c *Chunker) ChunkFile(path string) ([]Chunk, error) {
return nil, fmt.Errorf("opening file: %w", err)
}
defer func() {
if err := file.Close(); err != nil && err.Error() != "invalid argument" {
err := file.Close()
if err != nil && err.Error() != "invalid argument" {
// Log error or handle as needed
_ = err
}

4
internal/chunker/chunker_isolated_test.go
View File

@@ -42,7 +42,7 @@ func TestChunkerExpectedChunkCount(t *testing.T) {
// Create data with some variation to trigger chunk boundaries
data := make([]byte, tt.fileSize)
for i := 0; i < len(data); i++ {
for i := range data {
// Use a pattern that should create boundaries
data[i] = byte((i * 17) ^ (i >> 5))
}
@@ -59,6 +59,7 @@ func TestChunkerExpectedChunkCount(t *testing.T) {
t.Errorf("too few chunks: got %d, expected at least %d",
len(chunks), tt.minExpected)
}
if len(chunks) > tt.maxExpected {
t.Errorf("too many chunks: got %d, expected at most %d",
len(chunks), tt.maxExpected)
@@ -69,6 +70,7 @@ func TestChunkerExpectedChunkCount(t *testing.T) {
for _, chunk := range chunks {
reconstructed = append(reconstructed, chunk.Data...)
}
if !bytes.Equal(data, reconstructed) {
t.Error("reconstructed data doesn't match original")
}

3
internal/chunker/chunker_test.go
View File

@@ -60,6 +60,7 @@ func TestChunker(t *testing.T) {
if chunk.Offset != expectedOffset {
t.Errorf("chunk %d: expected offset %d, got %d", i, expectedOffset, chunk.Offset)
}
expectedOffset += chunk.Size
}
})
@@ -90,6 +91,7 @@ func TestChunker(t *testing.T) {
if chunks1[i].Hash != chunks2[i].Hash {
t.Errorf("chunk %d: different hashes", i)
}
if chunks1[i].Size != chunks2[i].Size {
t.Errorf("chunk %d: different sizes", i)
}
@@ -121,6 +123,7 @@ func TestChunkBoundaries(t *testing.T) {
if i < len(chunks)-1 && chunk.Size < minSize {
t.Errorf("chunk %d size %d is below minimum %d", i, chunk.Size, minSize)
}
if chunk.Size > maxSize {
t.Errorf("chunk %d size %d exceeds maximum %d", i, chunk.Size, maxSize)
}

21
internal/chunker/fastcdc.go
View File

@@ -28,7 +28,7 @@ type ReusableChunker struct {
// reusableChunkerPool pools ReusableChunker instances to avoid allocations.
var reusableChunkerPool = sync.Pool{
New: func() interface{} {
New: func() any {
return &ReusableChunker{}
},
}
@@ -39,17 +39,20 @@ var bufferPools = sync.Map{}
func getBuffer(size int) []byte {
poolI, _ := bufferPools.LoadOrStore(size, &sync.Pool{
New: func() interface{} {
New: func() any {
buf := make([]byte, size)
return &buf
},
})
pool := poolI.(*sync.Pool)
return *pool.Get().(*[]byte)
}
func putBuffer(buf []byte) {
size := cap(buf)
poolI, ok := bufferPools.Load(size)
if ok {
pool := poolI.(*sync.Pool)
@@ -77,6 +80,7 @@ func AcquireReusableChunker(rd io.Reader, minSize, avgSize, maxSize int) *Reusab
if c.buf != nil {
putBuffer(c.buf)
}
c.buf = getBuffer(bufSize)
} else {
// Restore buffer to full capacity (may have been truncated by previous EOF)
@@ -120,6 +124,7 @@ func (c *ReusableChunker) fillBuffer() error {
if c.eof {
c.buf = c.buf[:n]
return nil
}
@@ -134,15 +139,18 @@ func (c *ReusableChunker) fillBuffer() error {
} else if err != nil {
return err
}
return nil
}
// Next returns the next chunk or io.EOF when done.
// The returned Data slice is only valid until the next call to Next.
func (c *ReusableChunker) Next() (FastCDCChunk, error) {
if err := c.fillBuffer(); err != nil {
err := c.fillBuffer()
if err != nil {
return FastCDCChunk{}, err
}
if len(c.buf) == 0 {
return FastCDCChunk{}, io.EOF
}
@@ -189,13 +197,6 @@ func (c *ReusableChunker) nextChunk(data []byte) (int, uint64) {
return i, fp
}
func min(a, b int) int {
if a < b {
return a
}
return b
}
// 256 random uint64s for the rolling hash function (from FastCDC paper)
var table = [256]uint64{
0xe80e8d55032474b3, 0x11b25b61f5924e15, 0x03aa5bd82a9eb669, 0xc45a153ef107a38c,

27
internal/cli/app.go
View File

@@ -44,9 +44,11 @@ func setupGlobals(lc fx.Lifecycle, g *globals.Globals, v *vaultik.Vaultik, opts
lc.Append(fx.Hook{
OnStart: func(ctx context.Context) error {
g.StartTime = time.Now().UTC()
if opts.Cron || opts.Quiet {
v.UI.SetQuiet(true)
}
return nil
},
})
@@ -56,12 +58,12 @@ func setupGlobals(lc fx.Lifecycle, g *globals.Globals, v *vaultik.Vaultik, opts
// blank line. Used both from the fx hook (for subcommand invocations) and
// from the root cobra Run handler (for `vaultik` with no subcommand).
func writeStartupBanner(w *ui.Writer, startTime time.Time, shortCommit string) {
w.Banner("%s %s by %s (commit %s, built on %s) starting up at %s.",
w.Bannerf("%s %s by %s (commit %s, built on %s) starting up at %s.",
globals.Appname, globals.Version, globals.Author,
shortCommit, globals.CommitDate,
startTime.Format(time.RFC3339))
w.Banner("%s", globals.Homepage)
w.Banner("")
w.Bannerf("%s", globals.Homepage)
w.Bannerf("")
}
// NewApp creates a new fx application with common modules.
@@ -105,6 +107,7 @@ func cleanStartupError(err error) error {
if idx := strings.LastIndex(msg, "): "); idx >= 0 {
msg = msg[idx+3:]
}
return errors.New(msg)
}
@@ -122,7 +125,8 @@ func RunApp(ctx context.Context, app *fx.App) error {
defer cancel()
// Start the app
if err := app.Start(ctx); err != nil {
err := app.Start(ctx)
if err != nil {
return cleanStartupError(err)
}
@@ -130,6 +134,7 @@ func RunApp(ctx context.Context, app *fx.App) error {
shutdownComplete := make(chan struct{})
go func() {
defer close(shutdownComplete)
<-sigChan
log.Notice("Received interrupt signal, shutting down gracefully...")
@@ -137,7 +142,8 @@ func RunApp(ctx context.Context, app *fx.App) error {
shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 30*time.Second)
defer shutdownCancel()
if err := app.Stop(shutdownCtx); err != nil {
err := app.Stop(shutdownCtx)
if err != nil {
log.Error("Error during shutdown", "error", err)
}
}()
@@ -149,9 +155,11 @@ func RunApp(ctx context.Context, app *fx.App) error {
return nil
case <-ctx.Done():
// Context cancelled (shouldn't happen in normal operation)
if err := app.Stop(context.Background()); err != nil {
err := app.Stop(context.Background())
if err != nil {
log.Error("Error stopping app", "error", err)
}
return ctx.Err()
case <-app.Done():
// App finished running (e.g., backup completed)
@@ -166,19 +174,24 @@ func RunApp(ctx context.Context, app *fx.App) error {
func RunWithApp(ctx context.Context, opts AppOptions) error {
// Acquire PID lock to prevent concurrent instances
lockDir := filepath.Join(xdg.DataHome, "vaultik")
lock, err := pidlock.Acquire(lockDir)
if err != nil {
if errors.Is(err, pidlock.ErrAlreadyRunning) {
return fmt.Errorf("cannot start: %w", err)
}
return fmt.Errorf("failed to acquire lock: %w", err)
}
defer func() {
if err := lock.Release(); err != nil {
err := lock.Release()
if err != nil {
log.Warn("Failed to release PID lock", "error", err)
}
}()
app := NewApp(opts)
return RunApp(ctx, app)
}

33
internal/cli/config.go
View File

@@ -1,6 +1,7 @@
package cli
import (
"errors"
"fmt"
"os"
"os/exec"
@@ -240,16 +241,20 @@ on macOS, ~/.config/ on Linux, /etc/vaultik/ as root).`,
}
dir := filepath.Dir(path)
if err := os.MkdirAll(dir, 0o755); err != nil {
err := os.MkdirAll(dir, 0o755)
if err != nil {
return fmt.Errorf("creating config directory %s: %w", dir, err)
}
if err := os.WriteFile(path, []byte(defaultConfigTemplate), 0o600); err != nil {
err = os.WriteFile(path, []byte(defaultConfigTemplate), 0o600)
if err != nil {
return fmt.Errorf("writing config file: %w", err)
}
fmt.Printf("Config written to %s\n", path)
fmt.Println("Edit it to set your age_recipients, snapshots, and storage_url.")
return nil
},
}
@@ -276,6 +281,7 @@ func newConfigEditCommand() *cobra.Command {
ed.Stdin = os.Stdin
ed.Stdout = os.Stdout
ed.Stderr = os.Stderr
return ed.Run()
},
}
@@ -305,6 +311,7 @@ func newConfigGetCommand() *cobra.Command {
if node.Kind == yaml.ScalarNode {
fmt.Println(node.Value)
return nil
}
@@ -312,7 +319,9 @@ func newConfigGetCommand() *cobra.Command {
if err != nil {
return fmt.Errorf("marshaling value: %w", err)
}
fmt.Print(string(out))
return nil
},
}
@@ -363,6 +372,7 @@ Examples:
}
fmt.Printf("%s = %s\n", args[0], args[1])
return nil
},
}
@@ -399,8 +409,9 @@ func yamlPathGet(root *yaml.Node, keys []string) (*yaml.Node, error) {
node := root
if node.Kind == yaml.DocumentNode {
if len(node.Content) == 0 {
return nil, fmt.Errorf("empty config file")
return nil, errors.New("empty config file")
}
node = node.Content[0]
}
@@ -408,13 +419,16 @@ func yamlPathGet(root *yaml.Node, keys []string) (*yaml.Node, error) {
switch node.Kind {
case yaml.MappingNode:
found := false
for j := 0; j+1 < len(node.Content); j += 2 {
if node.Content[j].Value == key {
node = node.Content[j+1]
found = true
break
}
}
if !found {
return nil, fmt.Errorf("key not found: %s", strings.Join(keys[:i+1], "."))
}
@@ -423,9 +437,11 @@ func yamlPathGet(root *yaml.Node, keys []string) (*yaml.Node, error) {
if err != nil {
return nil, fmt.Errorf("key %q is a list; use a numeric index", strings.Join(keys[:i], "."))
}
if idx < 0 || idx >= len(node.Content) {
return nil, fmt.Errorf("index %d out of range for %s (len %d)", idx, strings.Join(keys[:i], "."), len(node.Content))
}
node = node.Content[idx]
default:
return nil, fmt.Errorf("key %q is not a map or list", strings.Join(keys[:i], "."))
@@ -445,6 +461,7 @@ func yamlPathSet(root *yaml.Node, keys []string, value string) error {
if len(node.Content) == 0 {
node.Content = []*yaml.Node{{Kind: yaml.MappingNode}}
}
node = node.Content[0]
}
@@ -454,19 +471,23 @@ func yamlPathSet(root *yaml.Node, keys []string, value string) error {
switch node.Kind {
case yaml.MappingNode:
var valueNode *yaml.Node
for j := 0; j+1 < len(node.Content); j += 2 {
if node.Content[j].Value == key {
valueNode = node.Content[j+1]
break
}
}
if valueNode == nil {
keyNode := &yaml.Node{Kind: yaml.ScalarNode, Value: key}
valueNode = &yaml.Node{Kind: yaml.MappingNode}
if last {
valueNode = &yaml.Node{Kind: yaml.ScalarNode, Value: value}
}
node.Content = append(node.Content, keyNode, valueNode)
} else if last {
setScalar(valueNode, value)
@@ -479,18 +500,22 @@ func yamlPathSet(root *yaml.Node, keys []string, value string) error {
if err != nil {
return fmt.Errorf("key %q is a list; use a numeric index", strings.Join(keys[:i], "."))
}
if idx < 0 || idx > len(node.Content) {
return fmt.Errorf("index %d out of range for %s (len %d)", idx, strings.Join(keys[:i], "."), len(node.Content))
}
if idx == len(node.Content) {
newNode := &yaml.Node{Kind: yaml.MappingNode}
if last {
newNode = &yaml.Node{Kind: yaml.ScalarNode, Value: value}
}
node.Content = append(node.Content, newNode)
} else if last {
setScalar(node.Content[idx], value)
}
node = node.Content[idx]
default:
@@ -516,8 +541,10 @@ func configPathForInit() string {
if rootFlags.ConfigPath != "" {
return rootFlags.ConfigPath
}
if envPath := os.Getenv("VAULTIK_CONFIG"); envPath != "" {
return envPath
}
return DefaultConfigPath()
}

22
internal/cli/config_test.go
View File

@@ -12,7 +12,9 @@ import (
// that unmarshals into the Config struct with the expected snapshots.
func TestDefaultConfigTemplateParses(t *testing.T) {
var cfg config.Config
if err := yaml.Unmarshal([]byte(defaultConfigTemplate), &cfg); err != nil {
err := yaml.Unmarshal([]byte(defaultConfigTemplate), &cfg)
if err != nil {
t.Fatalf("default config template is not valid YAML: %v", err)
}
@@ -24,9 +26,11 @@ func TestDefaultConfigTemplateParses(t *testing.T) {
if !ok {
t.Fatal("expected 'home' snapshot in default config")
}
if len(home.Paths) == 0 {
t.Error("home snapshot should have at least one path")
}
if len(home.Exclude) == 0 {
t.Error("home snapshot should have exclude patterns")
}
@@ -35,9 +39,11 @@ func TestDefaultConfigTemplateParses(t *testing.T) {
if !ok {
t.Fatal("expected 'apps' snapshot in default config")
}
if len(apps.Paths) != 1 || apps.Paths[0] != "/Applications" {
t.Errorf("apps snapshot should back up /Applications, got %v", apps.Paths)
}
if len(apps.Exclude) == 0 {
t.Error("apps snapshot should have exclude patterns")
}
@@ -58,10 +64,14 @@ snapshots:
func parseTestYAML(t *testing.T) *yaml.Node {
t.Helper()
var root yaml.Node
if err := yaml.Unmarshal([]byte(testYAML), &root); err != nil {
err := yaml.Unmarshal([]byte(testYAML), &root)
if err != nil {
t.Fatalf("parsing test yaml: %v", err)
}
return &root
}
@@ -91,11 +101,14 @@ func TestYAMLPathGet(t *testing.T) {
if err == nil {
t.Fatalf("expected error for %q", tt.path)
}
return
}
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if node.Value != tt.want {
t.Errorf("get %q = %q, want %q", tt.path, node.Value, tt.want)
}
@@ -115,6 +128,7 @@ func TestYAMLPathSet(t *testing.T) {
if err := yamlPathSet(root, splitPath("s3.endpoint"), "s3.example.com"); err != nil {
t.Fatalf("set s3.endpoint: %v", err)
}
if err := yamlPathSet(root, splitPath("newmap.newkey"), "val"); err != nil {
t.Fatalf("set newmap.newkey: %v", err)
}
@@ -123,9 +137,11 @@ func TestYAMLPathSet(t *testing.T) {
if err := yamlPathSet(root, splitPath("age_recipients.0"), "age1bbb"); err != nil {
t.Fatalf("set age_recipients.0: %v", err)
}
if err := yamlPathSet(root, splitPath("age_recipients.1"), "age1ccc"); err != nil {
t.Fatalf("append age_recipients.1: %v", err)
}
if err := yamlPathSet(root, splitPath("age_recipients.5"), "age1ddd"); err == nil {
t.Error("expected out-of-range append to fail")
}
@@ -135,6 +151,7 @@ func TestYAMLPathSet(t *testing.T) {
if err != nil {
t.Fatalf("marshal: %v", err)
}
text := string(out)
for _, want := range []string{"newbucket", "s3.example.com", "newkey: val", "# top comment", "# inline comment", "age1bbb", "age1ccc"} {
@@ -147,6 +164,7 @@ func TestYAMLPathSet(t *testing.T) {
if err != nil {
t.Fatalf("get after set: %v", err)
}
if got.Value != "newbucket" {
t.Errorf("s3.bucket = %q after set, want newbucket", got.Value)
}

4
internal/cli/database.go
View File

@@ -66,6 +66,7 @@ Use --force to skip the confirmation prompt.`,
// Check if database exists
if _, err := os.Stat(dbPath); os.IsNotExist(err) {
fmt.Printf("Database does not exist: %s\n", dbPath)
return nil
}
@@ -73,9 +74,11 @@ Use --force to skip the confirmation prompt.`,
if !force {
fmt.Printf("This will delete the local state database at:\n %s\n\n", dbPath)
fmt.Print("Are you sure? Type 'yes' to confirm: ")
var confirm string
if _, err := fmt.Scanln(&confirm); err != nil || confirm != "yes" {
fmt.Println("Aborted.")
return nil
}
}
@@ -97,6 +100,7 @@ Use --force to skip the confirmation prompt.`,
}
log.Info("Local state database deleted", "path", dbPath)
return nil
},
}

6
internal/cli/duration.go
View File

@@ -1,6 +1,7 @@
package cli
import (
"errors"
"fmt"
"regexp"
"strconv"
@@ -25,7 +26,7 @@ func parseDuration(s string) (time.Duration, error) {
// Extended duration parsing
// Check for negative values
if strings.HasPrefix(strings.TrimSpace(s), "-") {
return 0, fmt.Errorf("negative durations are not supported")
return 0, errors.New("negative durations are not supported")
}
// Pattern matches: number + unit, repeated
@@ -48,6 +49,7 @@ func parseDuration(s string) (time.Duration, error) {
}
var d time.Duration
switch unit {
// Standard time units
case "ns", "nanosecond", "nanoseconds":
@@ -75,7 +77,7 @@ func parseDuration(s string) (time.Duration, error) {
d = time.Duration(value * float64(365*24*time.Hour))
default:
// Try parsing as standard Go duration unit
testStr := fmt.Sprintf("1%s", unit)
testStr := "1" + unit
if _, err := time.ParseDuration(testStr); err == nil {
// It's a valid Go duration unit, parse the full value
fullStr := fmt.Sprintf("%g%s", value, unit)

1
internal/cli/duration_test.go
View File

@@ -185,6 +185,7 @@ func TestParseDuration(t *testing.T) {
if tt.wantErr {
assert.Error(t, err, "expected error for input %q", tt.input)
return
}

15
internal/cli/entry.go
View File

@@ -19,24 +19,26 @@ func CLIEntry() {
if len(short) > 12 {
short = short[:12]
}
writeStartupBanner(ui.New(os.Stdout), time.Now().UTC(), short)
}
rootCmd := NewRootCommand()
rootCmd.SilenceErrors = true
if err := rootCmd.Execute(); err != nil {
ReportError("%s", err.Error())
err := rootCmd.Execute()
if err != nil {
ReportErrorf("%s", err.Error())
os.Exit(1)
}
}
// ReportError emits a user-facing error to stderr in the standard
// ReportErrorf emits a user-facing error to stderr in the standard
// 🛑 ERROR: format. Use it from goroutine error paths (where returning
// an error to cobra isn't an option) and anywhere else a CLI command
// must surface a failure outside the normal RunE return path.
func ReportError(format string, args ...any) {
ui.New(os.Stderr).Error(format, args...)
func ReportErrorf(format string, args ...any) {
ui.New(os.Stderr).Errorf(format, args...)
}
// bannerSuppressedInArgs reports whether any of args is a flag that
@@ -48,10 +50,12 @@ func bannerSuppressedInArgs(args []string) bool {
if a == "--" {
return false
}
switch a {
case "--quiet", "-q", "--cron":
return true
}
if strings.HasPrefix(a, "--quiet=") || strings.HasPrefix(a, "--cron=") {
return true
}
@@ -64,5 +68,6 @@ func bannerSuppressedInArgs(args []string) bool {
}
}
}
return false
}

6
internal/cli/entry_test.go
View File

@@ -21,12 +21,15 @@ func TestCLIEntry(t *testing.T) {
expectedCommands := []string{"config", "snapshot", "prune", "info", "version", "remote", "database"}
for _, expected := range expectedCommands {
found := false
for _, cmd := range cmd.Commands() {
if cmd.Use == expected || cmd.Name() == expected {
found = true
break
}
}
if !found {
t.Errorf("Expected command '%s' not found", expected)
}
@@ -41,12 +44,15 @@ func TestCLIEntry(t *testing.T) {
expectedSubCommands := []string{"create", "list", "purge", "verify", "remove", "restore"}
for _, expected := range expectedSubCommands {
found := false
for _, subcmd := range snapshotCmd.Commands() {
if subcmd.Use == expected || subcmd.Name() == expected {
found = true
break
}
}
if !found {
t.Errorf("Expected snapshot subcommand '%s' not found", expected)
}

15
internal/cli/info.go
View File

@@ -2,6 +2,7 @@ package cli
import (
"context"
"errors"
"os"
"github.com/spf13/cobra"
@@ -31,6 +32,7 @@ func NewInfoCommand() *cobra.Command {
// Use the app framework
rootFlags := GetRootFlags()
return RunWithApp(cmd.Context(), AppOptions{
ConfigPath: configPath,
LogOptions: log.LogOptions{
@@ -44,21 +46,26 @@ func NewInfoCommand() *cobra.Command {
lc.Append(fx.Hook{
OnStart: func(ctx context.Context) error {
go func() {
if err := v.ShowInfo(); err != nil {
if err != context.Canceled {
err := v.ShowInfo()
if err != nil {
if !errors.Is(err, context.Canceled) {
log.Error("Failed to show info", "error", err)
ReportError("Failed to show info: %v", err)
ReportErrorf("Failed to show info: %v", err)
os.Exit(1)
}
}
if err := v.Shutdowner.Shutdown(); err != nil {
err = v.Shutdowner.Shutdown()
if err != nil {
log.Error("Failed to shutdown", "error", err)
}
}()
return nil
},
OnStop: func(ctx context.Context) error {
v.Cancel()
return nil
},
})

15
internal/cli/prune.go
View File

@@ -2,6 +2,7 @@ package cli
import (
"context"
"errors"
"os"
"github.com/spf13/cobra"
@@ -39,6 +40,7 @@ work (e.g. after a crashed backup or to reclaim storage).`,
// Use the app framework like other commands
rootFlags := GetRootFlags()
return RunWithApp(cmd.Context(), AppOptions{
ConfigPath: configPath,
LogOptions: log.LogOptions{
@@ -54,26 +56,31 @@ work (e.g. after a crashed backup or to reclaim storage).`,
// Start the prune operation in a goroutine
go func() {
// Run the prune operation
if err := v.Prune(opts); err != nil {
if err != context.Canceled {
err := v.Prune(opts)
if err != nil {
if !errors.Is(err, context.Canceled) {
if !opts.JSON {
log.Error("Prune operation failed", "error", err)
ReportError("Prune failed: %v", err)
ReportErrorf("Prune failed: %v", err)
}
os.Exit(1)
}
}
// Shutdown the app when prune completes
if err := v.Shutdowner.Shutdown(); err != nil {
err = v.Shutdowner.Shutdown()
if err != nil {
log.Error("Failed to shutdown", "error", err)
}
}()
return nil
},
OnStop: func(ctx context.Context) error {
log.Debug("Stopping prune operation")
v.Cancel()
return nil
},
})

33
internal/cli/remote.go
View File

@@ -2,7 +2,7 @@ package cli
import (
"context"
"fmt"
"errors"
"os"
"github.com/spf13/cobra"
@@ -41,7 +41,7 @@ This is destructive and irreversible. Requires --force.`,
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, args []string) error {
if !force {
return fmt.Errorf("remote nuke requires --force (this deletes ALL remote snapshots and blobs)")
return errors.New("remote nuke requires --force (this deletes ALL remote snapshots and blobs)")
}
configPath, err := ResolveConfigPath()
@@ -50,6 +50,7 @@ This is destructive and irreversible. Requires --force.`,
}
rootFlags := GetRootFlags()
return RunWithApp(cmd.Context(), AppOptions{
ConfigPath: configPath,
LogOptions: log.LogOptions{
@@ -63,21 +64,26 @@ This is destructive and irreversible. Requires --force.`,
lc.Append(fx.Hook{
OnStart: func(ctx context.Context) error {
go func() {
if err := v.NukeRemote(true); err != nil {
if err != context.Canceled {
err := v.NukeRemote(true)
if err != nil {
if !errors.Is(err, context.Canceled) {
log.Error("Remote nuke failed", "error", err)
ReportError("Remote nuke failed: %v", err)
ReportErrorf("Remote nuke failed: %v", err)
os.Exit(1)
}
}
if err := v.Shutdowner.Shutdown(); err != nil {
err = v.Shutdowner.Shutdown()
if err != nil {
log.Error("Failed to shutdown", "error", err)
}
}()
return nil
},
OnStop: func(ctx context.Context) error {
v.Cancel()
return nil
},
})
@@ -113,6 +119,7 @@ func newRemoteInfoCommand() *cobra.Command {
}
rootFlags := GetRootFlags()
return RunWithApp(cmd.Context(), AppOptions{
ConfigPath: configPath,
LogOptions: log.LogOptions{
@@ -126,23 +133,29 @@ func newRemoteInfoCommand() *cobra.Command {
lc.Append(fx.Hook{
OnStart: func(ctx context.Context) error {
go func() {
if err := v.RemoteInfo(jsonOutput); err != nil {
if err != context.Canceled {
err := v.RemoteInfo(jsonOutput)
if err != nil {
if !errors.Is(err, context.Canceled) {
if !jsonOutput {
log.Error("Failed to get remote info", "error", err)
ReportError("Failed to get remote info: %v", err)
ReportErrorf("Failed to get remote info: %v", err)
}
os.Exit(1)
}
}
if err := v.Shutdowner.Shutdown(); err != nil {
err = v.Shutdowner.Shutdown()
if err != nil {
log.Error("Failed to shutdown", "error", err)
}
}()
return nil
},
OnStop: func(ctx context.Context) error {
v.Cancel()
return nil
},
})

3
internal/cli/root.go
View File

@@ -78,6 +78,7 @@ func ResolveConfigPath() (string, error) {
if _, err := os.Stat(path); err != nil {
return "", fmt.Errorf("config file from --config not found: %s (run 'vaultik config init --config %s' to create it)", path, path)
}
return path, nil
}
@@ -85,6 +86,7 @@ func ResolveConfigPath() (string, error) {
if _, err := os.Stat(path); err != nil {
return "", fmt.Errorf("config file from $VAULTIK_CONFIG not found: %s (unset VAULTIK_CONFIG, point it at an existing file, or run 'vaultik config init')", path)
}
return path, nil
}
@@ -114,5 +116,6 @@ func DefaultConfigPath() string {
if os.Getuid() == 0 {
return "/etc/vaultik/config.yml"
}
return filepath.Join(xdg.ConfigHome, "vaultik", "config.yml")
}

81
internal/cli/snapshot.go
View File

@@ -2,6 +2,7 @@ package cli
import (
"context"
"errors"
"fmt"
"os"
@@ -58,6 +59,7 @@ specifying a path using --config or by setting VAULTIK_CONFIG to a path.`,
// Use the backup functionality from cli package
rootFlags := GetRootFlags()
return RunWithApp(cmd.Context(), AppOptions{
ConfigPath: configPath,
LogOptions: log.LogOptions{
@@ -74,25 +76,29 @@ specifying a path using --config or by setting VAULTIK_CONFIG to a path.`,
// Start the snapshot creation in a goroutine
go func() {
// --cron suppression is wired through v.UI by setupGlobals.
if err := v.CreateSnapshot(opts); err != nil {
if err != context.Canceled {
err := v.CreateSnapshot(opts)
if err != nil {
if !errors.Is(err, context.Canceled) {
log.Error("Snapshot creation failed", "error", err)
ReportError("Snapshot creation failed: %v", err)
ReportErrorf("Snapshot creation failed: %v", err)
os.Exit(1)
}
}
// Shutdown the app when snapshot completes
if err := v.Shutdowner.Shutdown(); err != nil {
err = v.Shutdowner.Shutdown()
if err != nil {
log.Error("Failed to shutdown", "error", err)
}
}()
return nil
},
OnStop: func(ctx context.Context) error {
log.Debug("Stopping snapshot creation")
// Cancel the Vaultik context
v.Cancel()
return nil
},
})
@@ -127,6 +133,7 @@ func newSnapshotListCommand() *cobra.Command {
}
rootFlags := GetRootFlags()
return RunWithApp(cmd.Context(), AppOptions{
ConfigPath: configPath,
LogOptions: log.LogOptions{
@@ -140,21 +147,26 @@ func newSnapshotListCommand() *cobra.Command {
lc.Append(fx.Hook{
OnStart: func(ctx context.Context) error {
go func() {
if err := v.ListSnapshots(jsonOutput); err != nil {
if err != context.Canceled {
err := v.ListSnapshots(jsonOutput)
if err != nil {
if !errors.Is(err, context.Canceled) {
log.Error("Failed to list snapshots", "error", err)
ReportError("Failed to list snapshots: %v", err)
ReportErrorf("Failed to list snapshots: %v", err)
os.Exit(1)
}
}
if err := v.Shutdowner.Shutdown(); err != nil {
err = v.Shutdowner.Shutdown()
if err != nil {
log.Error("Failed to shutdown", "error", err)
}
}()
return nil
},
OnStop: func(ctx context.Context) error {
v.Cancel()
return nil
},
})
@@ -185,10 +197,11 @@ restrict the operation to specific snapshot names.`,
RunE: func(cmd *cobra.Command, args []string) error {
// Validate flags
if !opts.KeepLatest && opts.OlderThan == "" {
return fmt.Errorf("must specify either --keep-latest or --older-than")
return errors.New("must specify either --keep-latest or --older-than")
}
if opts.KeepLatest && opts.OlderThan != "" {
return fmt.Errorf("cannot specify both --keep-latest and --older-than")
return errors.New("cannot specify both --keep-latest and --older-than")
}
// Use unified config resolution
@@ -198,6 +211,7 @@ restrict the operation to specific snapshot names.`,
}
rootFlags := GetRootFlags()
return RunWithApp(cmd.Context(), AppOptions{
ConfigPath: configPath,
LogOptions: log.LogOptions{
@@ -211,21 +225,26 @@ restrict the operation to specific snapshot names.`,
lc.Append(fx.Hook{
OnStart: func(ctx context.Context) error {
go func() {
if err := v.PurgeSnapshotsWithOptions(opts); err != nil {
if err != context.Canceled {
err := v.PurgeSnapshotsWithOptions(opts)
if err != nil {
if !errors.Is(err, context.Canceled) {
log.Error("Failed to purge snapshots", "error", err)
ReportError("Failed to purge snapshots: %v", err)
ReportErrorf("Failed to purge snapshots: %v", err)
os.Exit(1)
}
}
if err := v.Shutdowner.Shutdown(); err != nil {
err = v.Shutdowner.Shutdown()
if err != nil {
log.Error("Failed to shutdown", "error", err)
}
}()
return nil
},
OnStop: func(ctx context.Context) error {
v.Cancel()
return nil
},
})
@@ -254,11 +273,14 @@ func newSnapshotVerifyCommand() *cobra.Command {
Args: func(cmd *cobra.Command, args []string) error {
if len(args) != 1 {
_ = cmd.Help()
if len(args) == 0 {
return fmt.Errorf("snapshot ID required")
return errors.New("snapshot ID required")
}
return fmt.Errorf("expected 1 argument, got %d", len(args))
}
return nil
},
RunE: func(cmd *cobra.Command, args []string) error {
@@ -271,6 +293,7 @@ func newSnapshotVerifyCommand() *cobra.Command {
}
rootFlags := GetRootFlags()
return RunWithApp(cmd.Context(), AppOptions{
ConfigPath: configPath,
LogOptions: log.LogOptions{
@@ -284,23 +307,29 @@ func newSnapshotVerifyCommand() *cobra.Command {
lc.Append(fx.Hook{
OnStart: func(ctx context.Context) error {
go func() {
if err := v.VerifySnapshotWithOptions(snapshotID, opts); err != nil {
if err != context.Canceled {
err := v.VerifySnapshotWithOptions(snapshotID, opts)
if err != nil {
if !errors.Is(err, context.Canceled) {
if !opts.JSON {
log.Error("Verification failed", "error", err)
ReportError("Verification failed: %v", err)
ReportErrorf("Verification failed: %v", err)
}
os.Exit(1)
}
}
if err := v.Shutdowner.Shutdown(); err != nil {
err = v.Shutdowner.Shutdown()
if err != nil {
log.Error("Failed to shutdown", "error", err)
}
}()
return nil
},
OnStop: func(ctx context.Context) error {
v.Cancel()
return nil
},
})
@@ -345,11 +374,14 @@ nuke --force' — it is the single supported entry point for that.`,
Args: func(cmd *cobra.Command, args []string) error {
if len(args) != 1 {
_ = cmd.Help()
if len(args) == 0 {
return fmt.Errorf("snapshot ID required")
return errors.New("snapshot ID required")
}
return fmt.Errorf("expected 1 argument, got %d", len(args))
}
return nil
},
RunE: func(cmd *cobra.Command, args []string) error {
@@ -360,6 +392,7 @@ nuke --force' — it is the single supported entry point for that.`,
}
rootFlags := GetRootFlags()
return RunWithApp(cmd.Context(), AppOptions{
ConfigPath: configPath,
LogOptions: log.LogOptions{
@@ -375,22 +408,26 @@ nuke --force' — it is the single supported entry point for that.`,
go func() {
_, err := v.RemoveSnapshot(args[0], opts)
if err != nil {
if err != context.Canceled {
if !errors.Is(err, context.Canceled) {
if !opts.JSON {
log.Error("Failed to remove snapshot", "error", err)
ReportError("Failed to remove snapshot: %v", err)
ReportErrorf("Failed to remove snapshot: %v", err)
}
os.Exit(1)
}
}
if err := v.Shutdowner.Shutdown(); err != nil {
log.Error("Failed to shutdown", "error", err)
}
}()
return nil
},
OnStop: func(ctx context.Context) error {
v.Cancel()
return nil
},
})

16
internal/cli/snapshot_restore.go
View File

@@ -2,6 +2,7 @@ package cli
import (
"context"
"errors"
"os"
"github.com/spf13/cobra"
@@ -70,6 +71,7 @@ Examples:
// runRestore parses arguments and runs the restore operation through the app framework
func runRestore(cmd *cobra.Command, args []string, opts *RestoreOptions) error {
snapshotID := args[0]
opts.TargetDir = args[1]
if len(args) > 2 {
opts.Paths = args[2:]
@@ -83,6 +85,7 @@ func runRestore(cmd *cobra.Command, args []string, opts *RestoreOptions) error {
// Use the app framework like other commands
rootFlags := GetRootFlags()
return RunWithApp(cmd.Context(), AppOptions{
ConfigPath: configPath,
LogOptions: log.LogOptions{
@@ -129,24 +132,29 @@ func buildRestoreInvokes(snapshotID string, opts *RestoreOptions) []fx.Option {
Verify: opts.Verify,
SkipErrors: GetRootFlags().SkipErrors,
}
if err := app.Vaultik.Restore(restoreOpts); err != nil {
if err != context.Canceled {
err := app.Vaultik.Restore(restoreOpts)
if err != nil {
if !errors.Is(err, context.Canceled) {
log.Error("Restore operation failed", "error", err)
ReportError("Restore failed: %v", err)
ReportErrorf("Restore failed: %v", err)
os.Exit(1)
}
}
// Shutdown the app when restore completes
if err := app.Shutdowner.Shutdown(); err != nil {
err = app.Shutdowner.Shutdown()
if err != nil {
log.Error("Failed to shutdown", "error", err)
}
}()
return nil
},
OnStop: func(ctx context.Context) error {
log.Debug("Stopping restore operation")
app.Vaultik.Cancel()
return nil
},
})

1
internal/cli/version.go
View File

@@ -24,6 +24,7 @@ func NewVersionCommand() *cobra.Command {
fmt.Printf(" author: %s\n", globals.Author)
fmt.Printf(" homepage: %s\n", globals.Homepage)
fmt.Printf(" license: %s\n", globals.License)
if globals.Version == "dev" {
fmt.Println()
fmt.Println("This is a development build (no version information embedded).")

48
internal/config/config.go
View File

@@ -1,6 +1,7 @@
package config
import (
"errors"
"fmt"
"os"
"path/filepath"
@@ -21,12 +22,16 @@ const appName = "vaultik"
func expandTilde(path string) string {
if path == "~" {
home, _ := os.UserHomeDir()
return home
}
if strings.HasPrefix(path, "~/") {
home, _ := os.UserHomeDir()
return filepath.Join(home, path[2:])
}
return path
}
@@ -34,8 +39,10 @@ func expandTilde(path string) string {
func expandTildeInURL(url string) string {
if strings.HasPrefix(url, "file://~/") {
home, _ := os.UserHomeDir()
return "file://" + filepath.Join(home, url[9:])
}
return url
}
@@ -63,6 +70,7 @@ func (c *Config) GetExcludes(snapshotName string) []string {
combined := make([]string, 0, len(c.Exclude)+len(snap.Exclude))
combined = append(combined, c.Exclude...)
combined = append(combined, snap.Exclude...)
return combined
}
@@ -74,6 +82,7 @@ func (c *Config) SnapshotNames() []string {
}
// Sort for deterministic order
sort.Strings(names)
return names
}
@@ -126,7 +135,7 @@ type ConfigPath string
// Returns an error if the path is empty or if loading fails.
func New(path ConfigPath) (*Config, error) {
if path == "" {
return nil, fmt.Errorf("config path not provided")
return nil, errors.New("config path not provided")
}
cfg, err := Load(string(path))
@@ -159,6 +168,7 @@ func Load(path string) (*Config, error) {
// Convert smartconfig data to YAML then unmarshal
configData := sc.Data()
yamlBytes, err := yaml.Marshal(configData)
if err != nil {
return nil, fmt.Errorf("failed to marshal config data: %w", err)
@@ -177,6 +187,7 @@ func Load(path string) (*Config, error) {
for i, path := range snap.Paths {
snap.Paths[i] = expandTilde(path)
}
cfg.Snapshots[name] = snap
}
@@ -196,6 +207,7 @@ func Load(path string) (*Config, error) {
if err != nil {
return nil, fmt.Errorf("failed to get hostname: %w", err)
}
cfg.Hostname = hostname
}
@@ -203,6 +215,7 @@ func Load(path string) (*Config, error) {
if cfg.S3.Region == "" {
cfg.S3.Region = "us-east-1"
}
if cfg.S3.PartSize == 0 {
cfg.S3.PartSize = Size(5 * 1024 * 1024) // 5MB
}
@@ -236,11 +249,11 @@ func Load(path string) (*Config, error) {
// Returns an error describing the first validation failure encountered.
func (c *Config) Validate() error {
if len(c.AgeRecipients) == 0 {
return fmt.Errorf("at least one age_recipient is required (generate with: age-keygen)")
return errors.New("at least one age_recipient is required (generate with: age-keygen)")
}
if len(c.Snapshots) == 0 {
return fmt.Errorf("at least one snapshot must be configured (see config.example.yml)")
return errors.New("at least one snapshot must be configured (see config.example.yml)")
}
for name, snap := range c.Snapshots {
@@ -250,20 +263,21 @@ func (c *Config) Validate() error {
}
// Validate storage configuration
if err := c.validateStorage(); err != nil {
err := c.validateStorage()
if err != nil {
return err
}
if c.ChunkSize.Int64() < 1024*1024 { // 1MB minimum
return fmt.Errorf("chunk_size must be at least 1MB")
return errors.New("chunk_size must be at least 1MB")
}
if c.BlobSizeLimit.Int64() < c.ChunkSize.Int64() {
return fmt.Errorf("blob_size_limit must be at least chunk_size")
return errors.New("blob_size_limit must be at least chunk_size")
}
if c.CompressionLevel < 1 || c.CompressionLevel > 19 {
return fmt.Errorf("compression_level must be between 1 and 19")
return errors.New("compression_level must be between 1 and 19")
}
return nil
@@ -280,38 +294,43 @@ func (c *Config) validateStorage() error {
// File storage doesn't need S3 credentials
return nil
}
if strings.HasPrefix(c.StorageURL, "s3://") {
// S3 storage needs credentials
if c.S3.AccessKeyID == "" {
return fmt.Errorf("s3.access_key_id is required for s3:// URLs")
return errors.New("s3.access_key_id is required for s3:// URLs")
}
if c.S3.SecretAccessKey == "" {
return fmt.Errorf("s3.secret_access_key is required for s3:// URLs")
return errors.New("s3.secret_access_key is required for s3:// URLs")
}
return nil
}
if strings.HasPrefix(c.StorageURL, "rclone://") {
// Rclone storage uses rclone's own config
return nil
}
return fmt.Errorf("storage_url must start with s3://, file://, or rclone://")
return errors.New("storage_url must start with s3://, file://, or rclone://")
}
// Legacy S3 configuration
if c.S3.Endpoint == "" {
return fmt.Errorf("storage not configured; set storage_url or provide s3.endpoint + s3.bucket + credentials")
return errors.New("storage not configured; set storage_url or provide s3.endpoint + s3.bucket + credentials")
}
if c.S3.Bucket == "" {
return fmt.Errorf("s3.bucket is required (or set storage_url)")
return errors.New("s3.bucket is required (or set storage_url)")
}
if c.S3.AccessKeyID == "" {
return fmt.Errorf("s3.access_key_id is required")
return errors.New("s3.access_key_id is required")
}
if c.S3.SecretAccessKey == "" {
return fmt.Errorf("s3.secret_access_key is required")
return errors.New("s3.secret_access_key is required")
}
return nil
@@ -329,6 +348,7 @@ func extractAgeSecretKey(input string) string {
if id, ok := identities[0].(*age.X25519Identity); ok {
return id.String()
}
return strings.TrimSpace(input)
}

1
internal/config/config_test.go
View File

@@ -41,6 +41,7 @@ func TestConfigLoad(t *testing.T) {
if len(cfg.AgeRecipients) != 2 {
t.Errorf("Expected 2 age recipients, got %d", len(cfg.AgeRecipients))
}
if cfg.AgeRecipients[0] != TEST_SNEAK_AGE_PUBLIC_KEY {
t.Errorf("Expected first age recipient to be %s, got '%s'", TEST_SNEAK_AGE_PUBLIC_KEY, cfg.AgeRecipients[0])
}

8
internal/config/size.go
View File

@@ -1,6 +1,7 @@
package config
import (
"errors"
"fmt"
"github.com/dustin/go-humanize"
@@ -14,18 +15,19 @@ type Size int64
// UnmarshalYAML implements yaml.Unmarshaler for Size, allowing it to be
// parsed from YAML configuration files. It accepts both numeric values
// (interpreted as bytes) and string values with units (e.g., "10MB").
func (s *Size) UnmarshalYAML(unmarshal func(interface{}) error) error {
func (s *Size) UnmarshalYAML(unmarshal func(any) error) error {
// Try to unmarshal as int64 first
var intVal int64
if err := unmarshal(&intVal); err == nil {
*s = Size(intVal)
return nil
}
// Try to unmarshal as string
var strVal string
if err := unmarshal(&strVal); err != nil {
return fmt.Errorf("size must be a number or string")
return errors.New("size must be a number or string")
}
// Parse the string using go-humanize
@@ -35,6 +37,7 @@ func (s *Size) UnmarshalYAML(unmarshal func(interface{}) error) error {
}
*s = Size(bytes)
return nil
}
@@ -58,5 +61,6 @@ func ParseSize(s string) (Size, error) {
if err != nil {
return 0, fmt.Errorf("invalid size format: %w", err)
}
return Size(bytes), nil
}

7
internal/crypto/encryption.go
View File

@@ -2,6 +2,7 @@ package crypto
import (
"bytes"
"errors"
"fmt"
"io"
"sync"
@@ -25,7 +26,7 @@ type Encryptor struct {
// public keys are invalid or if no recipients are specified.
func NewEncryptor(publicKeys []string) (*Encryptor, error) {
if len(publicKeys) == 0 {
return nil, fmt.Errorf("at least one recipient is required")
return nil, errors.New("at least one recipient is required")
}
recipients := make([]age.Recipient, 0, len(publicKeys))
@@ -34,6 +35,7 @@ func NewEncryptor(publicKeys []string) (*Encryptor, error) {
if err != nil {
return nil, fmt.Errorf("parsing age recipient %s: %w", key, err)
}
recipients = append(recipients, recipient)
}
@@ -126,7 +128,7 @@ func (e *Encryptor) EncryptWriter(dst io.Writer) (io.WriteCloser, error) {
// of the public keys are invalid or if no recipients are specified.
func (e *Encryptor) UpdateRecipients(publicKeys []string) error {
if len(publicKeys) == 0 {
return fmt.Errorf("at least one recipient is required")
return errors.New("at least one recipient is required")
}
recipients := make([]age.Recipient, 0, len(publicKeys))
@@ -135,6 +137,7 @@ func (e *Encryptor) UpdateRecipients(publicKeys []string) error {
if err != nil {
return fmt.Errorf("parsing age recipient %s: %w", key, err)
}
recipients = append(recipients, recipient)
}

5
internal/crypto/encryption_test.go
View File

@@ -58,10 +58,12 @@ func TestEncryptorMultipleRecipients(t *testing.T) {
if err != nil {
t.Fatalf("failed to generate identity1: %v", err)
}
identity2, err := age.GenerateX25519Identity()
if err != nil {
t.Fatalf("failed to generate identity2: %v", err)
}
identity3, err := age.GenerateX25519Identity()
if err != nil {
t.Fatalf("failed to generate identity3: %v", err)
@@ -123,6 +125,7 @@ func TestEncryptorUpdateRecipients(t *testing.T) {
// Encrypt with first key
plaintext := []byte("test data")
ciphertext1, err := enc.Encrypt(plaintext)
if err != nil {
t.Fatalf("failed to encrypt: %v", err)
@@ -143,6 +146,7 @@ func TestEncryptorUpdateRecipients(t *testing.T) {
if _, err := age.Decrypt(bytes.NewReader(ciphertext1), identity1); err != nil {
t.Error("failed to decrypt with identity1")
}
if _, err := age.Decrypt(bytes.NewReader(ciphertext1), identity2); err == nil {
t.Error("should not decrypt with identity2")
}
@@ -151,6 +155,7 @@ func TestEncryptorUpdateRecipients(t *testing.T) {
if _, err := age.Decrypt(bytes.NewReader(ciphertext2), identity2); err != nil {
t.Error("failed to decrypt with identity2")
}
if _, err := age.Decrypt(bytes.NewReader(ciphertext2), identity1); err == nil {
t.Error("should not decrypt with identity1")
}

15
internal/database/blob_chunks.go
View File

@@ -49,12 +49,15 @@ func (r *BlobChunkRepository) GetByBlobID(ctx context.Context, blobID string) ([
defer CloseRows(rows)
var blobChunks []*BlobChunk
for rows.Next() {
var bc BlobChunk
err := rows.Scan(&bc.BlobID, &bc.ChunkHash, &bc.Offset, &bc.Length)
if err != nil {
return nil, fmt.Errorf("scanning blob chunk: %w", err)
}
blobChunks = append(blobChunks, &bc)
}
@@ -70,7 +73,9 @@ func (r *BlobChunkRepository) GetByChunkHash(ctx context.Context, chunkHash stri
`
LogSQL("GetByChunkHash", query, chunkHash)
var bc BlobChunk
err := r.db.conn.QueryRowContext(ctx, query, chunkHash).Scan(
&bc.BlobID,
&bc.ChunkHash,
@@ -80,14 +85,18 @@ func (r *BlobChunkRepository) GetByChunkHash(ctx context.Context, chunkHash stri
if err == sql.ErrNoRows {
LogSQL("GetByChunkHash", "No rows found", chunkHash)
return nil, nil
}
if err != nil {
LogSQL("GetByChunkHash", "Error", chunkHash, err)
return nil, fmt.Errorf("querying blob chunk: %w", err)
}
LogSQL("GetByChunkHash", "Found blob", chunkHash, "blob", bc.BlobID)
return &bc, nil
}
@@ -101,7 +110,9 @@ func (r *BlobChunkRepository) GetByChunkHashTx(ctx context.Context, tx *sql.Tx,
`
LogSQL("GetByChunkHashTx", query, chunkHash)
var bc BlobChunk
err := tx.QueryRowContext(ctx, query, chunkHash).Scan(
&bc.BlobID,
&bc.ChunkHash,
@@ -111,14 +122,18 @@ func (r *BlobChunkRepository) GetByChunkHashTx(ctx context.Context, tx *sql.Tx,
if err == sql.ErrNoRows {
LogSQL("GetByChunkHashTx", "No rows found", chunkHash)
return nil, nil
}
if err != nil {
LogSQL("GetByChunkHashTx", "Error", chunkHash, err)
return nil, fmt.Errorf("querying blob chunk: %w", err)
}
LogSQL("GetByChunkHashTx", "Found blob", chunkHash, "blob", bc.BlobID)
return &bc, nil
}

15
internal/database/blob_chunks_test.go
View File

@@ -22,6 +22,7 @@ func TestBlobChunkRepository(t *testing.T) {
Hash: types.BlobHash("blob1-hash"),
CreatedTS: time.Now(),
}
err := repos.Blobs.Create(ctx, nil, blob)
if err != nil {
t.Fatalf("failed to create blob: %v", err)
@@ -34,6 +35,7 @@ func TestBlobChunkRepository(t *testing.T) {
ChunkHash: chunkHash,
Size: 1024,
}
err = repos.Chunks.Create(ctx, nil, chunk)
if err != nil {
t.Fatalf("failed to create chunk %s: %v", chunkHash, err)
@@ -60,6 +62,7 @@ func TestBlobChunkRepository(t *testing.T) {
Offset: 1024,
Length: 2048,
}
err = repos.BlobChunks.Create(ctx, nil, bc2)
if err != nil {
t.Fatalf("failed to create second blob chunk: %v", err)
@@ -71,6 +74,7 @@ func TestBlobChunkRepository(t *testing.T) {
Offset: 3072,
Length: 512,
}
err = repos.BlobChunks.Create(ctx, nil, bc3)
if err != nil {
t.Fatalf("failed to create third blob chunk: %v", err)
@@ -81,6 +85,7 @@ func TestBlobChunkRepository(t *testing.T) {
if err != nil {
t.Fatalf("failed to get blob chunks: %v", err)
}
if len(blobChunks) != 3 {
t.Errorf("expected 3 chunks, got %d", len(blobChunks))
}
@@ -98,12 +103,15 @@ func TestBlobChunkRepository(t *testing.T) {
if err != nil {
t.Fatalf("failed to get blob chunk by chunk hash: %v", err)
}
if bc == nil {
t.Fatal("expected blob chunk, got nil")
}
if bc.BlobID != blob.ID {
t.Errorf("wrong blob ID: expected %s, got %s", blob.ID, bc.BlobID)
}
if bc.Offset != 1024 {
t.Errorf("wrong offset: expected 1024, got %d", bc.Offset)
}
@@ -113,6 +121,7 @@ func TestBlobChunkRepository(t *testing.T) {
if err == nil {
t.Fatal("duplicate blob_chunk insert should fail due to primary key constraint")
}
if !strings.Contains(err.Error(), "UNIQUE") && !strings.Contains(err.Error(), "constraint") {
t.Fatalf("expected constraint error, got: %v", err)
}
@@ -122,6 +131,7 @@ func TestBlobChunkRepository(t *testing.T) {
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if bc != nil {
t.Error("expected nil for non-existent chunk")
}
@@ -150,6 +160,7 @@ func TestBlobChunkRepositoryMultipleBlobs(t *testing.T) {
if err != nil {
t.Fatalf("failed to create blob1: %v", err)
}
err = repos.Blobs.Create(ctx, nil, blob2)
if err != nil {
t.Fatalf("failed to create blob2: %v", err)
@@ -162,6 +173,7 @@ func TestBlobChunkRepositoryMultipleBlobs(t *testing.T) {
ChunkHash: chunkHash,
Size: 1024,
}
err = repos.Chunks.Create(ctx, nil, chunk)
if err != nil {
t.Fatalf("failed to create chunk %s: %v", chunkHash, err)
@@ -189,6 +201,7 @@ func TestBlobChunkRepositoryMultipleBlobs(t *testing.T) {
if err != nil {
t.Fatalf("failed to get blob1 chunks: %v", err)
}
if len(chunks) != 2 {
t.Errorf("expected 2 chunks for blob1, got %d", len(chunks))
}
@@ -198,6 +211,7 @@ func TestBlobChunkRepositoryMultipleBlobs(t *testing.T) {
if err != nil {
t.Fatalf("failed to get blob2 chunks: %v", err)
}
if len(chunks) != 2 {
t.Errorf("expected 2 chunks for blob2, got %d", len(chunks))
}
@@ -207,6 +221,7 @@ func TestBlobChunkRepositoryMultipleBlobs(t *testing.T) {
if err != nil {
t.Fatalf("failed to get shared chunk: %v", err)
}
if bc == nil {
t.Fatal("expected shared chunk, got nil")
}

45
internal/database/blobs.go
View File

@@ -24,10 +24,12 @@ func (r *BlobRepository) Create(ctx context.Context, tx *sql.Tx, blob *Blob) err
`
var finishedTS, uploadedTS *int64
if blob.FinishedTS != nil {
ts := blob.FinishedTS.Unix()
finishedTS = &ts
}
if blob.UploadedTS != nil {
ts := blob.UploadedTS.Unix()
uploadedTS = &ts
@@ -56,9 +58,11 @@ func (r *BlobRepository) GetByHash(ctx context.Context, hash string) (*Blob, err
WHERE blob_hash = ?
`
var blob Blob
var createdTSUnix int64
var finishedTSUnix, uploadedTSUnix sql.NullInt64
var (
blob Blob
createdTSUnix int64
finishedTSUnix, uploadedTSUnix sql.NullInt64
)
err := r.db.conn.QueryRowContext(ctx, query, hash).Scan(
&blob.ID,
@@ -73,6 +77,7 @@ func (r *BlobRepository) GetByHash(ctx context.Context, hash string) (*Blob, err
if err == sql.ErrNoRows {
return nil, nil
}
if err != nil {
return nil, fmt.Errorf("querying blob: %w", err)
}
@@ -82,10 +87,12 @@ func (r *BlobRepository) GetByHash(ctx context.Context, hash string) (*Blob, err
ts := time.Unix(finishedTSUnix.Int64, 0).UTC()
blob.FinishedTS = &ts
}
if uploadedTSUnix.Valid {
ts := time.Unix(uploadedTSUnix.Int64, 0).UTC()
blob.UploadedTS = &ts
}
return &blob, nil
}
@@ -97,9 +104,11 @@ func (r *BlobRepository) GetByID(ctx context.Context, id string) (*Blob, error)
WHERE id = ?
`
var blob Blob
var createdTSUnix int64
var finishedTSUnix, uploadedTSUnix sql.NullInt64
var (
blob Blob
createdTSUnix int64
finishedTSUnix, uploadedTSUnix sql.NullInt64
)
err := r.db.conn.QueryRowContext(ctx, query, id).Scan(
&blob.ID,
@@ -114,6 +123,7 @@ func (r *BlobRepository) GetByID(ctx context.Context, id string) (*Blob, error)
if err == sql.ErrNoRows {
return nil, nil
}
if err != nil {
return nil, fmt.Errorf("querying blob: %w", err)
}
@@ -123,10 +133,12 @@ func (r *BlobRepository) GetByID(ctx context.Context, id string) (*Blob, error)
ts := time.Unix(finishedTSUnix.Int64, 0).UTC()
blob.FinishedTS = &ts
}
if uploadedTSUnix.Valid {
ts := time.Unix(uploadedTSUnix.Int64, 0).UTC()
blob.UploadedTS = &ts
}
return &blob, nil
}
@@ -146,11 +158,15 @@ func (r *BlobRepository) GetAll(ctx context.Context) (map[string]*Blob, error) {
defer CloseRows(rows)
out := make(map[string]*Blob)
for rows.Next() {
var blob Blob
var createdTSUnix int64
var finishedTSUnix, uploadedTSUnix sql.NullInt64
if err := rows.Scan(
var (
blob Blob
createdTSUnix int64
finishedTSUnix, uploadedTSUnix sql.NullInt64
)
err := rows.Scan(
&blob.ID,
&blob.Hash,
&createdTSUnix,
@@ -158,20 +174,25 @@ func (r *BlobRepository) GetAll(ctx context.Context) (map[string]*Blob, error) {
&blob.UncompressedSize,
&blob.CompressedSize,
&uploadedTSUnix,
); err != nil {
)
if err != nil {
return nil, fmt.Errorf("scanning blob: %w", err)
}
blob.CreatedTS = time.Unix(createdTSUnix, 0).UTC()
if finishedTSUnix.Valid {
ts := time.Unix(finishedTSUnix.Int64, 0).UTC()
blob.FinishedTS = &ts
}
if uploadedTSUnix.Valid {
ts := time.Unix(uploadedTSUnix.Int64, 0).UTC()
blob.UploadedTS = &ts
}
out[blob.ID.String()] = &blob
}
return out, rows.Err()
}
@@ -184,6 +205,7 @@ func (r *BlobRepository) UpdateFinished(ctx context.Context, tx *sql.Tx, id stri
`
now := time.Now().UTC().Unix()
var err error
if tx != nil {
_, err = tx.ExecContext(ctx, query, hash, now, uncompressedSize, compressedSize, id)
@@ -207,6 +229,7 @@ func (r *BlobRepository) UpdateUploaded(ctx context.Context, tx *sql.Tx, id stri
`
now := time.Now().UTC().Unix()
var err error
if tx != nil {
_, err = tx.ExecContext(ctx, query, now, id)

11
internal/database/blobs_test.go
View File

@@ -32,12 +32,15 @@ func TestBlobRepository(t *testing.T) {
if err != nil {
t.Fatalf("failed to get blob: %v", err)
}
if retrieved == nil {
t.Fatal("expected blob, got nil")
}
if retrieved.Hash != blob.Hash {
t.Errorf("blob hash mismatch: got %s, want %s", retrieved.Hash, blob.Hash)
}
if !retrieved.CreatedTS.Equal(blob.CreatedTS) {
t.Errorf("created timestamp mismatch: got %v, want %v", retrieved.CreatedTS, blob.CreatedTS)
}
@@ -47,9 +50,11 @@ func TestBlobRepository(t *testing.T) {
if err != nil {
t.Fatalf("failed to get blob by ID: %v", err)
}
if retrievedByID == nil {
t.Fatal("expected blob, got nil")
}
if retrievedByID.ID != blob.ID {
t.Errorf("blob ID mismatch: got %s, want %s", retrievedByID.ID, blob.ID)
}
@@ -60,6 +65,7 @@ func TestBlobRepository(t *testing.T) {
Hash: types.BlobHash("blobhash456"),
CreatedTS: time.Now().Truncate(time.Second),
}
err = repo.Create(ctx, nil, blob2)
if err != nil {
t.Fatalf("failed to create second blob: %v", err)
@@ -67,6 +73,7 @@ func TestBlobRepository(t *testing.T) {
// Test UpdateFinished
now := time.Now()
err = repo.UpdateFinished(ctx, nil, blob.ID.String(), blob.Hash.String(), 1000, 500)
if err != nil {
t.Fatalf("failed to update blob as finished: %v", err)
@@ -77,12 +84,15 @@ func TestBlobRepository(t *testing.T) {
if err != nil {
t.Fatalf("failed to get updated blob: %v", err)
}
if updated.FinishedTS == nil {
t.Fatal("expected finished timestamp to be set")
}
if updated.UncompressedSize != 1000 {
t.Errorf("expected uncompressed size 1000, got %d", updated.UncompressedSize)
}
if updated.CompressedSize != 500 {
t.Errorf("expected compressed size 500, got %d", updated.CompressedSize)
}
@@ -98,6 +108,7 @@ func TestBlobRepository(t *testing.T) {
if err != nil {
t.Fatalf("failed to get uploaded blob: %v", err)
}
if uploaded.UploadedTS == nil {
t.Fatal("expected uploaded timestamp to be set")
}

17
internal/database/cascade_debug_test.go
View File

@@ -19,10 +19,12 @@ func TestCascadeDeleteDebug(t *testing.T) {
// Check if foreign keys are enabled
var fkEnabled int
err := db.conn.QueryRow("PRAGMA foreign_keys").Scan(&fkEnabled)
if err != nil {
t.Fatal(err)
}
t.Logf("Foreign keys enabled: %d", fkEnabled)
// Create a file
@@ -34,18 +36,21 @@ func TestCascadeDeleteDebug(t *testing.T) {
UID: 1000,
GID: 1000,
}
err = repos.Files.Create(ctx, nil, file)
if err != nil {
t.Fatalf("failed to create file: %v", err)
}
t.Logf("Created file with ID: %s", file.ID)
// Create chunks and file-chunk mappings
for i := 0; i < 3; i++ {
for i := range 3 {
chunk := &Chunk{
ChunkHash: types.ChunkHash(fmt.Sprintf("cascade-chunk-%d", i)),
Size: 1024,
}
err = repos.Chunks.Create(ctx, nil, chunk)
if err != nil {
t.Fatalf("failed to create chunk: %v", err)
@@ -56,10 +61,12 @@ func TestCascadeDeleteDebug(t *testing.T) {
Idx: i,
ChunkHash: chunk.ChunkHash,
}
err = repos.FileChunks.Create(ctx, nil, fc)
if err != nil {
t.Fatalf("failed to create file chunk: %v", err)
}
t.Logf("Created file chunk mapping: file_id=%s, idx=%d, chunk=%s", fc.FileID, fc.Idx, fc.ChunkHash)
}
@@ -68,10 +75,12 @@ func TestCascadeDeleteDebug(t *testing.T) {
if err != nil {
t.Fatal(err)
}
t.Logf("File chunks before delete: %d", len(fileChunks))
// Check the foreign key constraint
var fkInfo string
err = db.conn.QueryRow(`
SELECT sql FROM sqlite_master
WHERE type='table' AND name='file_chunks'
@@ -79,10 +88,12 @@ func TestCascadeDeleteDebug(t *testing.T) {
if err != nil {
t.Fatal(err)
}
t.Logf("file_chunks table definition:\n%s", fkInfo)
// Delete the file
t.Log("Deleting file...")
err = repos.Files.DeleteByID(ctx, nil, file.ID)
if err != nil {
t.Fatalf("failed to delete file: %v", err)
@@ -93,6 +104,7 @@ func TestCascadeDeleteDebug(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if deletedFile != nil {
t.Error("file should have been deleted")
} else {
@@ -104,14 +116,17 @@ func TestCascadeDeleteDebug(t *testing.T) {
if err != nil {
t.Fatal(err)
}
t.Logf("File chunks after delete: %d", len(fileChunks))
// Manually check the database
var count int
err = db.conn.QueryRow("SELECT COUNT(*) FROM file_chunks WHERE file_id = ?", file.ID).Scan(&count)
if err != nil {
t.Fatal(err)
}
t.Logf("Manual count of file_chunks for deleted file: %d", count)
if len(fileChunks) != 0 {

44
internal/database/chunk_files.go
View File

@@ -4,6 +4,7 @@ import (
"context"
"database/sql"
"fmt"
"strings"
"sneak.berlin/go/vaultik/internal/types"
)
@@ -90,18 +91,25 @@ func (r *ChunkFileRepository) GetByFileID(ctx context.Context, fileID types.File
// scanChunkFiles is a helper that scans chunk file rows
func (r *ChunkFileRepository) scanChunkFiles(rows *sql.Rows) ([]*ChunkFile, error) {
var chunkFiles []*ChunkFile
for rows.Next() {
var cf ChunkFile
var chunkHashStr, fileIDStr string
var (
cf ChunkFile
chunkHashStr, fileIDStr string
)
err := rows.Scan(&chunkHashStr, &fileIDStr, &cf.FileOffset, &cf.Length)
if err != nil {
return nil, fmt.Errorf("scanning chunk file: %w", err)
}
cf.ChunkHash = types.ChunkHash(chunkHashStr)
cf.FileID, err = types.ParseFileID(fileIDStr)
if err != nil {
return nil, fmt.Errorf("parsing file ID: %w", err)
}
chunkFiles = append(chunkFiles, &cf)
}
@@ -136,14 +144,13 @@ func (r *ChunkFileRepository) DeleteByFileIDs(ctx context.Context, tx *sql.Tx, f
const batchSize = 500
for i := 0; i < len(fileIDs); i += batchSize {
end := i + batchSize
if end > len(fileIDs) {
end = len(fileIDs)
}
end := min(i+batchSize, len(fileIDs))
batch := fileIDs[i:end]
query := "DELETE FROM chunk_files WHERE file_id IN (?" + repeatPlaceholder(len(batch)-1) + ")"
args := make([]interface{}, len(batch))
args := make([]any, len(batch))
for j, id := range batch {
args[j] = id.String()
}
@@ -154,6 +161,7 @@ func (r *ChunkFileRepository) DeleteByFileIDs(ctx context.Context, tx *sql.Tx, f
} else {
_, err = r.db.ExecWithLog(ctx, query, args...)
}
if err != nil {
return fmt.Errorf("batch deleting chunk_files: %w", err)
}
@@ -172,21 +180,28 @@ func (r *ChunkFileRepository) CreateBatch(ctx context.Context, tx *sql.Tx, cfs [
const batchSize = 200
for i := 0; i < len(cfs); i += batchSize {
end := i + batchSize
if end > len(cfs) {
end = len(cfs)
}
end := min(i+batchSize, len(cfs))
batch := cfs[i:end]
query := "INSERT INTO chunk_files (chunk_hash, file_id, file_offset, length) VALUES "
args := make([]interface{}, 0, len(batch)*4)
args := make([]any, 0, len(batch)*4)
var querySb183 strings.Builder
for j, cf := range batch {
if j > 0 {
query += ", "
querySb183.WriteString(", ")
}
query += "(?, ?, ?, ?)"
querySb183.WriteString("(?, ?, ?, ?)")
args = append(args, cf.ChunkHash.String(), cf.FileID.String(), cf.FileOffset, cf.Length)
}
query += querySb183.String()
query += " ON CONFLICT(chunk_hash, file_id) DO NOTHING"
var err error
@@ -195,6 +210,7 @@ func (r *ChunkFileRepository) CreateBatch(ctx context.Context, tx *sql.Tx, cfs [
} else {
_, err = r.db.ExecWithLog(ctx, query, args...)
}
if err != nil {
return fmt.Errorf("batch inserting chunk_files: %w", err)
}

16
internal/database/chunk_files_test.go
View File

@@ -28,6 +28,7 @@ func TestChunkFileRepository(t *testing.T) {
GID: 1000,
LinkTarget: "",
}
err := fileRepo.Create(ctx, nil, file1)
if err != nil {
t.Fatalf("failed to create file1: %v", err)
@@ -42,6 +43,7 @@ func TestChunkFileRepository(t *testing.T) {
GID: 1000,
LinkTarget: "",
}
err = fileRepo.Create(ctx, nil, file2)
if err != nil {
t.Fatalf("failed to create file2: %v", err)
@@ -52,6 +54,7 @@ func TestChunkFileRepository(t *testing.T) {
ChunkHash: types.ChunkHash("chunk1"),
Size: 1024,
}
err = chunksRepo.Create(ctx, nil, chunk)
if err != nil {
t.Fatalf("failed to create chunk: %v", err)
@@ -77,6 +80,7 @@ func TestChunkFileRepository(t *testing.T) {
FileOffset: 2048,
Length: 1024,
}
err = repo.Create(ctx, nil, cf2)
if err != nil {
t.Fatalf("failed to create second chunk file: %v", err)
@@ -87,6 +91,7 @@ func TestChunkFileRepository(t *testing.T) {
if err != nil {
t.Fatalf("failed to get chunk files: %v", err)
}
if len(chunkFiles) != 2 {
t.Errorf("expected 2 files for chunk, got %d", len(chunkFiles))
}
@@ -94,14 +99,17 @@ func TestChunkFileRepository(t *testing.T) {
// Verify both files are returned
foundFile1 := false
foundFile2 := false
for _, cf := range chunkFiles {
if cf.FileID == file1.ID && cf.FileOffset == 0 {
foundFile1 = true
}
if cf.FileID == file2.ID && cf.FileOffset == 2048 {
foundFile2 = true
}
}
if !foundFile1 || !foundFile2 {
t.Error("not all expected files found")
}
@@ -111,9 +119,11 @@ func TestChunkFileRepository(t *testing.T) {
if err != nil {
t.Fatalf("failed to get chunks by file ID: %v", err)
}
if len(chunkFiles) != 1 {
t.Errorf("expected 1 chunk for file, got %d", len(chunkFiles))
}
if chunkFiles[0].ChunkHash != types.ChunkHash("chunk1") {
t.Errorf("wrong chunk hash: expected chunk1, got %s", chunkFiles[0].ChunkHash)
}
@@ -143,9 +153,11 @@ func TestChunkFileRepositoryComplexDeduplication(t *testing.T) {
if err := fileRepo.Create(ctx, nil, file1); err != nil {
t.Fatalf("failed to create file1: %v", err)
}
if err := fileRepo.Create(ctx, nil, file2); err != nil {
t.Fatalf("failed to create file2: %v", err)
}
if err := fileRepo.Create(ctx, nil, file3); err != nil {
t.Fatalf("failed to create file3: %v", err)
}
@@ -157,6 +169,7 @@ func TestChunkFileRepositoryComplexDeduplication(t *testing.T) {
ChunkHash: chunkHash,
Size: 1024,
}
err := chunksRepo.Create(ctx, nil, chunk)
if err != nil {
t.Fatalf("failed to create chunk %s: %v", chunkHash, err)
@@ -194,6 +207,7 @@ func TestChunkFileRepositoryComplexDeduplication(t *testing.T) {
if err != nil {
t.Fatalf("failed to get files for chunk1: %v", err)
}
if len(files) != 2 {
t.Errorf("expected 2 files for chunk1, got %d", len(files))
}
@@ -203,6 +217,7 @@ func TestChunkFileRepositoryComplexDeduplication(t *testing.T) {
if err != nil {
t.Fatalf("failed to get files for chunk2: %v", err)
}
if len(files) != 2 {
t.Errorf("expected 2 files for chunk2, got %d", len(files))
}
@@ -212,6 +227,7 @@ func TestChunkFileRepositoryComplexDeduplication(t *testing.T) {
if err != nil {
t.Fatalf("failed to get chunks for file2: %v", err)
}
if len(file2Chunks) != 3 {
t.Errorf("expected 3 chunks for file2, got %d", len(file2Chunks))
}

18
internal/database/chunks.go
View File

@@ -4,6 +4,7 @@ import (
"context"
"database/sql"
"fmt"
"strings"
"sneak.berlin/go/vaultik/internal/log"
)
@@ -54,6 +55,7 @@ func (r *ChunkRepository) GetByHash(ctx context.Context, hash string) (*Chunk, e
if err == sql.ErrNoRows {
return nil, nil
}
if err != nil {
return nil, fmt.Errorf("querying chunk: %w", err)
}
@@ -71,14 +73,22 @@ func (r *ChunkRepository) GetByHashes(ctx context.Context, hashes []string) ([]*
FROM chunks
WHERE chunk_hash IN (`
args := make([]interface{}, len(hashes))
args := make([]any, len(hashes))
var querySb75 strings.Builder
for i, hash := range hashes {
if i > 0 {
query += ", "
querySb75.WriteString(", ")
}
query += "?"
querySb75.WriteString("?")
args[i] = hash
}
query += querySb75.String()
query += ") ORDER BY chunk_hash"
rows, err := r.db.conn.QueryContext(ctx, query, args...)
@@ -88,6 +98,7 @@ func (r *ChunkRepository) GetByHashes(ctx context.Context, hashes []string) ([]*
defer CloseRows(rows)
var chunks []*Chunk
for rows.Next() {
var chunk Chunk
@@ -122,6 +133,7 @@ func (r *ChunkRepository) ListUnpacked(ctx context.Context, limit int) ([]*Chunk
defer CloseRows(rows)
var chunks []*Chunk
for rows.Next() {
var chunk Chunk

1
internal/database/chunks_ext.go
View File

@@ -19,6 +19,7 @@ func (r *ChunkRepository) List(ctx context.Context) ([]*Chunk, error) {
defer CloseRows(rows)
var chunks []*Chunk
for rows.Next() {
var chunk Chunk

8
internal/database/chunks_test.go
View File

@@ -30,12 +30,15 @@ func TestChunkRepository(t *testing.T) {
if err != nil {
t.Fatalf("failed to get chunk: %v", err)
}
if retrieved == nil {
t.Fatal("expected chunk, got nil")
}
if retrieved.ChunkHash != chunk.ChunkHash {
t.Errorf("chunk hash mismatch: got %s, want %s", retrieved.ChunkHash, chunk.ChunkHash)
}
if retrieved.Size != chunk.Size {
t.Errorf("size mismatch: got %d, want %d", retrieved.Size, chunk.Size)
}
@@ -51,6 +54,7 @@ func TestChunkRepository(t *testing.T) {
ChunkHash: types.ChunkHash("chunkhash456"),
Size: 8192,
}
err = repo.Create(ctx, nil, chunk2)
if err != nil {
t.Fatalf("failed to create second chunk: %v", err)
@@ -60,6 +64,7 @@ func TestChunkRepository(t *testing.T) {
if err != nil {
t.Fatalf("failed to get chunks by hashes: %v", err)
}
if len(chunks) != 2 {
t.Errorf("expected 2 chunks, got %d", len(chunks))
}
@@ -69,6 +74,7 @@ func TestChunkRepository(t *testing.T) {
if err != nil {
t.Fatalf("failed to list unpacked chunks: %v", err)
}
if len(unpacked) != 2 {
t.Errorf("expected 2 unpacked chunks, got %d", len(unpacked))
}
@@ -86,6 +92,7 @@ func TestChunkRepositoryNotFound(t *testing.T) {
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if chunk != nil {
t.Error("expected nil for non-existent chunk")
}
@@ -95,6 +102,7 @@ func TestChunkRepositoryNotFound(t *testing.T) {
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if chunks != nil {
t.Error("expected nil for empty hash list")
}

38
internal/database/database.go
View File

@@ -57,8 +57,8 @@ func ParseMigrationVersion(filename string) (int, error) {
// Split on underscore to separate version from description.
// If there's no underscore, the entire stem is the version.
versionStr := name
if idx := strings.IndexByte(name, '_'); idx >= 0 {
versionStr = name[:idx]
if before, _, ok := strings.Cut(name, "_"); ok {
versionStr = before
}
if versionStr == "" {
@@ -98,6 +98,7 @@ func New(ctx context.Context, path string) (*DB, error) {
// First attempt with standard WAL mode
log.Debug("Attempting to open database with WAL mode", "path", path)
conn, err := sql.Open(
"sqlite",
path+"?_journal_mode=WAL&_synchronous=NORMAL&_busy_timeout=10000&_locking_mode=NORMAL&_foreign_keys=ON",
@@ -110,7 +111,8 @@ func New(ctx context.Context, path string) (*DB, error) {
conn.SetMaxOpenConns(1)
conn.SetMaxIdleConns(1)
if err := conn.PingContext(ctx); err == nil {
err := conn.PingContext(ctx)
if err == nil {
// Success on first try
log.Debug("Database opened successfully with WAL mode", "path", path)
@@ -120,13 +122,19 @@ func New(ctx context.Context, path string) (*DB, error) {
}
db := &DB{conn: conn, path: path}
if err := applyMigrations(ctx, conn); err != nil {
err := applyMigrations(ctx, conn)
if err != nil {
_ = conn.Close()
return nil, fmt.Errorf("applying migrations: %w", err)
}
return db, nil
}
log.Debug("Failed to ping database, closing connection", "path", path, "error", err)
_ = conn.Close()
}
@@ -135,6 +143,7 @@ func New(ctx context.Context, path string) (*DB, error) {
"Database appears locked, attempting recovery with TRUNCATE mode",
"path", path,
)
conn, err = sql.Open(
"sqlite",
path+"?_journal_mode=TRUNCATE&_synchronous=NORMAL&_busy_timeout=10000&_foreign_keys=ON",
@@ -152,7 +161,9 @@ func New(ctx context.Context, path string) (*DB, error) {
if err := conn.PingContext(ctx); err != nil {
log.Debug("Failed to ping database in recovery mode, closing", "path", path, "error", err)
_ = conn.Close()
return nil, fmt.Errorf(
"database still locked after recovery attempt: %w",
err,
@@ -163,6 +174,7 @@ func New(ctx context.Context, path string) (*DB, error) {
// Switch back to WAL mode
log.Debug("Switching database back to WAL mode", "path", path)
if _, err := conn.ExecContext(ctx, "PRAGMA journal_mode=WAL"); err != nil {
log.Warn("Failed to switch back to WAL mode", "path", path, "error", err)
}
@@ -175,10 +187,12 @@ func New(ctx context.Context, path string) (*DB, error) {
db := &DB{conn: conn, path: path}
if err := applyMigrations(ctx, conn); err != nil {
_ = conn.Close()
return nil, fmt.Errorf("applying migrations: %w", err)
}
log.Debug("Database connection established successfully", "path", path)
return db, nil
}
@@ -187,11 +201,16 @@ func New(ctx context.Context, path string) (*DB, error) {
// Returns an error if the database connection cannot be closed properly.
func (db *DB) Close() error {
log.Debug("Closing database connection", "path", db.path)
if err := db.conn.Close(); err != nil {
err := db.conn.Close()
if err != nil {
log.Error("Failed to close database", "path", db.path, "error", err)
return fmt.Errorf("failed to close database: %w", err)
}
log.Debug("Database connection closed successfully", "path", db.path)
return nil
}
@@ -227,9 +246,10 @@ func (db *DB) BeginTx(
func (db *DB) ExecWithLog(
ctx context.Context,
query string,
args ...interface{},
args ...any,
) (sql.Result, error) {
LogSQL("Execute", query, args...)
return db.conn.ExecContext(ctx, query, args...)
}
@@ -240,9 +260,10 @@ func (db *DB) ExecWithLog(
func (db *DB) QueryRowWithLog(
ctx context.Context,
query string,
args ...interface{},
args ...any,
) *sql.Row {
LogSQL("QueryRow", query, args...)
return db.conn.QueryRowContext(ctx, query, args...)
}
@@ -375,6 +396,7 @@ func repeatPlaceholder(n int) string {
if n <= 0 {
return ""
}
return strings.Repeat(", ?", n)
}
@@ -385,7 +407,7 @@ func repeatPlaceholder(n int) string {
// The operation parameter describes the type of SQL operation (e.g., "Execute", "Query").
// The query parameter is the SQL statement being executed.
// The args parameter contains the query arguments that will be interpolated.
func LogSQL(operation, query string, args ...interface{}) {
func LogSQL(operation, query string, args ...any) {
if strings.Contains(os.Getenv("GODEBUG"), "vaultik") {
log.Debug(
"SQL "+operation,

27
internal/database/database_test.go
View File

@@ -17,7 +17,8 @@ func TestDatabase(t *testing.T) {
t.Fatalf("failed to create database: %v", err)
}
defer func() {
if err := db.Close(); err != nil {
err := db.Close()
if err != nil {
t.Errorf("failed to close database: %v", err)
}
}()
@@ -37,6 +38,7 @@ func TestDatabase(t *testing.T) {
for _, table := range tables {
var name string
err := db.conn.QueryRow("SELECT name FROM sqlite_master WHERE type='table' AND name=?", table).Scan(&name)
if err != nil {
t.Errorf("table %s does not exist: %v", table, err)
@@ -63,7 +65,8 @@ func TestDatabaseConcurrentAccess(t *testing.T) {
t.Fatalf("failed to create database: %v", err)
}
defer func() {
if err := db.Close(); err != nil {
err := db.Close()
if err != nil {
t.Errorf("failed to close database: %v", err)
}
}()
@@ -73,9 +76,10 @@ func TestDatabaseConcurrentAccess(t *testing.T) {
index int
err error
}
results := make(chan result, 10)
for i := 0; i < 10; i++ {
for i := range 10 {
go func(i int) {
_, err := db.ExecWithLog(ctx, "INSERT INTO chunks (chunk_hash, size) VALUES (?, ?)",
fmt.Sprintf("hash%d", i), i*1024)
@@ -84,7 +88,7 @@ func TestDatabaseConcurrentAccess(t *testing.T) {
}
// Wait for all goroutines and check results
for i := 0; i < 10; i++ {
for range 10 {
r := <-results
if r.err != nil {
t.Fatalf("concurrent insert %d failed: %v", r.index, r.err)
@@ -93,10 +97,12 @@ func TestDatabaseConcurrentAccess(t *testing.T) {
// Verify all inserts succeeded
var count int
err = db.conn.QueryRowContext(ctx, "SELECT COUNT(*) FROM chunks").Scan(&count)
if err != nil {
t.Fatalf("failed to count chunks: %v", err)
}
if count != 10 {
t.Errorf("expected 10 chunks, got %d", count)
}
@@ -127,12 +133,16 @@ func TestParseMigrationVersion(t *testing.T) {
if err == nil {
t.Errorf("ParseMigrationVersion(%q) = %d, nil; want error", tc.filename, got)
}
return
}
if err != nil {
t.Errorf("ParseMigrationVersion(%q) unexpected error: %v", tc.filename, err)
return
}
if got != tc.wantVer {
t.Errorf("ParseMigrationVersion(%q) = %d; want %d", tc.filename, got, tc.wantVer)
}
@@ -148,7 +158,8 @@ func TestApplyMigrations_Idempotent(t *testing.T) {
t.Fatalf("failed to open database: %v", err)
}
defer func() {
if err := conn.Close(); err != nil {
err := conn.Close()
if err != nil {
t.Errorf("failed to close database: %v", err)
}
}()
@@ -191,7 +202,8 @@ func TestBootstrapMigrationsTable_FreshDatabase(t *testing.T) {
t.Fatalf("failed to open database: %v", err)
}
defer func() {
if err := conn.Close(); err != nil {
err := conn.Close()
if err != nil {
t.Errorf("failed to close database: %v", err)
}
}()
@@ -206,6 +218,7 @@ func TestBootstrapMigrationsTable_FreshDatabase(t *testing.T) {
).Scan(&tableBefore); err != nil {
t.Fatalf("failed to check for table before bootstrap: %v", err)
}
if tableBefore != 0 {
t.Fatal("schema_migrations table should not exist before bootstrap")
}
@@ -222,6 +235,7 @@ func TestBootstrapMigrationsTable_FreshDatabase(t *testing.T) {
).Scan(&tableAfter); err != nil {
t.Fatalf("failed to check for table after bootstrap: %v", err)
}
if tableAfter != 1 {
t.Fatalf("schema_migrations table should exist after bootstrap, got count=%d", tableAfter)
}
@@ -233,6 +247,7 @@ func TestBootstrapMigrationsTable_FreshDatabase(t *testing.T) {
).Scan(&version); err != nil {
t.Fatalf("version 0 row not found in schema_migrations: %v", err)
}
if version != 0 {
t.Errorf("expected version 0, got %d", version)
}

9
internal/database/errors.go
View File

@@ -6,15 +6,16 @@ import (
"os"
)
// Fatal prints an error message to stderr and exits with status 1
func Fatal(format string, args ...interface{}) {
// Fatalf prints an error message to stderr and exits with status 1
func Fatalf(format string, args ...any) {
fmt.Fprintf(os.Stderr, "FATAL: "+format+"\n", args...)
os.Exit(1)
}
// CloseRows closes rows and exits on error
func CloseRows(rows *sql.Rows) {
if err := rows.Close(); err != nil {
Fatal("failed to close rows: %v", err)
err := rows.Close()
if err != nil {
Fatalf("failed to close rows: %v", err)
}
}

47
internal/database/file_chunks.go
View File

@@ -4,6 +4,7 @@ import (
"context"
"database/sql"
"fmt"
"strings"
"sneak.berlin/go/vaultik/internal/types"
)
@@ -84,6 +85,7 @@ func (r *FileChunkRepository) GetByPathTx(ctx context.Context, tx *sql.Tx, path
`
LogSQL("GetByPathTx", query, path)
rows, err := tx.QueryContext(ctx, query, path)
if err != nil {
return nil, fmt.Errorf("querying file chunks: %w", err)
@@ -92,23 +94,30 @@ func (r *FileChunkRepository) GetByPathTx(ctx context.Context, tx *sql.Tx, path
fileChunks, err := r.scanFileChunks(rows)
LogSQL("GetByPathTx", "Complete", path, "count", len(fileChunks))
return fileChunks, err
}
// scanFileChunks is a helper that scans file chunk rows
func (r *FileChunkRepository) scanFileChunks(rows *sql.Rows) ([]*FileChunk, error) {
var fileChunks []*FileChunk
for rows.Next() {
var fc FileChunk
var fileIDStr, chunkHashStr string
var (
fc FileChunk
fileIDStr, chunkHashStr string
)
err := rows.Scan(&fileIDStr, &fc.Idx, &chunkHashStr)
if err != nil {
return nil, fmt.Errorf("scanning file chunk: %w", err)
}
fc.FileID, err = types.ParseFileID(fileIDStr)
if err != nil {
return nil, fmt.Errorf("parsing file ID: %w", err)
}
fc.ChunkHash = types.ChunkHash(chunkHashStr)
fileChunks = append(fileChunks, &fc)
}
@@ -161,14 +170,13 @@ func (r *FileChunkRepository) DeleteByFileIDs(ctx context.Context, tx *sql.Tx, f
const batchSize = 500
for i := 0; i < len(fileIDs); i += batchSize {
end := i + batchSize
if end > len(fileIDs) {
end = len(fileIDs)
}
end := min(i+batchSize, len(fileIDs))
batch := fileIDs[i:end]
query := "DELETE FROM file_chunks WHERE file_id IN (?" + repeatPlaceholder(len(batch)-1) + ")"
args := make([]interface{}, len(batch))
args := make([]any, len(batch))
for j, id := range batch {
args[j] = id.String()
}
@@ -179,6 +187,7 @@ func (r *FileChunkRepository) DeleteByFileIDs(ctx context.Context, tx *sql.Tx, f
} else {
_, err = r.db.ExecWithLog(ctx, query, args...)
}
if err != nil {
return fmt.Errorf("batch deleting file_chunks: %w", err)
}
@@ -199,22 +208,29 @@ func (r *FileChunkRepository) CreateBatch(ctx context.Context, tx *sql.Tx, fcs [
const batchSize = 300
for i := 0; i < len(fcs); i += batchSize {
end := i + batchSize
if end > len(fcs) {
end = len(fcs)
}
end := min(i+batchSize, len(fcs))
batch := fcs[i:end]
// Build the query with multiple value sets
query := "INSERT INTO file_chunks (file_id, idx, chunk_hash) VALUES "
args := make([]interface{}, 0, len(batch)*3)
args := make([]any, 0, len(batch)*3)
var querySb211 strings.Builder
for j, fc := range batch {
if j > 0 {
query += ", "
querySb211.WriteString(", ")
}
query += "(?, ?, ?)"
querySb211.WriteString("(?, ?, ?)")
args = append(args, fc.FileID.String(), fc.Idx, fc.ChunkHash.String())
}
query += querySb211.String()
query += " ON CONFLICT(file_id, idx) DO NOTHING"
var err error
@@ -223,6 +239,7 @@ func (r *FileChunkRepository) CreateBatch(ctx context.Context, tx *sql.Tx, fcs [
} else {
_, err = r.db.ExecWithLog(ctx, query, args...)
}
if err != nil {
return fmt.Errorf("batch inserting file_chunks: %w", err)
}
@@ -236,6 +253,7 @@ func (r *FileChunkRepository) GetByFile(ctx context.Context, path string) ([]*Fi
LogSQL("GetByFile", "Starting", path)
result, err := r.GetByPath(ctx, path)
LogSQL("GetByFile", "Complete", path, "count", len(result))
return result, err
}
@@ -244,5 +262,6 @@ func (r *FileChunkRepository) GetByFileTx(ctx context.Context, tx *sql.Tx, path
LogSQL("GetByFileTx", "Starting", path)
result, err := r.GetByPathTx(ctx, tx, path)
LogSQL("GetByFileTx", "Complete", path, "count", len(result))
return result, err
}

17
internal/database/file_chunks_test.go
View File

@@ -28,6 +28,7 @@ func TestFileChunkRepository(t *testing.T) {
GID: 1000,
LinkTarget: "",
}
err := fileRepo.Create(ctx, nil, file)
if err != nil {
t.Fatalf("failed to create file: %v", err)
@@ -36,11 +37,13 @@ func TestFileChunkRepository(t *testing.T) {
// Create chunks first
chunks := []types.ChunkHash{"chunk1", "chunk2", "chunk3"}
chunkRepo := NewChunkRepository(db)
for _, chunkHash := range chunks {
chunk := &Chunk{
ChunkHash: chunkHash,
Size: 1024,
}
err = chunkRepo.Create(ctx, nil, chunk)
if err != nil {
t.Fatalf("failed to create chunk %s: %v", chunkHash, err)
@@ -65,6 +68,7 @@ func TestFileChunkRepository(t *testing.T) {
Idx: 1,
ChunkHash: types.ChunkHash("chunk2"),
}
err = repo.Create(ctx, nil, fc2)
if err != nil {
t.Fatalf("failed to create second file chunk: %v", err)
@@ -75,6 +79,7 @@ func TestFileChunkRepository(t *testing.T) {
Idx: 2,
ChunkHash: types.ChunkHash("chunk3"),
}
err = repo.Create(ctx, nil, fc3)
if err != nil {
t.Fatalf("failed to create third file chunk: %v", err)
@@ -85,6 +90,7 @@ func TestFileChunkRepository(t *testing.T) {
if err != nil {
t.Fatalf("failed to get file chunks: %v", err)
}
if len(fileChunks) != 3 {
t.Errorf("expected 3 chunks, got %d", len(fileChunks))
}
@@ -112,6 +118,7 @@ func TestFileChunkRepository(t *testing.T) {
if err != nil {
t.Fatalf("failed to get deleted file chunks: %v", err)
}
if len(fileChunks) != 0 {
t.Errorf("expected 0 chunks after delete, got %d", len(fileChunks))
}
@@ -140,22 +147,26 @@ func TestFileChunkRepositoryMultipleFiles(t *testing.T) {
GID: 1000,
LinkTarget: "",
}
err := fileRepo.Create(ctx, nil, file)
if err != nil {
t.Fatalf("failed to create file %s: %v", path, err)
}
files[i] = file
}
// Create all chunks first
chunkRepo := NewChunkRepository(db)
for i := range files {
for j := 0; j < 2; j++ {
for j := range 2 {
chunkHash := types.ChunkHash(fmt.Sprintf("file%d_chunk%d", i, j))
chunk := &Chunk{
ChunkHash: chunkHash,
Size: 1024,
}
err := chunkRepo.Create(ctx, nil, chunk)
if err != nil {
t.Fatalf("failed to create chunk %s: %v", chunkHash, err)
@@ -165,12 +176,13 @@ func TestFileChunkRepositoryMultipleFiles(t *testing.T) {
// Create chunks for multiple files
for i, file := range files {
for j := 0; j < 2; j++ {
for j := range 2 {
fc := &FileChunk{
FileID: file.ID,
Idx: j,
ChunkHash: types.ChunkHash(fmt.Sprintf("file%d_chunk%d", i, j)),
}
err := repo.Create(ctx, nil, fc)
if err != nil {
t.Fatalf("failed to create file chunk: %v", err)
@@ -184,6 +196,7 @@ func TestFileChunkRepositoryMultipleFiles(t *testing.T) {
if err != nil {
t.Fatalf("failed to get chunks for file %d: %v", i, err)
}
if len(chunks) != 2 {
t.Errorf("expected 2 chunks for file %d, got %d", i, len(chunks))
}

70
internal/database/files.go
View File

@@ -3,7 +3,9 @@ package database
import (
"context"
"database/sql"
"errors"
"fmt"
"strings"
"time"
"sneak.berlin/go/vaultik/internal/log"
@@ -38,8 +40,11 @@ func (r *FileRepository) Create(ctx context.Context, tx *sql.Tx, file *File) err
RETURNING id
`
var idStr string
var err error
var (
idStr string
err error
)
if tx != nil {
LogSQL("Execute", query, file.ID.String(), file.Path.String(), file.SourcePath.String(), file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID, file.LinkTarget.String())
err = tx.QueryRowContext(ctx, query, file.ID.String(), file.Path.String(), file.SourcePath.String(), file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID, file.LinkTarget.String()).Scan(&idStr)
@@ -68,9 +73,10 @@ func (r *FileRepository) GetByPath(ctx context.Context, path string) (*File, err
`
file, err := r.scanFile(r.db.conn.QueryRowContext(ctx, query, path))
if err == sql.ErrNoRows {
if errors.Is(err, sql.ErrNoRows) {
return nil, nil
}
if err != nil {
return nil, fmt.Errorf("querying file: %w", err)
}
@@ -87,9 +93,10 @@ func (r *FileRepository) GetByID(ctx context.Context, id types.FileID) (*File, e
`
file, err := r.scanFile(r.db.conn.QueryRowContext(ctx, query, id.String()))
if err == sql.ErrNoRows {
if errors.Is(err, sql.ErrNoRows) {
return nil, nil
}
if err != nil {
return nil, fmt.Errorf("querying file: %w", err)
}
@@ -108,9 +115,10 @@ func (r *FileRepository) GetByPathTx(ctx context.Context, tx *sql.Tx, path strin
file, err := r.scanFile(tx.QueryRowContext(ctx, query, path))
LogSQL("GetByPathTx Scan complete", query, path)
if err == sql.ErrNoRows {
if errors.Is(err, sql.ErrNoRows) {
return nil, nil
}
if err != nil {
return nil, fmt.Errorf("querying file: %w", err)
}
@@ -120,10 +128,12 @@ func (r *FileRepository) GetByPathTx(ctx context.Context, tx *sql.Tx, path strin
// scanFile is a helper that scans a single file row
func (r *FileRepository) scanFile(row *sql.Row) (*File, error) {
var file File
var idStr, pathStr, sourcePathStr string
var mtimeUnix int64
var linkTarget sql.NullString
var (
file File
idStr, pathStr, sourcePathStr string
mtimeUnix int64
linkTarget sql.NullString
)
err := row.Scan(
&idStr,
@@ -144,8 +154,10 @@ func (r *FileRepository) scanFile(row *sql.Row) (*File, error) {
if err != nil {
return nil, fmt.Errorf("parsing file ID: %w", err)
}
file.Path = types.FilePath(pathStr)
file.SourcePath = types.SourcePath(sourcePathStr)
file.MTime = time.Unix(mtimeUnix, 0).UTC()
if linkTarget.Valid {
file.LinkTarget = types.FilePath(linkTarget.String)
@@ -156,10 +168,12 @@ func (r *FileRepository) scanFile(row *sql.Row) (*File, error) {
// scanFileRows is a helper that scans a file row from rows iterator
func (r *FileRepository) scanFileRows(rows *sql.Rows) (*File, error) {
var file File
var idStr, pathStr, sourcePathStr string
var mtimeUnix int64
var linkTarget sql.NullString
var (
file File
idStr, pathStr, sourcePathStr string
mtimeUnix int64
linkTarget sql.NullString
)
err := rows.Scan(
&idStr,
@@ -180,8 +194,10 @@ func (r *FileRepository) scanFileRows(rows *sql.Rows) (*File, error) {
if err != nil {
return nil, fmt.Errorf("parsing file ID: %w", err)
}
file.Path = types.FilePath(pathStr)
file.SourcePath = types.SourcePath(sourcePathStr)
file.MTime = time.Unix(mtimeUnix, 0).UTC()
if linkTarget.Valid {
file.LinkTarget = types.FilePath(linkTarget.String)
@@ -205,11 +221,13 @@ func (r *FileRepository) ListModifiedSince(ctx context.Context, since time.Time)
defer CloseRows(rows)
var files []*File
for rows.Next() {
file, err := r.scanFileRows(rows)
if err != nil {
return nil, fmt.Errorf("scanning file: %w", err)
}
files = append(files, file)
}
@@ -266,11 +284,13 @@ func (r *FileRepository) ListByPrefix(ctx context.Context, prefix string) ([]*Fi
defer CloseRows(rows)
var files []*File
for rows.Next() {
file, err := r.scanFileRows(rows)
if err != nil {
return nil, fmt.Errorf("scanning file: %w", err)
}
files = append(files, file)
}
@@ -292,11 +312,13 @@ func (r *FileRepository) ListAll(ctx context.Context) ([]*File, error) {
defer CloseRows(rows)
var files []*File
for rows.Next() {
file, err := r.scanFileRows(rows)
if err != nil {
return nil, fmt.Errorf("scanning file: %w", err)
}
files = append(files, file)
}
@@ -314,21 +336,28 @@ func (r *FileRepository) CreateBatch(ctx context.Context, tx *sql.Tx, files []*F
const batchSize = 100
for i := 0; i < len(files); i += batchSize {
end := i + batchSize
if end > len(files) {
end = len(files)
}
end := min(i+batchSize, len(files))
batch := files[i:end]
query := `INSERT INTO files (id, path, source_path, mtime, size, mode, uid, gid, link_target) VALUES `
args := make([]interface{}, 0, len(batch)*9)
args := make([]any, 0, len(batch)*9)
var querySb325 strings.Builder
for j, f := range batch {
if j > 0 {
query += ", "
querySb325.WriteString(", ")
}
query += "(?, ?, ?, ?, ?, ?, ?, ?, ?)"
querySb325.WriteString("(?, ?, ?, ?, ?, ?, ?, ?, ?)")
args = append(args, f.ID.String(), f.Path.String(), f.SourcePath.String(), f.MTime.Unix(), f.Size, f.Mode, f.UID, f.GID, f.LinkTarget.String())
}
query += querySb325.String()
query += ` ON CONFLICT(path) DO UPDATE SET
source_path = excluded.source_path,
mtime = excluded.mtime,
@@ -344,6 +373,7 @@ func (r *FileRepository) CreateBatch(ctx context.Context, tx *sql.Tx, files []*F
} else {
_, err = r.db.ExecWithLog(ctx, query, args...)
}
if err != nil {
return fmt.Errorf("batch inserting files: %w", err)
}

22
internal/database/files_test.go
View File

@@ -3,7 +3,7 @@ package database
import (
"context"
"database/sql"
"fmt"
"errors"
"os"
"path/filepath"
"testing"
@@ -20,7 +20,8 @@ func setupTestDB(t *testing.T) (*DB, func()) {
}
cleanup := func() {
if err := db.Close(); err != nil {
err := db.Close()
if err != nil {
t.Errorf("failed to close database: %v", err)
}
}
@@ -56,18 +57,23 @@ func TestFileRepository(t *testing.T) {
if err != nil {
t.Fatalf("failed to get file: %v", err)
}
if retrieved == nil {
t.Fatal("expected file, got nil")
}
if retrieved.Path != file.Path {
t.Errorf("path mismatch: got %s, want %s", retrieved.Path, file.Path)
}
if !retrieved.MTime.Equal(file.MTime) {
t.Errorf("mtime mismatch: got %v, want %v", retrieved.MTime, file.MTime)
}
if retrieved.Size != file.Size {
t.Errorf("size mismatch: got %d, want %d", retrieved.Size, file.Size)
}
if retrieved.Mode != file.Mode {
t.Errorf("mode mismatch: got %o, want %o", retrieved.Mode, file.Mode)
}
@@ -75,6 +81,7 @@ func TestFileRepository(t *testing.T) {
// Test Update (upsert)
file.Size = 2048
file.MTime = time.Now().Truncate(time.Second)
err = repo.Create(ctx, nil, file)
if err != nil {
t.Fatalf("failed to update file: %v", err)
@@ -84,6 +91,7 @@ func TestFileRepository(t *testing.T) {
if err != nil {
t.Fatalf("failed to get updated file: %v", err)
}
if retrieved.Size != 2048 {
t.Errorf("size not updated: got %d, want %d", retrieved.Size, 2048)
}
@@ -93,6 +101,7 @@ func TestFileRepository(t *testing.T) {
if err != nil {
t.Fatalf("failed to list files: %v", err)
}
if len(files) != 1 {
t.Errorf("expected 1 file, got %d", len(files))
}
@@ -107,6 +116,7 @@ func TestFileRepository(t *testing.T) {
if err != nil {
t.Fatalf("error getting deleted file: %v", err)
}
if retrieved != nil {
t.Error("expected nil for deleted file")
}
@@ -139,9 +149,11 @@ func TestFileRepositorySymlink(t *testing.T) {
if err != nil {
t.Fatalf("failed to get symlink: %v", err)
}
if !retrieved.IsSymlink() {
t.Error("expected IsSymlink() to be true")
}
if retrieved.LinkTarget != symlink.LinkTarget {
t.Errorf("link target mismatch: got %s, want %s", retrieved.LinkTarget, symlink.LinkTarget)
}
@@ -165,12 +177,13 @@ func TestFileRepositoryTransaction(t *testing.T) {
GID: 1000,
}
if err := repos.Files.Create(ctx, tx, file); err != nil {
err := repos.Files.Create(ctx, tx, file)
if err != nil {
return err
}
// Return error to trigger rollback
return fmt.Errorf("test rollback")
return errors.New("test rollback")
})
if err == nil || err.Error() != "test rollback" {
@@ -182,6 +195,7 @@ func TestFileRepositoryTransaction(t *testing.T) {
if err != nil {
t.Fatalf("error checking for file: %v", err)
}
if retrieved != nil {
t.Error("file should not exist after rollback")
}

4
internal/database/local_meta.go
View File

@@ -27,15 +27,18 @@ func NewLocalMetaRepository(db *DB) *LocalMetaRepository {
// "unset" (bind on first use) from "set to something" (compare).
func (r *LocalMetaRepository) Get(ctx context.Context, key string) (string, error) {
var value string
err := r.db.conn.QueryRowContext(ctx,
"SELECT value FROM local_meta WHERE key = ?", key,
).Scan(&value)
if errors.Is(err, sql.ErrNoRows) {
return "", nil
}
if err != nil {
return "", fmt.Errorf("reading local_meta %q: %w", key, err)
}
return value, nil
}
@@ -49,5 +52,6 @@ func (r *LocalMetaRepository) Set(ctx context.Context, key, value string) error
if err != nil {
return fmt.Errorf("writing local_meta %q: %w", key, err)
}
return nil
}

5
internal/database/local_meta_test.go
View File

@@ -11,18 +11,20 @@ import (
func TestLocalMetaEmptyOnFresh(t *testing.T) {
db, err := database.NewTestDB()
require.NoError(t, err)
defer func() { _ = db.Close() }()
repos := database.NewRepositories(db)
got, err := repos.LocalMeta.Get(context.Background(), database.LocalMetaKeyStorageURL)
require.NoError(t, err)
require.Equal(t, "", got, "fresh DB must return empty for unset keys, not error")
require.Empty(t, got, "fresh DB must return empty for unset keys, not error")
}
func TestLocalMetaSetGetRoundTrip(t *testing.T) {
db, err := database.NewTestDB()
require.NoError(t, err)
defer func() { _ = db.Close() }()
repos := database.NewRepositories(db)
@@ -38,6 +40,7 @@ func TestLocalMetaSetGetRoundTrip(t *testing.T) {
func TestLocalMetaSetOverwrites(t *testing.T) {
db, err := database.NewTestDB()
require.NoError(t, err)
defer func() { _ = db.Close() }()
repos := database.NewRepositories(db)

7
internal/database/module.go
View File

@@ -34,11 +34,16 @@ func provideDatabase(lc fx.Lifecycle, cfg *config.Config) (*DB, error) {
lc.Append(fx.Hook{
OnStop: func(ctx context.Context) error {
log.Debug("Database module OnStop hook called")
if err := db.Close(); err != nil {
err := db.Close()
if err != nil {
log.Error("Failed to close database in OnStop hook", "error", err)
return err
}
log.Debug("Database closed successfully in OnStop hook")
return nil
},
})

25
internal/database/repositories.go
View File

@@ -50,21 +50,26 @@ type TxFunc func(ctx context.Context, tx *sql.Tx) error
// This method should be used for all write operations to ensure atomicity.
func (r *Repositories) WithTx(ctx context.Context, fn TxFunc) error {
LogSQL("WithTx", "Beginning transaction", "")
tx, err := r.db.BeginTx(ctx, nil)
if err != nil {
return fmt.Errorf("beginning transaction: %w", err)
}
LogSQL("WithTx", "Transaction started", "")
defer func() {
if p := recover(); p != nil {
if rollbackErr := tx.Rollback(); rollbackErr != nil {
Fatal("failed to rollback transaction: %v", rollbackErr)
rollbackErr := tx.Rollback()
if rollbackErr != nil {
Fatalf("failed to rollback transaction: %v", rollbackErr)
}
panic(p)
} else if err != nil {
if rollbackErr := tx.Rollback(); rollbackErr != nil {
Fatal("failed to rollback transaction: %v", rollbackErr)
rollbackErr := tx.Rollback()
if rollbackErr != nil {
Fatalf("failed to rollback transaction: %v", rollbackErr)
}
}
}()
@@ -90,6 +95,7 @@ func (r *Repositories) WithReadTx(ctx context.Context, fn TxFunc) error {
opts := &sql.TxOptions{
ReadOnly: true,
}
tx, err := r.db.BeginTx(ctx, opts)
if err != nil {
return fmt.Errorf("beginning read transaction: %w", err)
@@ -97,13 +103,16 @@ func (r *Repositories) WithReadTx(ctx context.Context, fn TxFunc) error {
defer func() {
if p := recover(); p != nil {
if rollbackErr := tx.Rollback(); rollbackErr != nil {
Fatal("failed to rollback transaction: %v", rollbackErr)
rollbackErr := tx.Rollback()
if rollbackErr != nil {
Fatalf("failed to rollback transaction: %v", rollbackErr)
}
panic(p)
} else if err != nil {
if rollbackErr := tx.Rollback(); rollbackErr != nil {
Fatal("failed to rollback transaction: %v", rollbackErr)
rollbackErr := tx.Rollback()
if rollbackErr != nil {
Fatalf("failed to rollback transaction: %v", rollbackErr)
}
}
}()

54
internal/database/repositories_test.go
View File

@@ -3,7 +3,7 @@ package database
import (
"context"
"database/sql"
"fmt"
"errors"
"testing"
"time"
@@ -28,7 +28,9 @@ func TestRepositoriesTransaction(t *testing.T) {
UID: 1000,
GID: 1000,
}
if err := repos.Files.Create(ctx, tx, file); err != nil {
err := repos.Files.Create(ctx, tx, file)
if err != nil {
return err
}
@@ -37,7 +39,9 @@ func TestRepositoriesTransaction(t *testing.T) {
ChunkHash: types.ChunkHash("tx_chunk1"),
Size: 512,
}
if err := repos.Chunks.Create(ctx, tx, chunk1); err != nil {
err = repos.Chunks.Create(ctx, tx, chunk1)
if err != nil {
return err
}
@@ -45,7 +49,9 @@ func TestRepositoriesTransaction(t *testing.T) {
ChunkHash: types.ChunkHash("tx_chunk2"),
Size: 512,
}
if err := repos.Chunks.Create(ctx, tx, chunk2); err != nil {
err = repos.Chunks.Create(ctx, tx, chunk2)
if err != nil {
return err
}
@@ -55,7 +61,9 @@ func TestRepositoriesTransaction(t *testing.T) {
Idx: 0,
ChunkHash: chunk1.ChunkHash,
}
if err := repos.FileChunks.Create(ctx, tx, fc1); err != nil {
err = repos.FileChunks.Create(ctx, tx, fc1)
if err != nil {
return err
}
@@ -64,7 +72,9 @@ func TestRepositoriesTransaction(t *testing.T) {
Idx: 1,
ChunkHash: chunk2.ChunkHash,
}
if err := repos.FileChunks.Create(ctx, tx, fc2); err != nil {
err = repos.FileChunks.Create(ctx, tx, fc2)
if err != nil {
return err
}
@@ -74,7 +84,9 @@ func TestRepositoriesTransaction(t *testing.T) {
Hash: types.BlobHash("tx_blob1"),
CreatedTS: time.Now().Truncate(time.Second),
}
if err := repos.Blobs.Create(ctx, tx, blob); err != nil {
err = repos.Blobs.Create(ctx, tx, blob)
if err != nil {
return err
}
@@ -85,7 +97,9 @@ func TestRepositoriesTransaction(t *testing.T) {
Offset: 0,
Length: 512,
}
if err := repos.BlobChunks.Create(ctx, tx, bc1); err != nil {
err = repos.BlobChunks.Create(ctx, tx, bc1)
if err != nil {
return err
}
@@ -95,13 +109,14 @@ func TestRepositoriesTransaction(t *testing.T) {
Offset: 512,
Length: 512,
}
if err := repos.BlobChunks.Create(ctx, tx, bc2); err != nil {
err = repos.BlobChunks.Create(ctx, tx, bc2)
if err != nil {
return err
}
return nil
})
if err != nil {
t.Fatalf("transaction failed: %v", err)
}
@@ -111,6 +126,7 @@ func TestRepositoriesTransaction(t *testing.T) {
if err != nil {
t.Fatalf("failed to get file: %v", err)
}
if file == nil {
t.Error("expected file after transaction")
}
@@ -119,6 +135,7 @@ func TestRepositoriesTransaction(t *testing.T) {
if err != nil {
t.Fatalf("failed to get file chunks: %v", err)
}
if len(chunks) != 2 {
t.Errorf("expected 2 file chunks, got %d", len(chunks))
}
@@ -127,6 +144,7 @@ func TestRepositoriesTransaction(t *testing.T) {
if err != nil {
t.Fatalf("failed to get blob: %v", err)
}
if blob == nil {
t.Error("expected blob after transaction")
}
@@ -150,7 +168,9 @@ func TestRepositoriesTransactionRollback(t *testing.T) {
UID: 1000,
GID: 1000,
}
if err := repos.Files.Create(ctx, tx, file); err != nil {
err := repos.Files.Create(ctx, tx, file)
if err != nil {
return err
}
@@ -159,12 +179,14 @@ func TestRepositoriesTransactionRollback(t *testing.T) {
ChunkHash: types.ChunkHash("rollback_chunk"),
Size: 1024,
}
if err := repos.Chunks.Create(ctx, tx, chunk); err != nil {
err = repos.Chunks.Create(ctx, tx, chunk)
if err != nil {
return err
}
// Return error to trigger rollback
return fmt.Errorf("intentional rollback")
return errors.New("intentional rollback")
})
if err == nil || err.Error() != "intentional rollback" {
@@ -176,6 +198,7 @@ func TestRepositoriesTransactionRollback(t *testing.T) {
if err != nil {
t.Fatalf("error checking for file: %v", err)
}
if file != nil {
t.Error("file should not exist after rollback")
}
@@ -184,6 +207,7 @@ func TestRepositoriesTransactionRollback(t *testing.T) {
if err != nil {
t.Fatalf("error checking for chunk: %v", err)
}
if chunk != nil {
t.Error("chunk should not exist after rollback")
}
@@ -205,6 +229,7 @@ func TestRepositoriesReadTransaction(t *testing.T) {
UID: 1000,
GID: 1000,
}
err := repos.Files.Create(ctx, nil, file)
if err != nil {
t.Fatalf("failed to create file: %v", err)
@@ -212,8 +237,10 @@ func TestRepositoriesReadTransaction(t *testing.T) {
// Test read-only transaction
var retrievedFile *File
err = repos.WithReadTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
var err error
retrievedFile, err = repos.Files.GetByPathTx(ctx, tx, "/test/read_file.txt")
if err != nil {
return err
@@ -232,7 +259,6 @@ func TestRepositoriesReadTransaction(t *testing.T) {
return nil
})
if err != nil {
t.Fatalf("read transaction failed: %v", err)
}

65
internal/database/repository_comprehensive_test.go
View File

@@ -3,6 +3,7 @@ package database
import (
"context"
"database/sql"
"errors"
"fmt"
"testing"
"time"
@@ -39,6 +40,7 @@ func TestFileRepositoryUUIDGeneration(t *testing.T) {
}
uuids := make(map[string]bool)
for _, file := range files {
err := repo.Create(ctx, nil, file)
if err != nil {
@@ -54,6 +56,7 @@ func TestFileRepositoryUUIDGeneration(t *testing.T) {
if uuids[file.ID.String()] {
t.Errorf("duplicate UUID generated: %s", file.ID)
}
uuids[file.ID.String()] = true
}
}
@@ -90,16 +93,19 @@ func TestFileRepositoryGetByID(t *testing.T) {
if retrieved.ID != file.ID {
t.Errorf("ID mismatch: expected %s, got %s", file.ID, retrieved.ID)
}
if retrieved.Path != file.Path {
t.Errorf("Path mismatch: expected %s, got %s", file.Path, retrieved.Path)
}
// Test non-existent ID
nonExistentID := types.NewFileID() // Generate a new UUID that won't exist in the database
nonExistent, err := repo.GetByID(ctx, nonExistentID)
if err != nil {
t.Fatalf("GetByID should not return error for non-existent ID: %v", err)
}
if nonExistent != nil {
t.Error("expected nil for non-existent ID")
}
@@ -135,6 +141,7 @@ func TestOrphanedFileCleanup(t *testing.T) {
if err != nil {
t.Fatalf("failed to create file1: %v", err)
}
err = repos.Files.Create(ctx, nil, file2)
if err != nil {
t.Fatalf("failed to create file2: %v", err)
@@ -146,6 +153,7 @@ func TestOrphanedFileCleanup(t *testing.T) {
Hostname: "test-host",
StartedAt: time.Now(),
}
err = repos.Snapshots.Create(ctx, nil, snapshot)
if err != nil {
t.Fatalf("failed to create snapshot: %v", err)
@@ -168,6 +176,7 @@ func TestOrphanedFileCleanup(t *testing.T) {
if err != nil {
t.Fatalf("error getting file: %v", err)
}
if orphanedFile != nil {
t.Error("orphaned file should have been deleted")
}
@@ -177,6 +186,7 @@ func TestOrphanedFileCleanup(t *testing.T) {
if err != nil {
t.Fatalf("error getting file: %v", err)
}
if referencedFile == nil {
t.Error("referenced file should not have been deleted")
}
@@ -204,6 +214,7 @@ func TestOrphanedChunkCleanup(t *testing.T) {
if err != nil {
t.Fatalf("failed to create chunk1: %v", err)
}
err = repos.Chunks.Create(ctx, nil, chunk2)
if err != nil {
t.Fatalf("failed to create chunk2: %v", err)
@@ -218,6 +229,7 @@ func TestOrphanedChunkCleanup(t *testing.T) {
UID: 1000,
GID: 1000,
}
err = repos.Files.Create(ctx, nil, file)
if err != nil {
t.Fatalf("failed to create file: %v", err)
@@ -229,6 +241,7 @@ func TestOrphanedChunkCleanup(t *testing.T) {
Idx: 0,
ChunkHash: chunk2.ChunkHash,
}
err = repos.FileChunks.Create(ctx, nil, fc)
if err != nil {
t.Fatalf("failed to create file chunk: %v", err)
@@ -245,6 +258,7 @@ func TestOrphanedChunkCleanup(t *testing.T) {
if err != nil {
t.Fatalf("error getting chunk: %v", err)
}
if orphanedChunk != nil {
t.Error("orphaned chunk should have been deleted")
}
@@ -254,6 +268,7 @@ func TestOrphanedChunkCleanup(t *testing.T) {
if err != nil {
t.Fatalf("error getting chunk: %v", err)
}
if referencedChunk == nil {
t.Error("referenced chunk should not have been deleted")
}
@@ -283,6 +298,7 @@ func TestOrphanedBlobCleanup(t *testing.T) {
if err != nil {
t.Fatalf("failed to create blob1: %v", err)
}
err = repos.Blobs.Create(ctx, nil, blob2)
if err != nil {
t.Fatalf("failed to create blob2: %v", err)
@@ -294,6 +310,7 @@ func TestOrphanedBlobCleanup(t *testing.T) {
Hostname: "test-host",
StartedAt: time.Now(),
}
err = repos.Snapshots.Create(ctx, nil, snapshot)
if err != nil {
t.Fatalf("failed to create snapshot: %v", err)
@@ -316,6 +333,7 @@ func TestOrphanedBlobCleanup(t *testing.T) {
if err != nil {
t.Fatalf("error getting blob: %v", err)
}
if orphanedBlob != nil {
t.Error("orphaned blob should have been deleted")
}
@@ -325,6 +343,7 @@ func TestOrphanedBlobCleanup(t *testing.T) {
if err != nil {
t.Fatalf("error getting blob: %v", err)
}
if referencedBlob == nil {
t.Error("referenced blob should not have been deleted")
}
@@ -347,6 +366,7 @@ func TestFileChunkRepositoryWithUUIDs(t *testing.T) {
UID: 1000,
GID: 1000,
}
err := repos.Files.Create(ctx, nil, file)
if err != nil {
t.Fatalf("failed to create file: %v", err)
@@ -359,6 +379,7 @@ func TestFileChunkRepositoryWithUUIDs(t *testing.T) {
ChunkHash: chunkHash,
Size: 1024,
}
err = repos.Chunks.Create(ctx, nil, chunk)
if err != nil {
t.Fatalf("failed to create chunk: %v", err)
@@ -370,6 +391,7 @@ func TestFileChunkRepositoryWithUUIDs(t *testing.T) {
Idx: i,
ChunkHash: chunkHash,
}
err = repos.FileChunks.Create(ctx, nil, fc)
if err != nil {
t.Fatalf("failed to create file chunk: %v", err)
@@ -381,6 +403,7 @@ func TestFileChunkRepositoryWithUUIDs(t *testing.T) {
if err != nil {
t.Fatalf("failed to get file chunks: %v", err)
}
if len(fileChunks) != 3 {
t.Errorf("expected 3 chunks, got %d", len(fileChunks))
}
@@ -395,6 +418,7 @@ func TestFileChunkRepositoryWithUUIDs(t *testing.T) {
if err != nil {
t.Fatalf("failed to get file chunks after delete: %v", err)
}
if len(fileChunks) != 0 {
t.Errorf("expected 0 chunks after delete, got %d", len(fileChunks))
}
@@ -430,6 +454,7 @@ func TestChunkFileRepositoryWithUUIDs(t *testing.T) {
if err != nil {
t.Fatalf("failed to create file1: %v", err)
}
err = repos.Files.Create(ctx, nil, file2)
if err != nil {
t.Fatalf("failed to create file2: %v", err)
@@ -440,6 +465,7 @@ func TestChunkFileRepositoryWithUUIDs(t *testing.T) {
ChunkHash: types.ChunkHash("shared-chunk"),
Size: 1024,
}
err = repos.Chunks.Create(ctx, nil, chunk)
if err != nil {
t.Fatalf("failed to create chunk: %v", err)
@@ -463,6 +489,7 @@ func TestChunkFileRepositoryWithUUIDs(t *testing.T) {
if err != nil {
t.Fatalf("failed to create chunk file 1: %v", err)
}
err = repos.ChunkFiles.Create(ctx, nil, cf2)
if err != nil {
t.Fatalf("failed to create chunk file 2: %v", err)
@@ -473,6 +500,7 @@ func TestChunkFileRepositoryWithUUIDs(t *testing.T) {
if err != nil {
t.Fatalf("failed to get chunk files: %v", err)
}
if len(chunkFiles) != 2 {
t.Errorf("expected 2 files for chunk, got %d", len(chunkFiles))
}
@@ -482,6 +510,7 @@ func TestChunkFileRepositoryWithUUIDs(t *testing.T) {
if err != nil {
t.Fatalf("failed to get chunks by file ID: %v", err)
}
if len(chunkFiles) != 1 {
t.Errorf("expected 1 chunk for file, got %d", len(chunkFiles))
}
@@ -528,15 +557,19 @@ func TestSnapshotRepositoryExtendedFields(t *testing.T) {
if retrieved.VaultikVersion != snapshot.VaultikVersion {
t.Errorf("version mismatch: expected %s, got %s", snapshot.VaultikVersion, retrieved.VaultikVersion)
}
if retrieved.VaultikGitRevision != snapshot.VaultikGitRevision {
t.Errorf("git revision mismatch: expected %s, got %s", snapshot.VaultikGitRevision, retrieved.VaultikGitRevision)
}
if retrieved.CompressionLevel != snapshot.CompressionLevel {
t.Errorf("compression level mismatch: expected %d, got %d", snapshot.CompressionLevel, retrieved.CompressionLevel)
}
if retrieved.BlobUncompressedSize != snapshot.BlobUncompressedSize {
t.Errorf("uncompressed size mismatch: expected %d, got %d", snapshot.BlobUncompressedSize, retrieved.BlobUncompressedSize)
}
if retrieved.UploadDurationMs != snapshot.UploadDurationMs {
t.Errorf("upload duration mismatch: expected %d, got %d", snapshot.UploadDurationMs, retrieved.UploadDurationMs)
}
@@ -566,6 +599,7 @@ func TestComplexOrphanedDataScenario(t *testing.T) {
if err != nil {
t.Fatalf("failed to create snapshot1: %v", err)
}
err = repos.Snapshots.Create(ctx, nil, snapshot2)
if err != nil {
t.Fatalf("failed to create snapshot2: %v", err)
@@ -582,6 +616,7 @@ func TestComplexOrphanedDataScenario(t *testing.T) {
UID: 1000,
GID: 1000,
}
err = repos.Files.Create(ctx, nil, files[i])
if err != nil {
t.Fatalf("failed to create file%d: %v", i, err)
@@ -598,14 +633,17 @@ func TestComplexOrphanedDataScenario(t *testing.T) {
if err != nil {
t.Fatal(err)
}
err = repos.Snapshots.AddFileByID(ctx, nil, snapshot1.ID.String(), files[1].ID)
if err != nil {
t.Fatal(err)
}
err = repos.Snapshots.AddFileByID(ctx, nil, snapshot2.ID.String(), files[1].ID)
if err != nil {
t.Fatal(err)
}
err = repos.Snapshots.AddFileByID(ctx, nil, snapshot2.ID.String(), files[2].ID)
if err != nil {
t.Fatal(err)
@@ -616,6 +654,7 @@ func TestComplexOrphanedDataScenario(t *testing.T) {
if err != nil {
t.Fatal(err)
}
err = repos.Snapshots.Delete(ctx, snapshot1.ID.String())
if err != nil {
t.Fatal(err)
@@ -633,6 +672,7 @@ func TestComplexOrphanedDataScenario(t *testing.T) {
if err != nil {
t.Fatalf("error getting file0: %v", err)
}
if file0 != nil {
t.Error("file0 should have been deleted")
}
@@ -642,6 +682,7 @@ func TestComplexOrphanedDataScenario(t *testing.T) {
if err != nil {
t.Fatalf("error getting file1: %v", err)
}
if file1 == nil {
t.Error("file1 should still exist")
}
@@ -651,6 +692,7 @@ func TestComplexOrphanedDataScenario(t *testing.T) {
if err != nil {
t.Fatalf("error getting file2: %v", err)
}
if file2 == nil {
t.Error("file2 should still exist")
}
@@ -673,17 +715,19 @@ func TestCascadeDelete(t *testing.T) {
UID: 1000,
GID: 1000,
}
err := repos.Files.Create(ctx, nil, file)
if err != nil {
t.Fatalf("failed to create file: %v", err)
}
// Create chunks and file-chunk mappings
for i := 0; i < 3; i++ {
for i := range 3 {
chunk := &Chunk{
ChunkHash: types.ChunkHash(fmt.Sprintf("cascade-chunk-%d", i)),
Size: 1024,
}
err = repos.Chunks.Create(ctx, nil, chunk)
if err != nil {
t.Fatalf("failed to create chunk: %v", err)
@@ -694,6 +738,7 @@ func TestCascadeDelete(t *testing.T) {
Idx: i,
ChunkHash: chunk.ChunkHash,
}
err = repos.FileChunks.Create(ctx, nil, fc)
if err != nil {
t.Fatalf("failed to create file chunk: %v", err)
@@ -705,6 +750,7 @@ func TestCascadeDelete(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if len(fileChunks) != 3 {
t.Errorf("expected 3 file chunks, got %d", len(fileChunks))
}
@@ -720,6 +766,7 @@ func TestCascadeDelete(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if len(fileChunks) != 0 {
t.Errorf("expected 0 file chunks after cascade delete, got %d", len(fileChunks))
}
@@ -744,6 +791,7 @@ func TestTransactionIsolation(t *testing.T) {
UID: 1000,
GID: 1000,
}
err := repos.Files.Create(ctx, tx, file)
if err != nil {
return err
@@ -754,9 +802,8 @@ func TestTransactionIsolation(t *testing.T) {
// For now, we'll just test that rollback works
// Return an error to trigger rollback
return fmt.Errorf("intentional rollback")
return errors.New("intentional rollback")
})
if err == nil {
t.Fatal("expected error from transaction")
}
@@ -766,6 +813,7 @@ func TestTransactionIsolation(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if len(files) != 0 {
t.Error("file should not exist after rollback")
}
@@ -790,13 +838,14 @@ func TestConcurrentOrphanedCleanup(t *testing.T) {
Hostname: "test-host",
StartedAt: time.Now(),
}
err := repos.Snapshots.Create(ctx, nil, snapshot)
if err != nil {
t.Fatal(err)
}
// Create many files, some orphaned
for i := 0; i < 20; i++ {
for i := range 20 {
file := &File{
Path: types.FilePath(fmt.Sprintf("/concurrent-%d.txt", i)),
MTime: time.Now().Truncate(time.Second),
@@ -805,6 +854,7 @@ func TestConcurrentOrphanedCleanup(t *testing.T) {
UID: 1000,
GID: 1000,
}
err = repos.Files.Create(ctx, nil, file)
if err != nil {
t.Fatal(err)
@@ -822,14 +872,15 @@ func TestConcurrentOrphanedCleanup(t *testing.T) {
// Run multiple cleanup operations concurrently
// Note: SQLite has limited support for concurrent writes, so we expect some to fail
done := make(chan error, 3)
for i := 0; i < 3; i++ {
for range 3 {
go func() {
done <- repos.Files.DeleteOrphaned(ctx)
}()
}
// Wait for all to complete
for i := 0; i < 3; i++ {
for i := range 3 {
err := <-done
if err != nil {
t.Errorf("cleanup %d failed: %v", i, err)
@@ -850,10 +901,12 @@ func TestConcurrentOrphanedCleanup(t *testing.T) {
// Verify all remaining files are even-numbered
for _, file := range files {
var num int
_, err := fmt.Sscanf(file.Path.String(), "/concurrent-%d.txt", &num)
if err != nil {
t.Logf("failed to parse file number from %s: %v", file.Path, err)
}
if num%2 != 0 {
t.Errorf("odd-numbered file %s should have been deleted", file.Path)
}

27
internal/database/repository_debug_test.go
View File

@@ -36,12 +36,14 @@ func TestOrphanedFileCleanupDebug(t *testing.T) {
if err != nil {
t.Fatalf("failed to create file1: %v", err)
}
t.Logf("Created file1 with ID: %s", file1.ID)
err = repos.Files.Create(ctx, nil, file2)
if err != nil {
t.Fatalf("failed to create file2: %v", err)
}
t.Logf("Created file2 with ID: %s", file2.ID)
// Create a snapshot and reference only file2
@@ -50,18 +52,22 @@ func TestOrphanedFileCleanupDebug(t *testing.T) {
Hostname: "test-host",
StartedAt: time.Now(),
}
err = repos.Snapshots.Create(ctx, nil, snapshot)
if err != nil {
t.Fatalf("failed to create snapshot: %v", err)
}
t.Logf("Created snapshot: %s", snapshot.ID)
// Check snapshot_files before adding
var count int
err = db.conn.QueryRow("SELECT COUNT(*) FROM snapshot_files").Scan(&count)
if err != nil {
t.Fatal(err)
}
t.Logf("snapshot_files count before add: %d", count)
// Add file2 to snapshot
@@ -69,6 +75,7 @@ func TestOrphanedFileCleanupDebug(t *testing.T) {
if err != nil {
t.Fatalf("failed to add file to snapshot: %v", err)
}
t.Logf("Added file2 to snapshot")
// Check snapshot_files after adding
@@ -76,6 +83,7 @@ func TestOrphanedFileCleanupDebug(t *testing.T) {
if err != nil {
t.Fatal(err)
}
t.Logf("snapshot_files count after add: %d", count)
// Check which files are referenced
@@ -84,16 +92,22 @@ func TestOrphanedFileCleanupDebug(t *testing.T) {
t.Fatal(err)
}
defer func() {
if err := rows.Close(); err != nil {
err := rows.Close()
if err != nil {
t.Logf("failed to close rows: %v", err)
}
}()
t.Log("Files in snapshot_files:")
for rows.Next() {
var fileID string
if err := rows.Scan(&fileID); err != nil {
err := rows.Scan(&fileID)
if err != nil {
t.Fatal(err)
}
t.Logf(" - %s", fileID)
}
@@ -102,6 +116,7 @@ func TestOrphanedFileCleanupDebug(t *testing.T) {
if err != nil {
t.Fatal(err)
}
t.Logf("Files count before cleanup: %d", count)
// Run orphaned cleanup
@@ -109,6 +124,7 @@ func TestOrphanedFileCleanupDebug(t *testing.T) {
if err != nil {
t.Fatalf("failed to delete orphaned files: %v", err)
}
t.Log("Ran orphaned cleanup")
// Check files after cleanup
@@ -116,6 +132,7 @@ func TestOrphanedFileCleanupDebug(t *testing.T) {
if err != nil {
t.Fatal(err)
}
t.Logf("Files count after cleanup: %d", count)
// List remaining files
@@ -123,7 +140,9 @@ func TestOrphanedFileCleanupDebug(t *testing.T) {
if err != nil {
t.Fatal(err)
}
t.Log("Remaining files:")
for _, f := range files {
t.Logf(" - ID: %s, Path: %s", f.ID, f.Path)
}
@@ -133,10 +152,12 @@ func TestOrphanedFileCleanupDebug(t *testing.T) {
if err != nil {
t.Fatalf("error getting file: %v", err)
}
if orphanedFile != nil {
t.Error("orphaned file should have been deleted")
// Let's check why it wasn't deleted
var exists bool
err = db.conn.QueryRow(`
SELECT EXISTS(
SELECT 1 FROM snapshot_files
@@ -145,6 +166,7 @@ func TestOrphanedFileCleanupDebug(t *testing.T) {
if err != nil {
t.Fatal(err)
}
t.Logf("File1 exists in snapshot_files: %v", exists)
} else {
t.Log("Orphaned file was correctly deleted")
@@ -155,6 +177,7 @@ func TestOrphanedFileCleanupDebug(t *testing.T) {
if err != nil {
t.Fatalf("error getting file: %v", err)
}
if referencedFile == nil {
t.Error("referenced file should not have been deleted")
} else {

22
internal/database/repository_edge_cases_test.go
View File

@@ -98,6 +98,7 @@ func TestFileRepositoryEdgeCases(t *testing.T) {
if (err != nil) != tt.wantErr {
t.Errorf("Create() error = %v, wantErr %v", err, tt.wantErr)
}
if err != nil && tt.errMsg != "" && !strings.Contains(err.Error(), tt.errMsg) {
t.Errorf("Create() error = %v, want error containing %q", err, tt.errMsg)
}
@@ -136,6 +137,7 @@ func TestDuplicateHandling(t *testing.T) {
if err != nil {
t.Fatalf("failed to create file1: %v", err)
}
originalID := file1.ID
// Create with same path should update the existing record (UPSERT behavior)
@@ -190,6 +192,7 @@ func TestDuplicateHandling(t *testing.T) {
UID: 1000,
GID: 1000,
}
err := repos.Files.Create(ctx, nil, file)
if err != nil {
t.Fatal(err)
@@ -199,6 +202,7 @@ func TestDuplicateHandling(t *testing.T) {
ChunkHash: types.ChunkHash("test-chunk-dup"),
Size: 1024,
}
err = repos.Chunks.Create(ctx, nil, chunk)
if err != nil {
t.Fatal(err)
@@ -325,6 +329,7 @@ func TestLargeDatasets(t *testing.T) {
Hostname: "test-host",
StartedAt: time.Now(),
}
err := repos.Snapshots.Create(ctx, nil, snapshot)
if err != nil {
t.Fatal(err)
@@ -332,11 +337,12 @@ func TestLargeDatasets(t *testing.T) {
// Create many files
const fileCount = 1000
fileIDs := make([]types.FileID, fileCount)
t.Run("create many files", func(t *testing.T) {
start := time.Now()
for i := 0; i < fileCount; i++ {
for i := range fileCount {
file := &File{
Path: types.FilePath(fmt.Sprintf("/large/file%05d.txt", i)),
MTime: time.Now(),
@@ -345,10 +351,12 @@ func TestLargeDatasets(t *testing.T) {
UID: uint32(1000 + (i % 10)),
GID: uint32(1000 + (i % 10)),
}
err := repos.Files.Create(ctx, nil, file)
if err != nil {
t.Fatalf("failed to create file %d: %v", i, err)
}
fileIDs[i] = file.ID
// Add half to snapshot
@@ -359,29 +367,35 @@ func TestLargeDatasets(t *testing.T) {
}
}
}
t.Logf("Created %d files in %v", fileCount, time.Since(start))
})
// Test ListByPrefix performance
t.Run("list by prefix performance", func(t *testing.T) {
start := time.Now()
files, err := repos.Files.ListByPrefix(ctx, "/large/")
if err != nil {
t.Fatal(err)
}
if len(files) != fileCount {
t.Errorf("expected %d files, got %d", fileCount, len(files))
}
t.Logf("Listed %d files in %v", len(files), time.Since(start))
})
// Test orphaned cleanup performance
t.Run("orphaned cleanup performance", func(t *testing.T) {
start := time.Now()
err := repos.Files.DeleteOrphaned(ctx)
if err != nil {
t.Fatal(err)
}
t.Logf("Cleaned up orphaned files in %v", time.Since(start))
// Verify correct number remain
@@ -389,6 +403,7 @@ func TestLargeDatasets(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if len(files) != fileCount/2 {
t.Errorf("expected %d files after cleanup, got %d", fileCount/2, len(files))
}
@@ -409,6 +424,7 @@ func TestErrorPropagation(t *testing.T) {
if err != nil {
t.Errorf("GetByID should not return error for non-existent ID, got: %v", err)
}
if file != nil {
t.Error("expected nil file for non-existent ID")
}
@@ -420,6 +436,7 @@ func TestErrorPropagation(t *testing.T) {
if err != nil {
t.Errorf("GetByPath should not return error for non-existent path, got: %v", err)
}
if file != nil {
t.Error("expected nil file for non-existent path")
}
@@ -432,10 +449,12 @@ func TestErrorPropagation(t *testing.T) {
Idx: 0,
ChunkHash: types.ChunkHash("some-chunk"),
}
err := repos.FileChunks.Create(ctx, nil, fc)
if err == nil {
t.Error("expected error for invalid foreign key")
}
if !strings.Contains(err.Error(), "FOREIGN KEY") {
t.Errorf("expected foreign key error, got: %v", err)
}
@@ -475,6 +494,7 @@ func TestQueryInjection(t *testing.T) {
// Verify tables still exist
var count int
err := db.conn.QueryRow("SELECT COUNT(*) FROM files").Scan(&count)
if err != nil {
t.Fatal("files table was damaged by injection")

80
internal/database/snapshots.go
View File

@@ -4,6 +4,7 @@ import (
"context"
"database/sql"
"fmt"
"strings"
"time"
"sneak.berlin/go/vaultik/internal/types"
@@ -26,6 +27,7 @@ func (r *SnapshotRepository) Create(ctx context.Context, tx *sql.Tx, snapshot *S
`
var completedAt *int64
if snapshot.CompletedAt != nil {
ts := snapshot.CompletedAt.Unix()
completedAt = &ts
@@ -84,9 +86,11 @@ func (r *SnapshotRepository) UpdateCounts(ctx context.Context, tx *sql.Tx, snaps
func (r *SnapshotRepository) UpdateExtendedStats(ctx context.Context, tx *sql.Tx, snapshotID string, blobUncompressedSize int64, compressionLevel int, uploadDurationMs int64) error {
// Calculate compression ratio based on uncompressed vs compressed sizes
var compressionRatio float64
if blobUncompressedSize > 0 {
// Get current blob_size from DB to calculate ratio
var blobSize int64
queryGet := `SELECT blob_size FROM snapshots WHERE id = ?`
if tx != nil {
err := tx.QueryRowContext(ctx, queryGet, snapshotID).Scan(&blobSize)
@@ -99,6 +103,7 @@ func (r *SnapshotRepository) UpdateExtendedStats(ctx context.Context, tx *sql.Tx
return fmt.Errorf("getting blob size: %w", err)
}
}
compressionRatio = float64(blobSize) / float64(blobUncompressedSize)
} else {
compressionRatio = 1.0
@@ -124,6 +129,7 @@ func (r *SnapshotRepository) UpdateExtendedStats(ctx context.Context, tx *sql.Tx
if err != nil {
return fmt.Errorf("updating extended stats: %w", err)
}
return nil
}
@@ -136,9 +142,11 @@ func (r *SnapshotRepository) GetByID(ctx context.Context, snapshotID string) (*S
WHERE id = ?
`
var snapshot Snapshot
var startedAtUnix int64
var completedAtUnix *int64
var (
snapshot Snapshot
startedAtUnix int64
completedAtUnix *int64
)
err := r.db.conn.QueryRowContext(ctx, query, snapshotID).Scan(
&snapshot.ID,
@@ -162,6 +170,7 @@ func (r *SnapshotRepository) GetByID(ctx context.Context, snapshotID string) (*S
if err == sql.ErrNoRows {
return nil, nil
}
if err != nil {
return nil, fmt.Errorf("querying snapshot: %w", err)
}
@@ -190,10 +199,13 @@ func (r *SnapshotRepository) ListRecent(ctx context.Context, limit int) ([]*Snap
defer CloseRows(rows)
var snapshots []*Snapshot
for rows.Next() {
var snapshot Snapshot
var startedAtUnix int64
var completedAtUnix *int64
var (
snapshot Snapshot
startedAtUnix int64
completedAtUnix *int64
)
err := rows.Scan(
&snapshot.ID,
@@ -301,28 +313,35 @@ func (r *SnapshotRepository) AddFilesByIDBatch(ctx context.Context, tx *sql.Tx,
const batchSize = 400
for i := 0; i < len(fileIDs); i += batchSize {
end := i + batchSize
if end > len(fileIDs) {
end = len(fileIDs)
}
end := min(i+batchSize, len(fileIDs))
batch := fileIDs[i:end]
query := "INSERT OR IGNORE INTO snapshot_files (snapshot_id, file_id) VALUES "
args := make([]interface{}, 0, len(batch)*2)
args := make([]any, 0, len(batch)*2)
var querySb312 strings.Builder
for j, fileID := range batch {
if j > 0 {
query += ", "
querySb312.WriteString(", ")
}
query += "(?, ?)"
querySb312.WriteString("(?, ?)")
args = append(args, snapshotID, fileID.String())
}
query += querySb312.String()
var err error
if tx != nil {
_, err = tx.ExecContext(ctx, query, args...)
} else {
_, err = r.db.ExecWithLog(ctx, query, args...)
}
if err != nil {
return fmt.Errorf("batch adding files to snapshot: %w", err)
}
@@ -353,18 +372,22 @@ func (r *SnapshotRepository) PopulateReferencedBlobs(ctx context.Context, tx *sq
AND blobs.blob_hash IS NOT NULL
`
var result sql.Result
var err error
var (
result sql.Result
err error
)
if tx != nil {
result, err = tx.ExecContext(ctx, query, snapshotID, snapshotID)
} else {
result, err = r.db.ExecWithLog(ctx, query, snapshotID, snapshotID)
}
if err != nil {
return 0, fmt.Errorf("populating referenced blobs: %w", err)
}
n, _ := result.RowsAffected()
return n, nil
}
@@ -405,11 +428,15 @@ func (r *SnapshotRepository) GetBlobHashes(ctx context.Context, snapshotID strin
defer CloseRows(rows)
var blobs []string
for rows.Next() {
var blobHash string
if err := rows.Scan(&blobHash); err != nil {
err := rows.Scan(&blobHash)
if err != nil {
return nil, fmt.Errorf("scanning blob hash: %w", err)
}
blobs = append(blobs, blobHash)
}
@@ -426,6 +453,7 @@ func (r *SnapshotRepository) GetSnapshotTotalCompressedSize(ctx context.Context,
`
var totalSize int64
err := r.db.conn.QueryRowContext(ctx, query, snapshotID).Scan(&totalSize)
if err != nil {
return 0, fmt.Errorf("querying total compressed size: %w", err)
@@ -449,6 +477,7 @@ func (r *SnapshotRepository) GetSnapshotUncompressedChunkSize(ctx context.Contex
`
var totalSize int64
err := r.db.conn.QueryRowContext(ctx, query, snapshotID).Scan(&totalSize)
if err != nil {
return 0, fmt.Errorf("querying uncompressed chunk size: %w", err)
@@ -485,6 +514,7 @@ func (r *SnapshotRepository) GetSnapshotNewChunkSize(ctx context.Context, snapsh
`
var totalSize int64
err := r.db.conn.QueryRowContext(ctx, query, snapshotID, snapshotID, snapshotID).Scan(&totalSize)
if err != nil {
return 0, fmt.Errorf("querying new chunk size: %w", err)
@@ -509,10 +539,13 @@ func (r *SnapshotRepository) GetIncompleteSnapshots(ctx context.Context) ([]*Sna
defer CloseRows(rows)
var snapshots []*Snapshot
for rows.Next() {
var snapshot Snapshot
var startedAtUnix int64
var completedAtUnix *int64
var (
snapshot Snapshot
startedAtUnix int64
completedAtUnix *int64
)
err := rows.Scan(
&snapshot.ID,
@@ -560,10 +593,13 @@ func (r *SnapshotRepository) GetIncompleteByHostname(ctx context.Context, hostna
defer CloseRows(rows)
var snapshots []*Snapshot
for rows.Next() {
var snapshot Snapshot
var startedAtUnix int64
var completedAtUnix *int64
var (
snapshot Snapshot
startedAtUnix int64
completedAtUnix *int64
)
err := rows.Scan(
&snapshot.ID,

15
internal/database/snapshots_test.go
View File

@@ -52,15 +52,19 @@ func TestSnapshotRepository(t *testing.T) {
if err != nil {
t.Fatalf("failed to get snapshot: %v", err)
}
if retrieved == nil {
t.Fatal("expected snapshot, got nil")
}
if retrieved.ID != snapshot.ID {
t.Errorf("ID mismatch: got %s, want %s", retrieved.ID, snapshot.ID)
}
if retrieved.Hostname != snapshot.Hostname {
t.Errorf("hostname mismatch: got %s, want %s", retrieved.Hostname, snapshot.Hostname)
}
if retrieved.FileCount != snapshot.FileCount {
t.Errorf("file count mismatch: got %d, want %d", retrieved.FileCount, snapshot.FileCount)
}
@@ -75,21 +79,27 @@ func TestSnapshotRepository(t *testing.T) {
if err != nil {
t.Fatalf("failed to get updated snapshot: %v", err)
}
if retrieved.FileCount != 200 {
t.Errorf("file count not updated: got %d, want %d", retrieved.FileCount, 200)
}
if retrieved.ChunkCount != 1000 {
t.Errorf("chunk count not updated: got %d, want %d", retrieved.ChunkCount, 1000)
}
if retrieved.BlobCount != 20 {
t.Errorf("blob count not updated: got %d, want %d", retrieved.BlobCount, 20)
}
if retrieved.TotalSize != twoHundredMebibytes {
t.Errorf("total size not updated: got %d, want %d", retrieved.TotalSize, twoHundredMebibytes)
}
if retrieved.BlobSize != sixtyMebibytes {
t.Errorf("blob size not updated: got %d, want %d", retrieved.BlobSize, sixtyMebibytes)
}
expectedRatio := compressionRatioPoint3 // 0.3
if math.Abs(retrieved.CompressionRatio-expectedRatio) > 0.001 {
t.Errorf("compression ratio not updated: got %f, want %f", retrieved.CompressionRatio, expectedRatio)
@@ -108,6 +118,7 @@ func TestSnapshotRepository(t *testing.T) {
ChunkCount: int64(500 * i),
BlobCount: int64(10 * i),
}
err := repo.Create(ctx, nil, s)
if err != nil {
t.Fatalf("failed to create snapshot %d: %v", i, err)
@@ -119,12 +130,13 @@ func TestSnapshotRepository(t *testing.T) {
if err != nil {
t.Fatalf("failed to list recent snapshots: %v", err)
}
if len(recent) != 3 {
t.Errorf("expected 3 recent snapshots, got %d", len(recent))
}
// Verify order (most recent first)
for i := 0; i < len(recent)-1; i++ {
for i := range len(recent) - 1 {
if recent[i].StartedAt.Before(recent[i+1].StartedAt) {
t.Error("snapshots not in descending order")
}
@@ -143,6 +155,7 @@ func TestSnapshotRepositoryNotFound(t *testing.T) {
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if snapshot != nil {
t.Error("expected nil for non-existent snapshot")
}

15
internal/database/uploads.go
View File

@@ -53,6 +53,7 @@ func (r *UploadRepository) GetByBlobHash(ctx context.Context, blobHash string) (
`
var upload Upload
err := r.conn.QueryRowContext(ctx, query, blobHash).Scan(
&upload.BlobHash,
&upload.UploadedAt,
@@ -63,6 +64,7 @@ func (r *UploadRepository) GetByBlobHash(ctx context.Context, blobHash string) (
if err == sql.ErrNoRows {
return nil, nil
}
if err != nil {
return nil, err
}
@@ -84,17 +86,22 @@ func (r *UploadRepository) GetRecentUploads(ctx context.Context, limit int) ([]*
return nil, err
}
defer func() {
if err := rows.Close(); err != nil {
err := rows.Close()
if err != nil {
log.Error("failed to close rows", "error", err)
}
}()
var uploads []*Upload
for rows.Next() {
var upload Upload
if err := rows.Scan(&upload.BlobHash, &upload.UploadedAt, &upload.Size, &upload.DurationMs); err != nil {
err := rows.Scan(&upload.BlobHash, &upload.UploadedAt, &upload.Size, &upload.DurationMs)
if err != nil {
return nil, err
}
uploads = append(uploads, &upload)
}
@@ -115,6 +122,7 @@ func (r *UploadRepository) GetUploadStats(ctx context.Context, since time.Time)
`
var stats UploadStats
err := r.conn.QueryRowContext(ctx, query, since).Scan(
&stats.Count,
&stats.TotalSize,
@@ -138,10 +146,13 @@ type UploadStats struct {
// GetCountBySnapshot returns the count of uploads for a specific snapshot
func (r *UploadRepository) GetCountBySnapshot(ctx context.Context, snapshotID string) (int64, error) {
query := `SELECT COUNT(*) FROM uploads WHERE snapshot_id = ?`
var count int64
err := r.conn.QueryRowContext(ctx, query, snapshotID).Scan(&count)
if err != nil {
return 0, err
}
return count, nil
}

1
internal/globals/globals.go
View File

@@ -50,5 +50,6 @@ func (g *Globals) ShortCommit() string {
if len(g.Commit) > 12 {
return g.Commit[:12]
}
return g.Commit
}

3
internal/log/log.go
View File

@@ -84,6 +84,7 @@ func getCaller(skip int) string {
if !ok {
return "unknown"
}
return fmt.Sprintf("%s:%d", filepath.Base(file), line)
}
@@ -94,6 +95,7 @@ func Fatal(msg string, args ...any) {
args = append(args, "caller", getCaller(2))
logger.Error(msg, args...)
}
os.Exit(1)
}
@@ -172,6 +174,7 @@ func With(args ...any) *slog.Logger {
if logger != nil {
return logger.With(args...)
}
return slog.Default()
}

8
internal/log/tty_handler.go
View File

@@ -33,6 +33,7 @@ func NewTTYHandler(out io.Writer, opts *slog.HandlerOptions) *TTYHandler {
if opts == nil {
opts = &slog.HandlerOptions{}
}
return &TTYHandler{
out: out,
opts: *opts,
@@ -54,7 +55,9 @@ func (h *TTYHandler) Handle(_ context.Context, r slog.Record) error {
// Level and color
level := r.Level.String()
var levelColor string
switch r.Level {
case slog.LevelDebug:
levelColor = colorGray
@@ -96,10 +99,12 @@ func (h *TTYHandler) Handle(_ context.Context, r slog.Record) error {
_, _ = fmt.Fprintf(h.out, " %s%s%s=%s%s%s",
colorCyan, a.Key, colorReset,
colorBlue, value, colorReset)
return true
})
_, _ = fmt.Fprintln(h.out)
return nil
}
@@ -122,6 +127,7 @@ func formatDuration(d time.Duration) string {
} else if d < time.Minute {
return fmt.Sprintf("%.1fs", d.Seconds())
}
return d.String()
}
@@ -131,10 +137,12 @@ func formatBytes(b int64) string {
if b < unit {
return fmt.Sprintf("%d B", b)
}
div, exp := int64(unit), 0
for n := b / unit; n >= unit; n /= unit {
div *= unit
exp++
}
return fmt.Sprintf("%.1f %cB", float64(b)/float64(div), "KMGTPE"[exp])
}

2
internal/pidlock/pidlock.go
View File

@@ -77,6 +77,7 @@ func (l *Lock) Release() error {
}
l.path = "" // Prevent double-release
return nil
}
@@ -104,5 +105,6 @@ func isProcessRunning(pid int) bool {
// On Unix, FindProcess always succeeds. We need to send signal 0 to check.
err = process.Signal(syscall.Signal(0))
return err == nil
}

4
internal/pidlock/pidlock_test.go
View File

@@ -40,6 +40,7 @@ func TestAcquireBlocksSecondInstance(t *testing.T) {
// Acquire first lock
lock1, err := Acquire(tmpDir)
require.NoError(t, err)
require.NotNil(t, lock1)
defer func() { _ = lock1.Release() }()
@@ -61,6 +62,7 @@ func TestAcquireWithStaleLock(t *testing.T) {
// Should be able to acquire lock (stale lock is cleaned up)
lock, err := Acquire(tmpDir)
require.NoError(t, err)
require.NotNil(t, lock)
defer func() { _ = lock.Release() }()
@@ -88,6 +90,7 @@ func TestReleaseIsIdempotent(t *testing.T) {
func TestReleaseNilLock(t *testing.T) {
var lock *Lock
err := lock.Release()
assert.NoError(t, err)
}
@@ -98,6 +101,7 @@ func TestAcquireCreatesDirectory(t *testing.T) {
lock, err := Acquire(nestedDir)
require.NoError(t, err)
require.NotNil(t, lock)
defer func() { _ = lock.Release() }()

26
internal/s3/client.go
View File

@@ -42,7 +42,7 @@ type Config struct {
// Used to suppress SDK warnings about checksums.
type nopLogger struct{}
func (nopLogger) Logf(classification logging.Classification, format string, v ...interface{}) {}
func (nopLogger) Logf(classification logging.Classification, format string, v ...any) {}
// NewClient creates a new S3 client with the provided configuration.
// It establishes a connection to the S3-compatible storage service and
@@ -92,6 +92,7 @@ func (c *Client) PutObject(ctx context.Context, key string, data io.Reader) erro
Key: aws.String(fullKey),
Body: data,
})
return err
}
@@ -137,6 +138,7 @@ func (c *Client) PutObjectWithProgress(ctx context.Context, key string, data io.
// close the returned reader when done to avoid resource leaks.
func (c *Client) GetObject(ctx context.Context, key string) (io.ReadCloser, error) {
fullKey := c.prefix + key
result, err := c.s3Client.GetObject(ctx, &s3.GetObjectInput{
Bucket: aws.String(c.bucket),
Key: aws.String(fullKey),
@@ -144,6 +146,7 @@ func (c *Client) GetObject(ctx context.Context, key string) (io.ReadCloser, erro
if err != nil {
return nil, err
}
return result.Body, nil
}
@@ -156,6 +159,7 @@ func (c *Client) DeleteObject(ctx context.Context, key string) error {
Bucket: aws.String(c.bucket),
Key: aws.String(fullKey),
})
return err
}
@@ -168,6 +172,7 @@ func (c *Client) ListObjects(ctx context.Context, prefix string) ([]string, erro
fullPrefix := c.prefix + prefix
var keys []string
paginator := s3.NewListObjectsV2Paginator(c.s3Client, &s3.ListObjectsV2Input{
Bucket: aws.String(c.bucket),
Prefix: aws.String(fullPrefix),
@@ -186,6 +191,7 @@ func (c *Client) ListObjects(ctx context.Context, prefix string) ([]string, erro
if len(key) > len(c.prefix) {
key = key[len(c.prefix):]
}
keys = append(keys, key)
}
}
@@ -200,18 +206,23 @@ func (c *Client) ListObjects(ctx context.Context, prefix string) ([]string, erro
// Note: This method returns false for any error, not just "not found".
func (c *Client) HeadObject(ctx context.Context, key string) (bool, error) {
fullKey := c.prefix + key
_, err := c.s3Client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: aws.String(c.bucket),
Key: aws.String(fullKey),
})
if err != nil {
var notFound *s3types.NotFound
var noSuchKey *s3types.NoSuchKey
var (
notFound *s3types.NotFound
noSuchKey *s3types.NoSuchKey
)
if errors.As(err, &notFound) || errors.As(err, &noSuchKey) {
return false, nil
}
return false, err
}
return true, nil
}
@@ -247,6 +258,7 @@ func (c *Client) ListObjectsStream(ctx context.Context, prefix string, recursive
page, err := paginator.NextPage(ctx)
if err != nil {
ch <- ObjectInfo{Err: err}
return
}
@@ -257,6 +269,7 @@ func (c *Client) ListObjectsStream(ctx context.Context, prefix string, recursive
if len(key) > len(c.prefix) {
key = key[len(c.prefix):]
}
ch <- ObjectInfo{
Key: key,
Size: *obj.Size,
@@ -275,6 +288,7 @@ func (c *Client) ListObjectsStream(ctx context.Context, prefix string, recursive
// Returns an error if the object doesn't exist or if the operation fails.
func (c *Client) StatObject(ctx context.Context, key string) (*ObjectInfo, error) {
fullKey := c.prefix + key
result, err := c.s3Client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: aws.String(c.bucket),
Key: aws.String(fullKey),
@@ -313,6 +327,7 @@ func (c *Client) Endpoint() string {
if c.endpoint == "" {
return "s3.amazonaws.com"
}
return c.endpoint
}
@@ -329,11 +344,14 @@ func (pr *progressReader) Read(p []byte) (int, error) {
n, err := pr.reader.Read(p)
if n > 0 {
atomic.AddInt64(&pr.read, int64(n))
if pr.callback != nil {
if callbackErr := pr.callback(atomic.LoadInt64(&pr.read)); callbackErr != nil {
callbackErr := pr.callback(atomic.LoadInt64(&pr.read))
if callbackErr != nil {
return n, callbackErr
}
}
}
return n, err
}

11
internal/s3/client_test.go
View File

@@ -12,7 +12,8 @@ import (
func TestClient(t *testing.T) {
ts := NewTestServer(t)
defer func() {
if err := ts.Cleanup(); err != nil {
err := ts.Cleanup()
if err != nil {
t.Errorf("cleanup failed: %v", err)
}
}()
@@ -35,6 +36,7 @@ func TestClient(t *testing.T) {
// Test PutObject
testKey := "foo/bar.txt"
testData := []byte("test data")
err = client.PutObject(ctx, testKey, bytes.NewReader(testData))
if err != nil {
t.Fatalf("failed to put object: %v", err)
@@ -46,7 +48,8 @@ func TestClient(t *testing.T) {
t.Fatalf("failed to get object: %v", err)
}
defer func() {
if err := reader.Close(); err != nil {
err := reader.Close()
if err != nil {
t.Errorf("failed to close reader: %v", err)
}
}()
@@ -65,6 +68,7 @@ func TestClient(t *testing.T) {
if err != nil {
t.Fatalf("failed to head object: %v", err)
}
if !exists {
t.Error("expected object to exist")
}
@@ -74,9 +78,11 @@ func TestClient(t *testing.T) {
if err != nil {
t.Fatalf("failed to list objects: %v", err)
}
if len(keys) != 1 {
t.Errorf("expected 1 key, got %d", len(keys))
}
if keys[0] != testKey {
t.Errorf("unexpected key: got %s, want %s", keys[0], testKey)
}
@@ -92,6 +98,7 @@ func TestClient(t *testing.T) {
if err != nil {
t.Fatalf("failed to head object after deletion: %v", err)
}
if exists {
t.Error("expected object to not exist after deletion")
}

20
internal/s3/s3_test.go
View File

@@ -57,7 +57,8 @@ func NewTestServer(t *testing.T) *TestServer {
// Start server in background
go func() {
if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
err := server.ListenAndServe()
if err != nil && err != http.ErrServerClosed {
t.Logf("test server error: %v", err)
}
}()
@@ -77,7 +78,7 @@ func NewTestServer(t *testing.T) *TestServer {
"",
)),
config.WithClientLogMode(aws.LogRetries|aws.LogRequestWithBody|aws.LogResponseWithBody),
config.WithLogger(logging.LoggerFunc(func(classification logging.Classification, format string, v ...interface{}) {
config.WithLogger(logging.LoggerFunc(func(classification logging.Classification, format string, v ...any) {
// Capture logs to buffer instead of stdout
fmt.Fprintf(logBuf, "SDK %s %s %s\n",
time.Now().Format("2006/01/02 15:04:05"),
@@ -125,7 +126,8 @@ func (ts *TestServer) Cleanup() error {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if err := ts.server.Shutdown(ctx); err != nil {
err := ts.server.Shutdown(ctx)
if err != nil {
return err
}
@@ -141,7 +143,8 @@ func (ts *TestServer) Client() *s3.Client {
func TestBasicS3Operations(t *testing.T) {
ts := NewTestServer(t)
defer func() {
if err := ts.Cleanup(); err != nil {
err := ts.Cleanup()
if err != nil {
t.Errorf("cleanup failed: %v", err)
}
}()
@@ -172,7 +175,8 @@ func TestBasicS3Operations(t *testing.T) {
t.Fatalf("failed to get object: %v", err)
}
defer func() {
if err := result.Body.Close(); err != nil {
err := result.Body.Close()
if err != nil {
t.Errorf("failed to close body: %v", err)
}
}()
@@ -192,7 +196,8 @@ func TestBasicS3Operations(t *testing.T) {
func TestBlobOperations(t *testing.T) {
ts := NewTestServer(t)
defer func() {
if err := ts.Cleanup(); err != nil {
err := ts.Cleanup()
if err != nil {
t.Errorf("cleanup failed: %v", err)
}
}()
@@ -255,7 +260,8 @@ func TestBlobOperations(t *testing.T) {
func TestMetadataOperations(t *testing.T) {
ts := NewTestServer(t)
defer func() {
if err := ts.Cleanup(); err != nil {
err := ts.Cleanup()
if err != nil {
t.Errorf("cleanup failed: %v", err)
}
}()

41
internal/snapshot/backup_test.go
View File

@@ -4,6 +4,8 @@ import (
"context"
"crypto/sha256"
"database/sql"
"encoding/hex"
"errors"
"fmt"
"io"
"io/fs"
@@ -30,6 +32,7 @@ func NewMockS3Client() *MockS3Client {
func (m *MockS3Client) PutBlob(ctx context.Context, hash string, data []byte) error {
m.storage[hash] = data
return nil
}
@@ -38,11 +41,13 @@ func (m *MockS3Client) GetBlob(ctx context.Context, hash string) ([]byte, error)
if !ok {
return nil, fmt.Errorf("blob not found: %s", hash)
}
return data, nil
}
func (m *MockS3Client) BlobExists(ctx context.Context, hash string) (bool, error) {
_, ok := m.storage[hash]
return ok, nil
}
@@ -81,12 +86,15 @@ func TestBackupWithInMemoryFS(t *testing.T) {
// Initialize the database
ctx := context.Background()
db, err := database.New(ctx, dbPath)
if err != nil {
t.Fatalf("Failed to create database: %v", err)
}
defer func() {
if err := db.Close(); err != nil {
err := db.Close()
if err != nil {
t.Logf("Failed to close database: %v", err)
}
}()
@@ -142,12 +150,14 @@ func TestBackupWithInMemoryFS(t *testing.T) {
if !expectedFiles[file.Path.String()] {
t.Errorf("Unexpected file in database: %s", file.Path)
}
delete(expectedFiles, file.Path.String())
// Verify file metadata
fsFile := testFS[file.Path.String()]
if fsFile == nil {
t.Errorf("File %s not found in test filesystem", file.Path)
continue
}
@@ -187,6 +197,7 @@ func TestBackupWithInMemoryFS(t *testing.T) {
if err != nil {
t.Fatalf("Failed to get blob hashes: %v", err)
}
if len(blobHashes) == 0 {
t.Error("Expected at least one blob to be created")
}
@@ -197,6 +208,7 @@ func TestBackupWithInMemoryFS(t *testing.T) {
if err != nil {
t.Errorf("Failed to check blob %s: %v", blobHash, err)
}
if !exists {
t.Errorf("Blob %s not found in S3", blobHash)
}
@@ -229,12 +241,15 @@ func TestBackupDeduplication(t *testing.T) {
// Initialize the database
ctx := context.Background()
db, err := database.New(ctx, dbPath)
if err != nil {
t.Fatalf("Failed to create database: %v", err)
}
defer func() {
if err := db.Close(); err != nil {
err := db.Close()
if err != nil {
t.Logf("Failed to close database: %v", err)
}
}()
@@ -348,6 +363,7 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
UID: 1000, // Default UID for test
GID: 1000, // Default GID for test
}
err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
return b.repos.Files.Create(ctx, tx, file)
})
@@ -364,7 +380,8 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
return err
}
defer func() {
if err := f.Close(); err != nil {
err := f.Close()
if err != nil {
// Log but don't fail since we're already in an error path potentially
fmt.Fprintf(os.Stderr, "Failed to close file: %v\n", err)
}
@@ -376,9 +393,10 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
for {
n, err := f.Read(buffer)
if err != nil && err != io.EOF {
if err != nil && !errors.Is(err, io.EOF) {
return err
}
if n == 0 {
break
}
@@ -395,11 +413,13 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
ChunkHash: types.ChunkHash(chunkHash),
Size: int64(n),
}
return b.repos.Chunks.Create(ctx, tx, chunk)
})
if err != nil {
return err
}
processedChunks[chunkHash] = true
}
@@ -410,6 +430,7 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
Idx: chunkIndex,
ChunkHash: types.ChunkHash(chunkHash),
}
return b.repos.FileChunks.Create(ctx, tx, fileChunk)
})
if err != nil {
@@ -424,6 +445,7 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
FileOffset: int64(chunkIndex * defaultChunkSize),
Length: int64(n),
}
return b.repos.ChunkFiles.Create(ctx, tx, chunkFile)
})
if err != nil {
@@ -435,7 +457,6 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
return nil
})
if err != nil {
return "", err
}
@@ -464,12 +485,14 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
// Create blob entry in a short transaction
blobID := types.NewBlobID()
err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
blob := &database.Blob{
ID: blobID,
Hash: types.BlobHash(blobHash),
CreatedTS: time.Now(),
}
return b.repos.Blobs.Create(ctx, tx, blob)
})
if err != nil {
@@ -487,6 +510,7 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
Offset: 0,
Length: chunk.Size,
}
return b.repos.BlobChunks.Create(ctx, tx, blobChunk)
})
if err != nil {
@@ -506,7 +530,6 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
return b.repos.Snapshots.UpdateCounts(ctx, tx, snapshotID, fileCount, chunkCount, blobCount, totalSize, blobSize)
})
if err != nil {
return "", err
}
@@ -517,16 +540,18 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
func calculateHash(data []byte) string {
h := sha256.New()
h.Write(data)
return fmt.Sprintf("%x", h.Sum(nil))
return hex.EncodeToString(h.Sum(nil))
}
func generateLargeFileContent(size int) []byte {
data := make([]byte, size)
// Fill with pattern that changes every chunk to avoid deduplication
for i := 0; i < size; i++ {
for i := range size {
chunkNum := i / defaultChunkSize
data[i] = byte((i + chunkNum) % 256)
}
return data
}

34
internal/snapshot/exclude_test.go
View File

@@ -63,6 +63,7 @@ func setupExcludeTestFS(t *testing.T) afero.Fs {
}
testTime := time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC)
for path, content := range files {
dir := filepath.Dir(path)
err := fs.MkdirAll(dir, 0755)
@@ -107,6 +108,7 @@ func createTestScanner(t *testing.T, fs afero.Fs, excludePatterns []string) (*sn
func createSnapshotRecord(t *testing.T, ctx context.Context, repos *database.Repositories, snapshotID string) {
t.Helper()
err := repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
snap := &database.Snapshot{
ID: types.SnapshotID(snapshotID),
@@ -121,6 +123,7 @@ func createSnapshotRecord(t *testing.T, ctx context.Context, repos *database.Rep
BlobSize: 0,
CompressionRatio: 1.0,
}
return repos.Snapshots.Create(ctx, tx, snap)
})
require.NoError(t, err)
@@ -128,8 +131,10 @@ func createSnapshotRecord(t *testing.T, ctx context.Context, repos *database.Rep
func TestExcludePatterns_ExcludeGitDirectory(t *testing.T) {
fs := setupExcludeTestFS(t)
scanner, repos, cleanup := createTestScanner(t, fs, []string{".git"})
defer cleanup()
require.NotNil(t, scanner)
ctx := context.Background()
@@ -148,8 +153,10 @@ func TestExcludePatterns_ExcludeGitDirectory(t *testing.T) {
func TestExcludePatterns_ExcludeByExtension(t *testing.T) {
fs := setupExcludeTestFS(t)
scanner, repos, cleanup := createTestScanner(t, fs, []string{"*.log"})
defer cleanup()
require.NotNil(t, scanner)
ctx := context.Background()
@@ -165,8 +172,10 @@ func TestExcludePatterns_ExcludeByExtension(t *testing.T) {
func TestExcludePatterns_ExcludeNodeModules(t *testing.T) {
fs := setupExcludeTestFS(t)
scanner, repos, cleanup := createTestScanner(t, fs, []string{"node_modules"})
defer cleanup()
require.NotNil(t, scanner)
ctx := context.Background()
@@ -182,8 +191,10 @@ func TestExcludePatterns_ExcludeNodeModules(t *testing.T) {
func TestExcludePatterns_MultiplePatterns(t *testing.T) {
fs := setupExcludeTestFS(t)
scanner, repos, cleanup := createTestScanner(t, fs, []string{".git", "node_modules", "*.log", ".DS_Store", "thumbs.db", "cache", "build"})
defer cleanup()
require.NotNil(t, scanner)
ctx := context.Background()
@@ -199,8 +210,10 @@ func TestExcludePatterns_MultiplePatterns(t *testing.T) {
func TestExcludePatterns_NoExclusions(t *testing.T) {
fs := setupExcludeTestFS(t)
scanner, repos, cleanup := createTestScanner(t, fs, []string{})
defer cleanup()
require.NotNil(t, scanner)
ctx := context.Background()
@@ -215,8 +228,10 @@ func TestExcludePatterns_NoExclusions(t *testing.T) {
func TestExcludePatterns_ExcludeHiddenFiles(t *testing.T) {
fs := setupExcludeTestFS(t)
scanner, repos, cleanup := createTestScanner(t, fs, []string{".*"})
defer cleanup()
require.NotNil(t, scanner)
ctx := context.Background()
@@ -232,8 +247,10 @@ func TestExcludePatterns_ExcludeHiddenFiles(t *testing.T) {
func TestExcludePatterns_DoubleStarGlob(t *testing.T) {
fs := setupExcludeTestFS(t)
scanner, repos, cleanup := createTestScanner(t, fs, []string{"**/*.pack"})
defer cleanup()
require.NotNil(t, scanner)
ctx := context.Background()
@@ -249,8 +266,10 @@ func TestExcludePatterns_DoubleStarGlob(t *testing.T) {
func TestExcludePatterns_ExactFileName(t *testing.T) {
fs := setupExcludeTestFS(t)
scanner, repos, cleanup := createTestScanner(t, fs, []string{"thumbs.db", ".DS_Store"})
defer cleanup()
require.NotNil(t, scanner)
ctx := context.Background()
@@ -267,8 +286,10 @@ func TestExcludePatterns_ExactFileName(t *testing.T) {
func TestExcludePatterns_CaseSensitive(t *testing.T) {
// Pattern matching should be case-sensitive
fs := setupExcludeTestFS(t)
scanner, repos, cleanup := createTestScanner(t, fs, []string{"THUMBS.DB"})
defer cleanup()
require.NotNil(t, scanner)
ctx := context.Background()
@@ -287,6 +308,7 @@ func TestExcludePatterns_DirectoryWithTrailingSlash(t *testing.T) {
// Some users might add trailing slashes to directory patterns
scanner, repos, cleanup := createTestScanner(t, fs, []string{"cache/", "build/"})
defer cleanup()
require.NotNil(t, scanner)
ctx := context.Background()
@@ -305,6 +327,7 @@ func TestExcludePatterns_PatternInSubdirectory(t *testing.T) {
// Exclude .hidden file specifically in src directory
scanner, repos, cleanup := createTestScanner(t, fs, []string{"src/.hidden"})
defer cleanup()
require.NotNil(t, scanner)
ctx := context.Background()
@@ -343,6 +366,7 @@ func setupAnchoredTestFS(t *testing.T) afero.Fs {
}
testTime := time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC)
for path, content := range files {
dir := filepath.Dir(path)
err := fs.MkdirAll(dir, 0755)
@@ -359,8 +383,10 @@ func setupAnchoredTestFS(t *testing.T) afero.Fs {
func TestExcludePatterns_AnchoredPattern(t *testing.T) {
// Pattern starting with / should only match from root of source dir
fs := setupAnchoredTestFS(t)
scanner, repos, cleanup := createTestScanner(t, fs, []string{"/projectname"})
defer cleanup()
require.NotNil(t, scanner)
ctx := context.Background()
@@ -378,8 +404,10 @@ func TestExcludePatterns_AnchoredPattern(t *testing.T) {
func TestExcludePatterns_UnanchoredPattern(t *testing.T) {
// Pattern without leading / should match anywhere in path
fs := setupAnchoredTestFS(t)
scanner, repos, cleanup := createTestScanner(t, fs, []string{"projectname"})
defer cleanup()
require.NotNil(t, scanner)
ctx := context.Background()
@@ -398,8 +426,10 @@ func TestExcludePatterns_UnanchoredPattern(t *testing.T) {
func TestExcludePatterns_AnchoredPatternWithGlob(t *testing.T) {
// Anchored pattern with glob
fs := setupAnchoredTestFS(t)
scanner, repos, cleanup := createTestScanner(t, fs, []string{"/src/*.go"})
defer cleanup()
require.NotNil(t, scanner)
ctx := context.Background()
@@ -416,8 +446,10 @@ func TestExcludePatterns_AnchoredPatternWithGlob(t *testing.T) {
func TestExcludePatterns_AnchoredPatternFile(t *testing.T) {
// Anchored pattern for exact file at root
fs := setupAnchoredTestFS(t)
scanner, repos, cleanup := createTestScanner(t, fs, []string{"/file.txt"})
defer cleanup()
require.NotNil(t, scanner)
ctx := context.Background()
@@ -435,8 +467,10 @@ func TestExcludePatterns_AnchoredPatternFile(t *testing.T) {
func TestExcludePatterns_UnanchoredPatternFile(t *testing.T) {
// Unanchored pattern for file should match anywhere
fs := setupAnchoredTestFS(t)
scanner, repos, cleanup := createTestScanner(t, fs, []string{"file.txt"})
defer cleanup()
require.NotNil(t, scanner)
ctx := context.Background()

15
internal/snapshot/file_change_test.go
View File

@@ -30,9 +30,11 @@ func TestFileContentChange(t *testing.T) {
// Create test database
db, err := database.NewTestDB()
require.NoError(t, err)
defer func() {
if err := db.Close(); err != nil {
err := db.Close()
if err != nil {
t.Errorf("failed to close database: %v", err)
}
}()
@@ -59,6 +61,7 @@ func TestFileContentChange(t *testing.T) {
VaultikVersion: "test",
StartedAt: time.Now(),
}
return repos.Snapshots.Create(ctx, tx, snapshot)
})
require.NoError(t, err)
@@ -81,6 +84,7 @@ func TestFileContentChange(t *testing.T) {
// Modify the file
time.Sleep(10 * time.Millisecond) // Ensure mtime changes
err = afero.WriteFile(fs, "/test.txt", []byte("Modified content with different data"), 0644)
require.NoError(t, err)
@@ -93,6 +97,7 @@ func TestFileContentChange(t *testing.T) {
VaultikVersion: "test",
StartedAt: time.Now(),
}
return repos.Snapshots.Create(ctx, tx, snapshot)
})
require.NoError(t, err)
@@ -130,6 +135,7 @@ func TestFileContentChange(t *testing.T) {
// Verify that chunk_files for old chunk no longer references this file
oldChunkFiles, err := repos.ChunkFiles.GetByChunkHash(ctx, oldChunkHash)
require.NoError(t, err)
for _, cf := range oldChunkFiles {
file, err := repos.Files.GetByID(ctx, cf.FileID)
require.NoError(t, err)
@@ -159,9 +165,11 @@ func TestMultipleFileChanges(t *testing.T) {
// Create test database
db, err := database.NewTestDB()
require.NoError(t, err)
defer func() {
if err := db.Close(); err != nil {
err := db.Close()
if err != nil {
t.Errorf("failed to close database: %v", err)
}
}()
@@ -188,6 +196,7 @@ func TestMultipleFileChanges(t *testing.T) {
VaultikVersion: "test",
StartedAt: time.Now(),
}
return repos.Snapshots.Create(ctx, tx, snapshot)
})
require.NoError(t, err)
@@ -200,6 +209,7 @@ func TestMultipleFileChanges(t *testing.T) {
// Modify two files
time.Sleep(10 * time.Millisecond) // Ensure mtime changes
err = afero.WriteFile(fs, "/file1.txt", []byte("Modified content 1"), 0644)
require.NoError(t, err)
err = afero.WriteFile(fs, "/file3.txt", []byte("Modified content 3"), 0644)
@@ -214,6 +224,7 @@ func TestMultipleFileChanges(t *testing.T) {
VaultikVersion: "test",
StartedAt: time.Now(),
}
return repos.Snapshots.Create(ctx, tx, snapshot)
})
require.NoError(t, err)

2
internal/snapshot/manifest.go
View File

@@ -52,6 +52,7 @@ func EncodeManifest(manifest *Manifest, compressionLevel int) ([]byte, error) {
// Compress using zstd
var compressedBuf bytes.Buffer
writer, err := zstd.NewWriter(&compressedBuf, zstd.WithEncoderLevel(zstd.EncoderLevelFromZstd(compressionLevel)))
if err != nil {
return nil, fmt.Errorf("creating zstd writer: %w", err)
@@ -59,6 +60,7 @@ func EncodeManifest(manifest *Manifest, compressionLevel int) ([]byte, error) {
if _, err := writer.Write(jsonData); err != nil {
_ = writer.Close()
return nil, fmt.Errorf("writing compressed data: %w", err)
}

6
internal/snapshot/permission_error_test.go
View File

@@ -12,7 +12,9 @@ import (
func TestWrapPermissionError(t *testing.T) {
// Non-permission errors pass through unchanged.
plain := errors.New("disk on fire")
if got := wrapPermissionError("/some/path", plain); got != plain {
got := wrapPermissionError("/some/path", plain)
if !errors.Is(got, plain) {
t.Errorf("non-permission error should pass through, got %v", got)
}
@@ -23,6 +25,7 @@ func TestWrapPermissionError(t *testing.T) {
if !errors.Is(wrapped, os.ErrPermission) {
t.Error("wrapped error should still match os.ErrPermission")
}
if !strings.Contains(wrapped.Error(), "/Users/u/Library/Calendars") {
t.Error("wrapped error should name the offending path")
}
@@ -31,6 +34,7 @@ func TestWrapPermissionError(t *testing.T) {
if !strings.Contains(wrapped.Error(), "Full Disk Access") {
t.Errorf("macOS permission error should mention Full Disk Access:\n%s", wrapped.Error())
}
if !strings.Contains(wrapped.Error(), "System Settings") {
t.Errorf("macOS permission error should point at System Settings:\n%s", wrapped.Error())
}

18
internal/snapshot/progress.go
View File

@@ -153,6 +153,7 @@ func (pr *ProgressReporter) printSummaryStatus() {
if uploadInfo, ok := pr.stats.CurrentUpload.Load().(*UploadInfo); ok && uploadInfo != nil {
// Show upload progress instead
pr.printUploadProgress(uploadInfo)
return
}
@@ -175,16 +176,18 @@ func (pr *ProgressReporter) printSummaryStatus() {
// Calculate ETA if we have total size and are processing
etaStr := ""
if totalSize > 0 && bytesProcessed > 0 {
processStart, ok := pr.stats.ProcessStartTime.Load().(time.Time)
if ok && !processStart.IsZero() {
processElapsed := time.Since(processStart)
rate := float64(bytesProcessed) / processElapsed.Seconds()
if rate > 0 {
remainingBytes := totalSize - bytesProcessed
remainingSeconds := float64(remainingBytes) / rate
eta := time.Duration(remainingSeconds * float64(time.Second))
etaStr = fmt.Sprintf(" | ETA: %s", formatDuration(eta))
etaStr = " | ETA: " + formatDuration(eta)
}
}
}
@@ -206,7 +209,7 @@ func (pr *ProgressReporter) printSummaryStatus() {
)
if currentFile != "" {
status += fmt.Sprintf(" | Current: %s", truncatePath(currentFile, 40))
status += " | Current: " + truncatePath(currentFile, 40)
}
log.Info(status)
@@ -242,6 +245,7 @@ func (pr *ProgressReporter) printDetailedStatus() {
processStart, ok := pr.stats.ProcessStartTime.Load().(time.Time)
if ok && !processStart.IsZero() {
processElapsed := time.Since(processStart)
processRate := float64(bytesProcessed) / processElapsed.Seconds()
if processRate > 0 {
remainingBytes := totalSize - bytesProcessed
@@ -276,9 +280,11 @@ func (pr *ProgressReporter) printDetailedStatus() {
log.Info("Total uploaded to remote",
"uploaded", humanize.Bytes(uint64(bytesUploaded)),
"compression_ratio", formatRatio(bytesUploaded, bytesScanned))
if currentFile != "" {
log.Info("Current file", "path", currentFile)
}
log.Notice("=============================")
}
@@ -288,12 +294,15 @@ func formatDuration(d time.Duration) string {
if d < 0 {
return "unknown"
}
if d < time.Minute {
return fmt.Sprintf("%ds", int(d.Seconds()))
}
if d < time.Hour {
return fmt.Sprintf("%dm%ds", int(d.Minutes()), int(d.Seconds())%60)
}
return fmt.Sprintf("%dh%dm", int(d.Hours()), int(d.Minutes())%60)
}
@@ -301,6 +310,7 @@ func formatPercent(numerator, denominator int64) string {
if denominator == 0 {
return "0.0%"
}
return fmt.Sprintf("%.1f%%", float64(numerator)/float64(denominator)*100)
}
@@ -308,7 +318,9 @@ func formatRatio(compressed, uncompressed int64) string {
if uncompressed == 0 {
return "1.00"
}
ratio := float64(compressed) / float64(uncompressed)
return fmt.Sprintf("%.2f", ratio)
}
@@ -353,6 +365,7 @@ func (pr *ProgressReporter) ReportUploadComplete(blobHash string, size int64, du
if duration < time.Millisecond {
duration = time.Millisecond
}
bytesPerSec := float64(size) / duration.Seconds()
bitsPerSec := bytesPerSec * 8
@@ -398,6 +411,7 @@ func (pr *ProgressReporter) ReportUploadProgress(blobHash string, bytesUploaded,
// Calculate ETA based on current speed
etaStr := "unknown"
if instantSpeed > 0 && bytesUploaded < totalSize {
remainingBytes := totalSize - bytesUploaded
remainingSeconds := float64(remainingBytes) / instantSpeed

1
internal/snapshot/remotekey.go
View File

@@ -36,5 +36,6 @@ const remoteKeyPrefix = "vaultik|"
func RemoteSnapshotKey(snapshotID string) string {
first := sha256.Sum256([]byte(remoteKeyPrefix + snapshotID))
second := sha256.Sum256(first[:])
return hex.EncodeToString(second[:])
}

276
internal/snapshot/scanner.go
View File

@@ -119,6 +119,7 @@ func NewScanner(cfg ScannerConfig) *Scanner {
// Create encryptor (required for blob packing)
if len(cfg.AgeRecipients) == 0 {
log.Error("No age recipients configured - encryption is required")
return nil
}
@@ -130,9 +131,11 @@ func NewScanner(cfg ScannerConfig) *Scanner {
Repositories: cfg.Repositories,
Fs: cfg.FS,
}
packer, err := blob.NewPacker(packerCfg)
if err != nil {
log.Error("Failed to create packer", "error", err)
return nil
}
@@ -199,11 +202,14 @@ func (s *Scanner) Scan(ctx context.Context, path string, snapshotID string) (*Sc
// Phase 1: Scan directory, collect files to process, and track existing files
// (builds existingFiles map during walk to avoid double traversal)
log.Info("Phase 1/3: Scanning directory structure")
existingFiles := make(map[string]struct{})
scanResult, err := s.scanPhase(ctx, path, result, existingFiles, knownFiles)
if err != nil {
return nil, fmt.Errorf("scan phase failed: %w", err)
}
filesToProcess := scanResult.FilesToProcess
// Phase 1b: Detect deleted files by comparing DB against scanned files
@@ -213,8 +219,10 @@ func (s *Scanner) Scan(ctx context.Context, path string, snapshotID string) (*Sc
// Phase 1c: Associate unchanged files with this snapshot (no new records needed)
if len(scanResult.UnchangedFileIDs) > 0 {
s.ui.Begin("Associating %s unchanged files with the snapshot.", s.ui.Count(len(scanResult.UnchangedFileIDs)))
if err := s.batchAddFilesToSnapshot(ctx, scanResult.UnchangedFileIDs); err != nil {
s.ui.Beginf("Associating %s unchanged files with the snapshot.", s.ui.Count(len(scanResult.UnchangedFileIDs)))
err := s.batchAddFilesToSnapshot(ctx, scanResult.UnchangedFileIDs)
if err != nil {
return nil, fmt.Errorf("associating unchanged files: %w", err)
}
}
@@ -224,13 +232,15 @@ func (s *Scanner) Scan(ctx context.Context, path string, snapshotID string) (*Sc
// Phase 2: Process files and create chunks
if len(filesToProcess) > 0 {
s.ui.Begin("Backing up %s snapshot source files (chunking, compressing, encrypting, uploading).", s.ui.Count(len(filesToProcess)))
s.ui.Beginf("Backing up %s snapshot source files (chunking, compressing, encrypting, uploading).", s.ui.Count(len(filesToProcess)))
log.Info("Phase 2/3: Creating snapshot (chunking, compressing, encrypting, and uploading blobs)")
if err := s.processPhase(ctx, filesToProcess, result); err != nil {
err := s.processPhase(ctx, filesToProcess, result)
if err != nil {
return nil, fmt.Errorf("process phase failed: %w", err)
}
} else {
s.ui.Info("Snapshot file backup skipped: no changed files (creating metadata-only snapshot).")
s.ui.Infof("Snapshot file backup skipped: no changed files (creating metadata-only snapshot).")
log.Info("Phase 2/3: Skipping (no files need processing, metadata-only snapshot)")
}
@@ -243,18 +253,22 @@ func (s *Scanner) Scan(ctx context.Context, path string, snapshotID string) (*Sc
// loadDatabaseState loads known files and chunks from the database into memory for fast lookup
// This avoids per-file and per-chunk database queries during the scan and process phases
func (s *Scanner) loadDatabaseState(ctx context.Context, path string) (map[string]*database.File, error) {
s.ui.Begin("Loading known files from local index database.")
s.ui.Beginf("Loading known files from local index database.")
knownFiles, err := s.loadKnownFiles(ctx, path)
if err != nil {
return nil, fmt.Errorf("loading known files: %w", err)
}
s.ui.Complete("Loaded %s known files from local index database.", s.ui.Count(len(knownFiles)))
s.ui.Begin("Loading known chunks from local index database.")
s.ui.Completef("Loaded %s known files from local index database.", s.ui.Count(len(knownFiles)))
s.ui.Beginf("Loading known chunks from local index database.")
if err := s.loadKnownChunks(ctx); err != nil {
return nil, fmt.Errorf("loading known chunks: %w", err)
}
s.ui.Complete("Loaded %s known chunks from local index database.", s.ui.Count(len(s.knownChunks)))
s.ui.Completef("Loaded %s known chunks from local index database.", s.ui.Count(len(s.knownChunks)))
return knownFiles, nil
}
@@ -288,7 +302,8 @@ func (s *Scanner) summarizeScanPhase(result *ScanResult, filesToProcess []*FileT
s.ui.Count(result.FilesDeleted),
s.ui.Size(result.BytesDeleted))
}
s.ui.Complete("%s.", msg)
s.ui.Completef("%s.", msg)
}
// finalizeScanResult populates final blob statistics in the scan result
@@ -337,6 +352,7 @@ func (s *Scanner) loadKnownChunks(ctx context.Context) error {
}
s.knownChunksMu.Lock()
s.knownChunks = make(map[string]struct{}, len(chunks))
for _, c := range chunks {
s.knownChunks[c.ChunkHash.String()] = struct{}{}
@@ -351,6 +367,7 @@ func (s *Scanner) chunkExists(hash string) bool {
s.knownChunksMu.RLock()
_, exists := s.knownChunks[hash]
s.knownChunksMu.RUnlock()
return exists
}
@@ -371,7 +388,9 @@ func (s *Scanner) addPendingChunkHash(hash string) {
// removePendingChunkHashes removes committed chunk hashes from the pending set
func (s *Scanner) removePendingChunkHashes(hashes []string) {
log.Debug("removePendingChunkHashes: starting", "count", len(hashes))
start := time.Now()
s.pendingChunkHashesMu.Lock()
for _, hash := range hashes {
delete(s.pendingChunkHashes, hash)
@@ -385,6 +404,7 @@ func (s *Scanner) isChunkPending(hash string) bool {
s.pendingChunkHashesMu.Lock()
_, pending := s.pendingChunkHashes[hash]
s.pendingChunkHashesMu.Unlock()
return pending
}
@@ -411,37 +431,45 @@ func (s *Scanner) flushPendingFiles(ctx context.Context) error {
return s.repos.WithTx(ctx, func(txCtx context.Context, tx *sql.Tx) error {
for _, data := range files {
// Create or update the file record
if err := s.repos.Files.Create(txCtx, tx, data.file); err != nil {
err := s.repos.Files.Create(txCtx, tx, data.file)
if err != nil {
return fmt.Errorf("creating file record: %w", err)
}
// Delete any existing file_chunks and chunk_files for this file
if err := s.repos.FileChunks.DeleteByFileID(txCtx, tx, data.file.ID); err != nil {
err = s.repos.FileChunks.DeleteByFileID(txCtx, tx, data.file.ID)
if err != nil {
return fmt.Errorf("deleting old file chunks: %w", err)
}
if err := s.repos.ChunkFiles.DeleteByFileID(txCtx, tx, data.file.ID); err != nil {
err = s.repos.ChunkFiles.DeleteByFileID(txCtx, tx, data.file.ID)
if err != nil {
return fmt.Errorf("deleting old chunk files: %w", err)
}
// Create file-chunk mappings
for i := range data.fileChunks {
if err := s.repos.FileChunks.Create(txCtx, tx, &data.fileChunks[i]); err != nil {
err := s.repos.FileChunks.Create(txCtx, tx, &data.fileChunks[i])
if err != nil {
return fmt.Errorf("creating file chunk: %w", err)
}
}
// Create chunk-file mappings
for i := range data.chunkFiles {
if err := s.repos.ChunkFiles.Create(txCtx, tx, &data.chunkFiles[i]); err != nil {
err := s.repos.ChunkFiles.Create(txCtx, tx, &data.chunkFiles[i])
if err != nil {
return fmt.Errorf("creating chunk file: %w", err)
}
}
// Add file to snapshot
if err := s.repos.Snapshots.AddFileByID(txCtx, tx, s.snapshotID, data.file.ID); err != nil {
err = s.repos.Snapshots.AddFileByID(txCtx, tx, s.snapshotID, data.file.ID)
if err != nil {
return fmt.Errorf("adding file to snapshot: %w", err)
}
}
return nil
})
}
@@ -455,6 +483,7 @@ func (s *Scanner) flushAllPending(ctx context.Context) error {
// Files with pending chunks are kept in the queue for later flushing
func (s *Scanner) flushCompletedPendingFiles(ctx context.Context) error {
flushStart := time.Now()
log.Debug("flushCompletedPendingFiles: starting")
// Partition pending files into those ready to flush and those still waiting
@@ -462,6 +491,7 @@ func (s *Scanner) flushCompletedPendingFiles(ctx context.Context) error {
if len(canFlush) == 0 {
log.Debug("flushCompletedPendingFiles: nothing to flush")
return nil
}
@@ -474,10 +504,13 @@ func (s *Scanner) flushCompletedPendingFiles(ctx context.Context) error {
// Execute the batch flush in a single transaction
log.Debug("flushCompletedPendingFiles: starting transaction")
txStart := time.Now()
err := s.executeBatchFileFlush(ctx, allFiles, allFileIDs, allFileChunks, allChunkFiles)
log.Debug("flushCompletedPendingFiles: transaction done", "duration", time.Since(txStart))
log.Debug("flushCompletedPendingFiles: total duration", "duration", time.Since(flushStart))
return err
}
@@ -492,21 +525,27 @@ func (s *Scanner) partitionPendingByChunkStatus() (canFlush []pendingFileData, s
var stillPending []pendingFileData
log.Debug("flushCompletedPendingFiles: checking which files can flush")
checkStart := time.Now()
for _, data := range s.pendingFiles {
allChunksCommitted := true
for _, fc := range data.fileChunks {
if s.isChunkPending(fc.ChunkHash.String()) {
allChunksCommitted = false
break
}
}
if allChunksCommitted {
canFlush = append(canFlush, data)
} else {
stillPending = append(stillPending, data)
}
}
log.Debug("flushCompletedPendingFiles: check done", "duration", time.Since(checkStart), "can_flush", len(canFlush), "still_pending", len(stillPending))
s.pendingFiles = stillPending
@@ -520,12 +559,15 @@ func (s *Scanner) partitionPendingByChunkStatus() (canFlush []pendingFileData, s
// mappings from the given pending file data for efficient batch database operations
func (s *Scanner) collectBatchFlushData(canFlush []pendingFileData) ([]*database.File, []types.FileID, []database.FileChunk, []database.ChunkFile) {
log.Debug("flushCompletedPendingFiles: collecting data for batch ops")
collectStart := time.Now()
var allFileChunks []database.FileChunk
var allChunkFiles []database.ChunkFile
var allFileIDs []types.FileID
var allFiles []*database.File
var (
allFileChunks []database.FileChunk
allChunkFiles []database.ChunkFile
allFileIDs []types.FileID
allFiles []*database.File
)
for _, data := range canFlush {
allFileChunks = append(allFileChunks, data.fileChunks...)
@@ -551,52 +593,77 @@ func (s *Scanner) executeBatchFileFlush(ctx context.Context, allFiles []*databas
// Batch delete old file_chunks and chunk_files
log.Debug("flushCompletedPendingFiles: deleting old file_chunks")
opStart := time.Now()
if err := s.repos.FileChunks.DeleteByFileIDs(txCtx, tx, allFileIDs); err != nil {
err := s.repos.FileChunks.DeleteByFileIDs(txCtx, tx, allFileIDs)
if err != nil {
return fmt.Errorf("batch deleting old file chunks: %w", err)
}
log.Debug("flushCompletedPendingFiles: deleted file_chunks", "duration", time.Since(opStart))
log.Debug("flushCompletedPendingFiles: deleting old chunk_files")
opStart = time.Now()
if err := s.repos.ChunkFiles.DeleteByFileIDs(txCtx, tx, allFileIDs); err != nil {
err = s.repos.ChunkFiles.DeleteByFileIDs(txCtx, tx, allFileIDs)
if err != nil {
return fmt.Errorf("batch deleting old chunk files: %w", err)
}
log.Debug("flushCompletedPendingFiles: deleted chunk_files", "duration", time.Since(opStart))
// Batch create/update file records
log.Debug("flushCompletedPendingFiles: creating files")
opStart = time.Now()
if err := s.repos.Files.CreateBatch(txCtx, tx, allFiles); err != nil {
err = s.repos.Files.CreateBatch(txCtx, tx, allFiles)
if err != nil {
return fmt.Errorf("batch creating file records: %w", err)
}
log.Debug("flushCompletedPendingFiles: created files", "duration", time.Since(opStart))
// Batch insert file_chunks
log.Debug("flushCompletedPendingFiles: inserting file_chunks")
opStart = time.Now()
if err := s.repos.FileChunks.CreateBatch(txCtx, tx, allFileChunks); err != nil {
err = s.repos.FileChunks.CreateBatch(txCtx, tx, allFileChunks)
if err != nil {
return fmt.Errorf("batch creating file chunks: %w", err)
}
log.Debug("flushCompletedPendingFiles: inserted file_chunks", "duration", time.Since(opStart))
// Batch insert chunk_files
log.Debug("flushCompletedPendingFiles: inserting chunk_files")
opStart = time.Now()
if err := s.repos.ChunkFiles.CreateBatch(txCtx, tx, allChunkFiles); err != nil {
err = s.repos.ChunkFiles.CreateBatch(txCtx, tx, allChunkFiles)
if err != nil {
return fmt.Errorf("batch creating chunk files: %w", err)
}
log.Debug("flushCompletedPendingFiles: inserted chunk_files", "duration", time.Since(opStart))
// Batch add files to snapshot
log.Debug("flushCompletedPendingFiles: adding files to snapshot")
opStart = time.Now()
if err := s.repos.Snapshots.AddFilesByIDBatch(txCtx, tx, s.snapshotID, allFileIDs); err != nil {
err = s.repos.Snapshots.AddFilesByIDBatch(txCtx, tx, s.snapshotID, allFileIDs)
if err != nil {
return fmt.Errorf("batch adding files to snapshot: %w", err)
}
log.Debug("flushCompletedPendingFiles: added files to snapshot", "duration", time.Since(opStart))
log.Debug("flushCompletedPendingFiles: transaction complete")
return nil
})
}
@@ -616,24 +683,31 @@ func (s *Scanner) scanPhase(ctx context.Context, path string, result *ScanResult
estimatedTotal := int64(len(knownFiles))
var filesToProcess []*FileToProcess
var unchangedFileIDs []types.FileID // Just IDs - no new records needed
var mu sync.Mutex
// Set up periodic status output
startTime := time.Now()
lastStatusTime := time.Now()
statusInterval := 15 * time.Second
var filesScanned int64
log.Debug("Starting directory walk", "path", path)
err := afero.Walk(s.fs, path, func(filePath string, info os.FileInfo, err error) error {
if err != nil {
if s.skipErrors {
log.Error("Failed to access file (skipping due to --skip-errors)", "path", filePath, "error", err)
s.ui.Error("Failed to access %s: %v. Skipping (--skip-errors).", s.ui.Path(filePath), err)
s.ui.Errorf("Failed to access %s: %v. Skipping (--skip-errors).", s.ui.Path(filePath), err)
return nil // Continue scanning
}
log.Debug("Error accessing filesystem entry", "path", filePath, "error", err)
return wrapPermissionError(filePath, err)
}
@@ -649,6 +723,7 @@ func (s *Scanner) scanPhase(ctx context.Context, path string, result *ScanResult
if info.IsDir() {
return filepath.SkipDir
}
return nil
}
@@ -657,7 +732,9 @@ func (s *Scanner) scanPhase(ctx context.Context, path string, result *ScanResult
file := s.buildSymlinkEntry(filePath, info)
if file != nil {
existingFiles[filePath] = struct{}{}
mu.Lock()
filesToProcess = append(filesToProcess, &FileToProcess{
Path: filePath,
FileInfo: info,
@@ -667,6 +744,7 @@ func (s *Scanner) scanPhase(ctx context.Context, path string, result *ScanResult
mu.Unlock()
s.updateScanEntryStats(result, true, info)
}
return nil
}
@@ -674,7 +752,9 @@ func (s *Scanner) scanPhase(ctx context.Context, path string, result *ScanResult
if info.IsDir() {
file := s.buildDirectoryEntry(filePath, info)
existingFiles[filePath] = struct{}{}
mu.Lock()
filesToProcess = append(filesToProcess, &FileToProcess{
Path: filePath,
FileInfo: info,
@@ -682,6 +762,7 @@ func (s *Scanner) scanPhase(ctx context.Context, path string, result *ScanResult
})
filesScanned++
mu.Unlock()
return nil
}
@@ -708,6 +789,7 @@ func (s *Scanner) scanPhase(ctx context.Context, path string, result *ScanResult
// Unchanged file with existing ID - just need snapshot association
unchangedFileIDs = append(unchangedFileIDs, file.ID)
}
filesScanned++
changedCount := len(filesToProcess)
mu.Unlock()
@@ -718,12 +800,12 @@ func (s *Scanner) scanPhase(ctx context.Context, path string, result *ScanResult
// Output periodic status
if time.Since(lastStatusTime) >= statusInterval {
s.printScanProgressLine(filesScanned, changedCount, estimatedTotal, startTime)
lastStatusTime = time.Now()
}
return nil
})
if err != nil {
return nil, err
}
@@ -745,11 +827,13 @@ func (s *Scanner) updateScanEntryStats(result *ScanResult, needsProcessing bool,
} else {
result.FilesSkipped++
result.BytesSkipped += info.Size()
if s.progress != nil {
s.progress.GetStats().FilesSkipped.Add(1)
s.progress.GetStats().BytesSkipped.Add(info.Size())
}
}
result.FilesScanned++
if s.progress != nil {
s.progress.GetStats().FilesScanned.Add(1)
@@ -768,16 +852,16 @@ func (s *Scanner) printScanProgressLine(filesScanned int64, changedCount int, es
if pct > 100 {
pct = 100 // Cap at 100% for display
}
remaining := estimatedTotal - filesScanned
if remaining < 0 {
remaining = 0
}
remaining := max(estimatedTotal-filesScanned, 0)
var eta time.Duration
if rate > 0 && remaining > 0 {
eta = time.Duration(float64(remaining)/rate) * time.Second
}
if eta > 0 {
s.ui.Progress("Snapshot source files enumeration: %s files (~%s), %s changed or new, %.0f files/sec, enumeration elapsed: %s, enumeration ETA: %s (est remain %s).",
s.ui.Progressf("Snapshot source files enumeration: %s files (~%s), %s changed or new, %.0f files/sec, enumeration elapsed: %s, enumeration ETA: %s (est remain %s).",
s.ui.Count(int(filesScanned)),
s.ui.Percent(pct),
s.ui.Count(changedCount),
@@ -786,7 +870,7 @@ func (s *Scanner) printScanProgressLine(filesScanned int64, changedCount int, es
s.ui.Time(time.Now().Add(eta)),
s.ui.Duration(eta))
} else {
s.ui.Progress("Snapshot source files enumeration: %s files (~%s), %s changed or new, %.0f files/sec, enumeration elapsed: %s.",
s.ui.Progressf("Snapshot source files enumeration: %s files (~%s), %s changed or new, %.0f files/sec, enumeration elapsed: %s.",
s.ui.Count(int(filesScanned)),
s.ui.Percent(pct),
s.ui.Count(changedCount),
@@ -794,7 +878,7 @@ func (s *Scanner) printScanProgressLine(filesScanned int64, changedCount int, es
s.ui.Duration(elapsed))
}
} else {
s.ui.Progress("Snapshot source files enumeration: %s files seen, %s changed or new, %.0f files/sec, enumeration elapsed: %s.",
s.ui.Progressf("Snapshot source files enumeration: %s files seen, %s changed or new, %.0f files/sec, enumeration elapsed: %s.",
s.ui.Count(int(filesScanned)),
s.ui.Count(changedCount),
rate,
@@ -808,6 +892,7 @@ func (s *Scanner) buildSymlinkEntry(path string, info os.FileInfo) *database.Fil
target, err := os.Readlink(path)
if err != nil {
log.Debug("Cannot read symlink target", "path", path, "error", err)
return nil
}
@@ -860,9 +945,11 @@ func (s *Scanner) buildDirectoryEntry(path string, info os.FileInfo) *database.F
// and associates it with the current snapshot. No chunking is performed.
func (s *Scanner) recordNonRegularFile(ctx context.Context, ftp *FileToProcess) error {
return s.repos.WithTx(ctx, func(txCtx context.Context, tx *sql.Tx) error {
if err := s.repos.Files.Create(txCtx, tx, ftp.File); err != nil {
err := s.repos.Files.Create(txCtx, tx, ftp.File)
if err != nil {
return fmt.Errorf("creating non-regular file record: %w", err)
}
return s.repos.Snapshots.AddFileByID(txCtx, tx, s.snapshotID, ftp.File.ID)
})
}
@@ -941,18 +1028,18 @@ func (s *Scanner) batchAddFilesToSnapshot(ctx context.Context, fileIDs []types.F
default:
}
end := i + batchSize
if end > len(fileIDs) {
end = len(fileIDs)
}
end := min(i+batchSize, len(fileIDs))
batch := fileIDs[i:end]
err := s.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
for _, fileID := range batch {
if err := s.repos.Snapshots.AddFileByID(ctx, tx, s.snapshotID, fileID); err != nil {
err := s.repos.Snapshots.AddFileByID(ctx, tx, s.snapshotID, fileID)
if err != nil {
return fmt.Errorf("adding file to snapshot: %w", err)
}
}
return nil
})
if err != nil {
@@ -964,15 +1051,16 @@ func (s *Scanner) batchAddFilesToSnapshot(ctx context.Context, fileIDs []types.F
elapsed := time.Since(startTime)
rate := float64(end) / elapsed.Seconds()
pct := float64(end) / float64(len(fileIDs)) * 100
s.ui.Progress("Snapshot unchanged-file association: %s/%s (%s), %.0f files/sec.",
s.ui.Progressf("Snapshot unchanged-file association: %s/%s (%s), %.0f files/sec.",
s.ui.Count(end), s.ui.Count(len(fileIDs)), s.ui.Percent(pct), rate)
lastStatusTime = time.Now()
}
}
elapsed := time.Since(startTime)
rate := float64(len(fileIDs)) / elapsed.Seconds()
s.ui.Complete("Associated %s unchanged files with the snapshot in %s (%.0f files/sec).",
s.ui.Completef("Associated %s unchanged files with the snapshot in %s (%.0f files/sec).",
s.ui.Count(len(fileIDs)), s.ui.Duration(elapsed), rate)
return nil
@@ -991,7 +1079,9 @@ func (s *Scanner) processPhase(ctx context.Context, filesToProcess []*FileToProc
statusInterval := 15 * time.Second
startTime := time.Now()
filesProcessed := 0
var bytesProcessed int64
totalFiles := len(filesToProcess)
// Process each file
@@ -1006,6 +1096,7 @@ func (s *Scanner) processPhase(ctx context.Context, filesToProcess []*FileToProc
if err != nil {
return err
}
if skipped {
continue
}
@@ -1021,6 +1112,7 @@ func (s *Scanner) processPhase(ctx context.Context, filesToProcess []*FileToProc
// Output periodic status
if time.Since(lastStatusTime) >= statusInterval {
s.printProcessingProgress(filesProcessed, totalFiles, bytesProcessed, totalBytes, startTime)
lastStatusTime = time.Now()
}
}
@@ -1032,22 +1124,29 @@ func (s *Scanner) processPhase(ctx context.Context, filesToProcess []*FileToProc
// processFileWithErrorHandling wraps processFileStreaming with error recovery for
// deleted files and skip-errors mode. Returns (skipped, error).
func (s *Scanner) processFileWithErrorHandling(ctx context.Context, fileToProcess *FileToProcess, result *ScanResult) (bool, error) {
if err := s.processFileStreaming(ctx, fileToProcess, result); err != nil {
err := s.processFileStreaming(ctx, fileToProcess, result)
if err != nil {
// Handle files that were deleted between scan and process phases
if errors.Is(err, os.ErrNotExist) {
log.Warn("File was deleted during backup, skipping", "path", fileToProcess.Path)
result.FilesSkipped++
return true, nil
}
// Skip file read errors if --skip-errors is enabled
if s.skipErrors {
log.Error("Failed to process file (skipping due to --skip-errors)", "path", fileToProcess.Path, "error", err)
s.ui.Error("Failed to process %s: %v. Skipping (--skip-errors).", s.ui.Path(fileToProcess.Path), err)
s.ui.Errorf("Failed to process %s: %v. Skipping (--skip-errors).", s.ui.Path(fileToProcess.Path), err)
result.FilesSkipped++
return true, nil
}
return false, fmt.Errorf("processing file %s: %w", fileToProcess.Path, err)
}
return false, nil
}
@@ -1061,13 +1160,14 @@ func (s *Scanner) printProcessingProgress(filesProcessed, totalFiles int, bytesP
// Calculate ETA based on bytes (more accurate than files)
remainingBytes := totalBytes - bytesProcessed
var eta time.Duration
if byteRate > 0 {
eta = time.Duration(float64(remainingBytes)/byteRate) * time.Second
}
if eta > 0 {
s.ui.Progress("Snapshot backup: %s/%s files (%s), %s/%s, %s, %.0f files/sec, backup elapsed: %s, backup ETA: %s (est remain %s).",
s.ui.Progressf("Snapshot backup: %s/%s files (%s), %s/%s, %s, %.0f files/sec, backup elapsed: %s, backup ETA: %s (est remain %s).",
s.ui.Count(filesProcessed),
s.ui.Count(totalFiles),
s.ui.Percent(pct),
@@ -1079,7 +1179,7 @@ func (s *Scanner) printProcessingProgress(filesProcessed, totalFiles int, bytesP
s.ui.Time(time.Now().Add(eta)),
s.ui.Duration(eta))
} else {
s.ui.Progress("Snapshot backup: %s/%s files (%s), %s/%s, %s, %.0f files/sec, backup elapsed: %s.",
s.ui.Progressf("Snapshot backup: %s/%s files (%s), %s/%s, %s, %.0f files/sec, backup elapsed: %s.",
s.ui.Count(filesProcessed),
s.ui.Count(totalFiles),
s.ui.Percent(pct),
@@ -1097,15 +1197,19 @@ func (s *Scanner) finalizeProcessPhase(ctx context.Context, result *ScanResult)
// Final packer flush first - this commits remaining chunks to DB
// and handleBlobReady will flush files whose chunks are now committed
s.packerMu.Lock()
if err := s.packer.Flush(); err != nil {
err := s.packer.Flush()
if err != nil {
s.packerMu.Unlock()
return fmt.Errorf("flushing packer: %w", err)
}
s.packerMu.Unlock()
// Flush any remaining pending files (e.g., files with only pre-existing chunks
// that didn't trigger a blob finalize)
if err := s.flushAllPending(ctx); err != nil {
err = s.flushAllPending(ctx)
if err != nil {
return fmt.Errorf("flushing remaining pending files: %w", err)
}
@@ -1119,6 +1223,7 @@ func (s *Scanner) finalizeProcessPhase(ctx context.Context, result *ScanResult)
if err != nil {
return fmt.Errorf("parsing blob ID: %w", err)
}
err = s.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
return s.repos.Snapshots.AddBlob(ctx, tx, s.snapshotID, blobID, types.BlobHash(b.Hash))
})
@@ -1126,6 +1231,7 @@ func (s *Scanner) finalizeProcessPhase(ctx context.Context, result *ScanResult)
return fmt.Errorf("storing blob metadata: %w", err)
}
}
result.BlobsCreated += len(blobs)
}
@@ -1148,14 +1254,17 @@ func (s *Scanner) handleBlobReady(blobWithReader *blob.BlobWithReader) error {
}
blobPath := fmt.Sprintf("blobs/%s/%s/%s", finishedBlob.Hash[:2], finishedBlob.Hash[2:4], finishedBlob.Hash)
blobExists, err := s.uploadBlobIfNeeded(ctx, blobPath, blobWithReader, startTime)
if err != nil {
s.cleanupBlobTempFile(blobWithReader)
return fmt.Errorf("uploading blob %s: %w", finishedBlob.Hash, err)
}
if err := s.recordBlobMetadata(ctx, finishedBlob, blobExists, startTime); err != nil {
s.cleanupBlobTempFile(blobWithReader)
return err
}
@@ -1181,25 +1290,28 @@ func (s *Scanner) uploadBlobIfNeeded(ctx context.Context, blobPath string, blobW
if _, err := s.storage.Stat(ctx, blobPath); err == nil {
log.Info("Blob already exists in storage, skipping upload",
"hash", finishedBlob.Hash, "size", humanize.Bytes(uint64(finishedBlob.Compressed)))
s.ui.Info("Blob %s (%s) already exists at %s. Skipping upload.",
s.ui.Infof("Blob %s (%s) already exists at %s. Skipping upload.",
s.ui.Hex(finishedBlob.Hash), s.ui.Size(finishedBlob.Compressed), s.ui.Path(destination))
return true, nil
}
s.ui.Begin("Uploading blob %s (%s) to %s.",
s.ui.Beginf("Uploading blob %s (%s) to %s.",
s.ui.Hex(finishedBlob.Hash), s.ui.Size(finishedBlob.Compressed), s.ui.Path(destination))
progressCallback := s.makeUploadProgressCallback(ctx, finishedBlob, startTime)
if err := s.storage.PutWithProgress(ctx, blobPath, blobWithReader.Reader, finishedBlob.Compressed, progressCallback); err != nil {
err := s.storage.PutWithProgress(ctx, blobPath, blobWithReader.Reader, finishedBlob.Compressed, progressCallback)
if err != nil {
log.Error("Failed to upload blob", "hash", finishedBlob.Hash, "error", err)
return false, fmt.Errorf("uploading blob to storage: %w", err)
}
uploadDuration := time.Since(startTime)
uploadSpeedBps := float64(finishedBlob.Compressed) / uploadDuration.Seconds()
s.ui.Complete("Uploaded blob %s (%s) in %s at %s.",
s.ui.Completef("Uploaded blob %s (%s) in %s at %s.",
s.ui.Hex(finishedBlob.Hash),
s.ui.Size(finishedBlob.Compressed),
s.ui.Duration(uploadDuration),
@@ -1228,17 +1340,21 @@ func (s *Scanner) makeUploadProgressCallback(ctx context.Context, finishedBlob *
lastProgressTime := time.Now()
lastProgressBytes := int64(0)
lastStdoutTime := time.Now()
const stdoutInterval = 15 * time.Second
return func(uploaded int64) error {
now := time.Now()
elapsed := now.Sub(lastProgressTime).Seconds()
if elapsed > 0.5 {
bytesSinceLastUpdate := uploaded - lastProgressBytes
speed := float64(bytesSinceLastUpdate) / elapsed
if s.progress != nil {
s.progress.ReportUploadProgress(finishedBlob.Hash, uploaded, finishedBlob.Compressed, speed)
}
lastProgressTime = now
lastProgressBytes = uploaded
}
@@ -1248,11 +1364,13 @@ func (s *Scanner) makeUploadProgressCallback(ctx context.Context, finishedBlob *
totalElapsed := now.Sub(uploadStart)
pct := float64(uploaded) / float64(finishedBlob.Compressed) * 100
avgSpeed := float64(uploaded) / totalElapsed.Seconds()
var eta time.Duration
if avgSpeed > 0 {
eta = time.Duration(float64(finishedBlob.Compressed-uploaded)/avgSpeed) * time.Second
}
s.ui.Progress("Blob upload %s: %s / %s (%s) at %s, blob upload elapsed: %s, blob upload ETA: %s (est remain %s).",
s.ui.Progressf("Blob upload %s: %s / %s (%s) at %s, blob upload elapsed: %s, blob upload ETA: %s (est remain %s).",
s.ui.Hex(finishedBlob.Hash),
s.ui.Size(uploaded),
s.ui.Size(finishedBlob.Compressed),
@@ -1283,11 +1401,13 @@ func (s *Scanner) recordBlobMetadata(ctx context.Context, finishedBlob *blob.Fin
uploadDuration := time.Since(startTime)
return s.repos.WithTx(ctx, func(txCtx context.Context, tx *sql.Tx) error {
if err := s.repos.Blobs.UpdateUploaded(txCtx, tx, finishedBlob.ID); err != nil {
err := s.repos.Blobs.UpdateUploaded(txCtx, tx, finishedBlob.ID)
if err != nil {
return fmt.Errorf("updating blob upload timestamp: %w", err)
}
if err := s.repos.Snapshots.AddBlob(txCtx, tx, s.snapshotID, finishedBlobID, types.BlobHash(finishedBlob.Hash)); err != nil {
err = s.repos.Snapshots.AddBlob(txCtx, tx, s.snapshotID, finishedBlobID, types.BlobHash(finishedBlob.Hash))
if err != nil {
return fmt.Errorf("adding blob to snapshot: %w", err)
}
@@ -1299,7 +1419,9 @@ func (s *Scanner) recordBlobMetadata(ctx context.Context, finishedBlob *blob.Fin
Size: finishedBlob.Compressed,
DurationMs: uploadDuration.Milliseconds(),
}
if err := s.repos.Uploads.Create(txCtx, tx, upload); err != nil {
err := s.repos.Uploads.Create(txCtx, tx, upload)
if err != nil {
return fmt.Errorf("recording upload metrics: %w", err)
}
}
@@ -1312,10 +1434,14 @@ func (s *Scanner) recordBlobMetadata(ctx context.Context, finishedBlob *blob.Fin
func (s *Scanner) cleanupBlobTempFile(blobWithReader *blob.BlobWithReader) {
if blobWithReader.TempFile != nil {
tempName := blobWithReader.TempFile.Name()
if err := blobWithReader.TempFile.Close(); err != nil {
err := blobWithReader.TempFile.Close()
if err != nil {
log.Fatal("Failed to close temp file", "file", tempName, "error", err)
}
if err := s.fs.Remove(tempName); err != nil {
err = s.fs.Remove(tempName)
if err != nil {
log.Fatal("Failed to remove temp file", "file", tempName, "error", err)
}
}
@@ -1343,6 +1469,7 @@ func (s *Scanner) processFileStreaming(ctx context.Context, fileToProcess *FileT
defer func() { _ = file.Close() }()
var chunks []streamingChunkInfo
chunkIndex := 0
fileHash, err := s.chunker.ChunkReaderStreaming(file, func(chunk chunker.Chunk) error {
@@ -1372,16 +1499,17 @@ func (s *Scanner) processFileStreaming(ctx context.Context, fileToProcess *FileT
s.updateChunkStats(chunkExists, chunk.Size, result)
if !chunkExists {
if err := s.addChunkToPacker(chunk); err != nil {
err := s.addChunkToPacker(chunk)
if err != nil {
return err
}
}
chunk.Data = nil
chunkIndex++
return nil
})
if err != nil {
return fmt.Errorf("chunking file: %w", err)
}
@@ -1390,6 +1518,7 @@ func (s *Scanner) processFileStreaming(ctx context.Context, fileToProcess *FileT
"path", fileToProcess.Path, "file_hash", fileHash, "chunks", len(chunks))
s.queueFileForBatchInsert(ctx, fileToProcess, chunks)
return nil
}
@@ -1397,6 +1526,7 @@ func (s *Scanner) processFileStreaming(ctx context.Context, fileToProcess *FileT
func (s *Scanner) updateChunkStats(chunkExists bool, chunkSize int64, result *ScanResult) {
if chunkExists {
result.FilesSkipped++
result.BytesSkipped += chunkSize
if s.progress != nil {
s.progress.GetStats().BytesSkipped.Add(chunkSize)
@@ -1404,6 +1534,7 @@ func (s *Scanner) updateChunkStats(chunkExists bool, chunkSize int64, result *Sc
} else {
result.ChunksCreated++
result.BytesScanned += chunkSize
if s.progress != nil {
s.progress.GetStats().ChunksCreated.Add(1)
s.progress.GetStats().BytesProcessed.Add(chunkSize)
@@ -1415,27 +1546,36 @@ func (s *Scanner) updateChunkStats(chunkExists bool, chunkSize int64, result *Sc
// addChunkToPacker adds a chunk to the blob packer, finalizing the current blob if needed
func (s *Scanner) addChunkToPacker(chunk chunker.Chunk) error {
s.packerMu.Lock()
err := s.packer.AddChunk(&blob.ChunkRef{Hash: chunk.Hash, Data: chunk.Data})
if err == blob.ErrBlobSizeLimitExceeded {
if err := s.packer.FinalizeBlob(); err != nil {
if errors.Is(err, blob.ErrBlobSizeLimitExceeded) {
err := s.packer.FinalizeBlob()
if err != nil {
s.packerMu.Unlock()
return fmt.Errorf("finalizing blob: %w", err)
}
if err := s.packer.AddChunk(&blob.ChunkRef{Hash: chunk.Hash, Data: chunk.Data}); err != nil {
err = s.packer.AddChunk(&blob.ChunkRef{Hash: chunk.Hash, Data: chunk.Data})
if err != nil {
s.packerMu.Unlock()
return fmt.Errorf("adding chunk after finalize: %w", err)
}
} else if err != nil {
s.packerMu.Unlock()
return fmt.Errorf("adding chunk to packer: %w", err)
}
s.packerMu.Unlock()
return nil
}
// queueFileForBatchInsert builds file/chunk associations and queues the file for batch DB insert
func (s *Scanner) queueFileForBatchInsert(ctx context.Context, fileToProcess *FileToProcess, chunks []streamingChunkInfo) {
fileChunks := make([]database.FileChunk, len(chunks))
chunkFiles := make([]database.ChunkFile, len(chunks))
for i, ci := range chunks {
fileChunks[i] = database.FileChunk{
@@ -1489,7 +1629,7 @@ func (s *Scanner) detectDeletedFilesFromMap(ctx context.Context, knownFiles map[
}
if result.FilesDeleted > 0 {
s.ui.Info("Snapshot source files enumeration detected %s deleted files.", s.ui.Count(result.FilesDeleted))
s.ui.Infof("Snapshot source files enumeration detected %s deleted files.", s.ui.Count(result.FilesDeleted))
}
return nil
@@ -1503,6 +1643,7 @@ func wrapPermissionError(path string, err error) error {
if !errors.Is(err, os.ErrPermission) {
return err
}
if runtime.GOOS == "darwin" {
return fmt.Errorf("cannot read %s: %w\n\n"+
"macOS is blocking access to this path. Grant Full Disk Access to your\n"+
@@ -1510,12 +1651,14 @@ func wrapPermissionError(path string, err error) error {
" System Settings → Privacy & Security → Full Disk Access\n\n"+
"then quit and reopen the terminal and re-run the backup", path, err)
}
return fmt.Errorf("cannot read %s: %w (check file permissions, or run with --skip-errors to continue past unreadable files)", path, err)
}
// compileExcludePatterns compiles the exclude patterns into glob matchers
func compileExcludePatterns(patterns []string) []compiledPattern {
var compiled []compiledPattern
for _, p := range patterns {
if p == "" {
continue
@@ -1523,6 +1666,7 @@ func compileExcludePatterns(patterns []string) []compiledPattern {
// Check if pattern is anchored (starts with /)
anchored := strings.HasPrefix(p, "/")
pattern := p
if anchored {
pattern = p[1:] // Remove leading /
@@ -1537,6 +1681,7 @@ func compileExcludePatterns(patterns []string) []compiledPattern {
g, err := glob.Compile(pattern, '/')
if err != nil {
log.Warn("Invalid exclude pattern, skipping", "pattern", p, "error", err)
continue
}
@@ -1546,6 +1691,7 @@ func compileExcludePatterns(patterns []string) []compiledPattern {
original: p,
})
}
return compiled
}

27
internal/snapshot/scanner_test.go
View File

@@ -33,16 +33,22 @@ func TestScannerSimpleDirectory(t *testing.T) {
// Create files with specific times
testTime := time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC)
for path, content := range testFiles {
dir := filepath.Dir(path)
if err := fs.MkdirAll(dir, 0755); err != nil {
err := fs.MkdirAll(dir, 0755)
if err != nil {
t.Fatalf("failed to create directory %s: %v", dir, err)
}
if err := afero.WriteFile(fs, path, []byte(content), 0644); err != nil {
err = afero.WriteFile(fs, path, []byte(content), 0644)
if err != nil {
t.Fatalf("failed to write file %s: %v", path, err)
}
// Set times
if err := fs.Chtimes(path, testTime, testTime); err != nil {
err = fs.Chtimes(path, testTime, testTime)
if err != nil {
t.Fatalf("failed to set times for %s: %v", path, err)
}
}
@@ -53,7 +59,8 @@ func TestScannerSimpleDirectory(t *testing.T) {
t.Fatalf("failed to create test database: %v", err)
}
defer func() {
if err := db.Close(); err != nil {
err := db.Close()
if err != nil {
t.Errorf("failed to close database: %v", err)
}
}()
@@ -73,6 +80,7 @@ func TestScannerSimpleDirectory(t *testing.T) {
// Create a snapshot record for testing
ctx := context.Background()
snapshotID := "test-snapshot-001"
err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
snapshot := &database.Snapshot{
ID: types.SnapshotID(snapshotID),
@@ -87,6 +95,7 @@ func TestScannerSimpleDirectory(t *testing.T) {
BlobSize: 0,
CompressionRatio: 1.0,
}
return repos.Snapshots.Create(ctx, tx, snapshot)
})
if err != nil {
@@ -95,6 +104,7 @@ func TestScannerSimpleDirectory(t *testing.T) {
// Scan the directory
var result *snapshot.ScanResult
result, err = scanner.Scan(ctx, "/source", snapshotID)
if err != nil {
t.Fatalf("scan failed: %v", err)
@@ -170,7 +180,7 @@ func TestScannerLargeFile(t *testing.T) {
// Use random content to ensure good chunk boundaries
largeContent := make([]byte, 1024*1024) // 1MB
// Fill with pseudo-random data to ensure chunk boundaries
for i := 0; i < len(largeContent); i++ {
for i := range largeContent {
// Simple pseudo-random generator for deterministic tests
largeContent[i] = byte((i * 7919) ^ (i >> 3))
}
@@ -178,6 +188,7 @@ func TestScannerLargeFile(t *testing.T) {
if err := fs.MkdirAll("/source", 0755); err != nil {
t.Fatal(err)
}
if err := afero.WriteFile(fs, "/source/large.bin", largeContent, 0644); err != nil {
t.Fatal(err)
}
@@ -188,7 +199,8 @@ func TestScannerLargeFile(t *testing.T) {
t.Fatalf("failed to create test database: %v", err)
}
defer func() {
if err := db.Close(); err != nil {
err := db.Close()
if err != nil {
t.Errorf("failed to close database: %v", err)
}
}()
@@ -208,6 +220,7 @@ func TestScannerLargeFile(t *testing.T) {
// Create a snapshot record for testing
ctx := context.Background()
snapshotID := "test-snapshot-001"
err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
snapshot := &database.Snapshot{
ID: types.SnapshotID(snapshotID),
@@ -222,6 +235,7 @@ func TestScannerLargeFile(t *testing.T) {
BlobSize: 0,
CompressionRatio: 1.0,
}
return repos.Snapshots.Create(ctx, tx, snapshot)
})
if err != nil {
@@ -230,6 +244,7 @@ func TestScannerLargeFile(t *testing.T) {
// Scan the directory
var result *snapshot.ScanResult
result, err = scanner.Scan(ctx, "/source", snapshotID)
if err != nil {
t.Fatalf("scan failed: %v", err)

160
internal/snapshot/snapshot.go
View File

@@ -37,6 +37,8 @@ import (
"bytes"
"context"
"database/sql"
"encoding/hex"
"errors"
"fmt"
"io"
"os/exec"
@@ -97,12 +99,13 @@ func (sm *SnapshotManager) CreateSnapshot(ctx context.Context, hostname, version
func (sm *SnapshotManager) CreateSnapshotWithName(ctx context.Context, hostname, name, version, gitRevision string) (string, error) {
// Use short hostname (strip domain if present)
shortHostname := hostname
if idx := strings.Index(hostname, "."); idx != -1 {
shortHostname = hostname[:idx]
if before, _, ok := strings.Cut(hostname, "."); ok {
shortHostname = before
}
// Build snapshot ID with optional name
timestamp := time.Now().UTC().Format("2006-01-02T15:04:05Z")
var snapshotID string
if name != "" {
snapshotID = fmt.Sprintf("%s_%s_%s", shortHostname, name, timestamp)
@@ -128,12 +131,12 @@ func (sm *SnapshotManager) CreateSnapshotWithName(ctx context.Context, hostname,
err := sm.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
return sm.repos.Snapshots.Create(ctx, tx, snapshot)
})
if err != nil {
return "", fmt.Errorf("creating snapshot: %w", err)
}
log.Info("Created snapshot", "snapshot_id", snapshotID)
return snapshotID, nil
}
@@ -148,7 +151,6 @@ func (sm *SnapshotManager) UpdateSnapshotStats(ctx context.Context, snapshotID s
stats.BytesUploaded,
)
})
if err != nil {
return fmt.Errorf("updating snapshot stats: %w", err)
}
@@ -161,13 +163,14 @@ func (sm *SnapshotManager) UpdateSnapshotStats(ctx context.Context, snapshotID s
func (sm *SnapshotManager) UpdateSnapshotStatsExtended(ctx context.Context, snapshotID string, stats ExtendedBackupStats) error {
return sm.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
// First update basic stats
if err := sm.repos.Snapshots.UpdateCounts(ctx, tx, snapshotID,
err := sm.repos.Snapshots.UpdateCounts(ctx, tx, snapshotID,
int64(stats.FilesScanned),
int64(stats.ChunksCreated),
int64(stats.BlobsCreated),
stats.BytesScanned,
stats.BytesUploaded,
); err != nil {
)
if err != nil {
return err
}
@@ -190,18 +193,20 @@ func (sm *SnapshotManager) CompleteSnapshot(ctx context.Context, snapshotID stri
if err != nil {
return err
}
if added > 0 {
log.Info("Populated snapshot_blobs with dedup-referenced blobs",
"snapshot_id", snapshotID, "added", added)
}
return sm.repos.Snapshots.MarkComplete(ctx, tx, snapshotID)
})
if err != nil {
return fmt.Errorf("marking snapshot complete: %w", err)
}
log.Info("Completed snapshot", "snapshot_id", snapshotID)
return nil
}
@@ -229,10 +234,13 @@ func (sm *SnapshotManager) ExportSnapshotMetadata(ctx context.Context, dbPath st
if err != nil {
return fmt.Errorf("creating temp dir: %w", err)
}
log.Debug("Created temporary directory", "path", tempDir)
defer func() {
log.Debug("Cleaning up temporary directory", "path", tempDir)
if err := sm.fs.RemoveAll(tempDir); err != nil {
err := sm.fs.RemoveAll(tempDir)
if err != nil {
log.Debug("Failed to remove temp dir", "path", tempDir, "error", err)
}
}()
@@ -258,6 +266,7 @@ func (sm *SnapshotManager) ExportSnapshotMetadata(ctx context.Context, dbPath st
"snapshot_id", snapshotID,
"db_size", len(finalData),
"manifest_size", len(blobManifest))
return nil
}
@@ -268,17 +277,21 @@ func (sm *SnapshotManager) prepareExportDB(ctx context.Context, dbPath, snapshot
// The main database should be closed at this point
tempDBPath := filepath.Join(tempDir, "snapshot.db")
log.Debug("Copying database to temporary location", "source", dbPath, "destination", tempDBPath)
if err := sm.copyFile(dbPath, tempDBPath); err != nil {
return nil, "", fmt.Errorf("copying database: %w", err)
}
log.Debug("Database copy complete", "size", sm.getFileSize(tempDBPath))
// Step 2: Clean the temp database to only contain current snapshot data
log.Debug("Cleaning temporary database", "snapshot_id", snapshotID)
stats, err := sm.cleanSnapshotDB(ctx, tempDBPath, snapshotID)
if err != nil {
return nil, "", fmt.Errorf("cleaning snapshot database: %w", err)
}
log.Info("Temporary database cleanup complete",
"db_path", tempDBPath,
"size_after_clean", humanize.Bytes(uint64(sm.getFileSize(tempDBPath))),
@@ -294,6 +307,7 @@ func (sm *SnapshotManager) prepareExportDB(ctx context.Context, dbPath, snapshot
if err := sm.vacuumDatabase(tempDBPath); err != nil {
return nil, "", fmt.Errorf("vacuuming database: %w", err)
}
log.Debug("Database vacuumed", "size", humanize.Bytes(uint64(sm.getFileSize(tempDBPath))))
// Step 4: Compress and encrypt the binary database file
@@ -301,6 +315,7 @@ func (sm *SnapshotManager) prepareExportDB(ctx context.Context, dbPath, snapshot
if err := sm.compressFile(tempDBPath, compressedPath); err != nil {
return nil, "", fmt.Errorf("compressing database: %w", err)
}
log.Debug("Compression complete",
"original_size", humanize.Bytes(uint64(sm.getFileSize(tempDBPath))),
"compressed_size", humanize.Bytes(uint64(sm.getFileSize(compressedPath))))
@@ -327,9 +342,12 @@ func (sm *SnapshotManager) uploadSnapshotArtifacts(ctx context.Context, snapshot
dbKey := fmt.Sprintf("metadata/%s/db.zst.age", remoteKey)
dbUploadStart := time.Now()
if err := sm.storage.Put(ctx, dbKey, bytes.NewReader(dbData)); err != nil {
err := sm.storage.Put(ctx, dbKey, bytes.NewReader(dbData))
if err != nil {
return fmt.Errorf("uploading snapshot database: %w", err)
}
dbUploadDuration := time.Since(dbUploadStart)
dbUploadSpeed := float64(len(dbData)) * 8 / dbUploadDuration.Seconds() // bits per second
log.Info("Uploaded snapshot database",
@@ -341,9 +359,12 @@ func (sm *SnapshotManager) uploadSnapshotArtifacts(ctx context.Context, snapshot
// Upload blob manifest (compressed only, not encrypted)
manifestKey := fmt.Sprintf("metadata/%s/manifest.json.zst", remoteKey)
manifestUploadStart := time.Now()
if err := sm.storage.Put(ctx, manifestKey, bytes.NewReader(manifestData)); err != nil {
err = sm.storage.Put(ctx, manifestKey, bytes.NewReader(manifestData))
if err != nil {
return fmt.Errorf("uploading blob manifest: %w", err)
}
manifestUploadDuration := time.Since(manifestUploadStart)
manifestUploadSpeed := float64(len(manifestData)) * 8 / manifestUploadDuration.Seconds() // bits per second
log.Info("Uploaded blob manifest",
@@ -383,7 +404,8 @@ func (sm *SnapshotManager) cleanSnapshotDB(ctx context.Context, dbPath string, s
return nil, fmt.Errorf("opening temp database: %w", err)
}
defer func() {
if err := db.Close(); err != nil {
err := db.Close()
if err != nil {
log.Debug("Failed to close temp database", "error", err)
}
}()
@@ -394,7 +416,8 @@ func (sm *SnapshotManager) cleanSnapshotDB(ctx context.Context, dbPath string, s
return nil, fmt.Errorf("beginning transaction: %w", err)
}
defer func() {
if rbErr := tx.Rollback(); rbErr != nil && rbErr != sql.ErrTxDone {
rbErr := tx.Rollback()
if rbErr != nil && !errors.Is(rbErr, sql.ErrTxDone) {
log.Debug("Failed to rollback transaction", "error", rbErr)
}
}()
@@ -430,6 +453,7 @@ func (sm *SnapshotManager) cleanSnapshotDB(ctx context.Context, dbPath string, s
// Commit transaction
log.Debug("[Temp DB Cleanup] Committing cleanup transaction")
if err := tx.Commit(); err != nil {
return nil, fmt.Errorf("committing transaction: %w", err)
}
@@ -439,23 +463,30 @@ func (sm *SnapshotManager) cleanSnapshotDB(ctx context.Context, dbPath string, s
// Count files
var fileCount int
err = db.QueryRowWithLog(ctx, "SELECT COUNT(*) FROM files").Scan(&fileCount)
if err != nil {
return nil, fmt.Errorf("counting files: %w", err)
}
stats.FileCount = fileCount
// Count chunks
var chunkCount int
err = db.QueryRowWithLog(ctx, "SELECT COUNT(*) FROM chunks").Scan(&chunkCount)
if err != nil {
return nil, fmt.Errorf("counting chunks: %w", err)
}
stats.ChunkCount = chunkCount
// Count blobs and get sizes
var blobCount int
var compressedSize, uncompressedSize sql.NullInt64
var (
blobCount int
compressedSize, uncompressedSize sql.NullInt64
)
err = db.QueryRowWithLog(ctx, `
SELECT COUNT(*), COALESCE(SUM(compressed_size), 0), COALESCE(SUM(uncompressed_size), 0)
FROM blobs
@@ -464,6 +495,7 @@ func (sm *SnapshotManager) cleanSnapshotDB(ctx context.Context, dbPath string, s
if err != nil {
return nil, fmt.Errorf("counting blobs and sizes: %w", err)
}
stats.BlobCount = blobCount
stats.CompressedSize = compressedSize.Int64
stats.UncompressedSize = uncompressedSize.Int64
@@ -491,7 +523,8 @@ func (sm *SnapshotManager) compressFile(inputPath, outputPath string) error {
return fmt.Errorf("opening input file: %w", err)
}
defer func() {
if err := input.Close(); err != nil {
err := input.Close()
if err != nil {
log.Debug("Failed to close input file", "path", inputPath, "error", err)
}
}()
@@ -501,13 +534,15 @@ func (sm *SnapshotManager) compressFile(inputPath, outputPath string) error {
return fmt.Errorf("creating output file: %w", err)
}
defer func() {
if err := output.Close(); err != nil {
err := output.Close()
if err != nil {
log.Debug("Failed to close output file", "path", outputPath, "error", err)
}
}()
// Use blobgen for compression and encryption
log.Debug("Compressing and encrypting data")
writer, err := blobgen.NewWriter(output, sm.config.CompressionLevel, sm.config.AgeRecipients)
if err != nil {
return fmt.Errorf("creating blobgen writer: %w", err)
@@ -517,7 +552,8 @@ func (sm *SnapshotManager) compressFile(inputPath, outputPath string) error {
writerClosed := false
defer func() {
if !writerClosed {
if err := writer.Close(); err != nil {
err := writer.Close()
if err != nil {
log.Debug("Failed to close writer", "error", err)
}
}
@@ -531,9 +567,10 @@ func (sm *SnapshotManager) compressFile(inputPath, outputPath string) error {
if err := writer.Close(); err != nil {
return fmt.Errorf("closing writer: %w", err)
}
writerClosed = true
log.Debug("Compression complete", "hash", fmt.Sprintf("%x", writer.Sum256()))
log.Debug("Compression complete", "hash", hex.EncodeToString(writer.Sum256()))
return nil
}
@@ -541,34 +578,42 @@ func (sm *SnapshotManager) compressFile(inputPath, outputPath string) error {
// copyFile copies a file from src to dst
func (sm *SnapshotManager) copyFile(src, dst string) error {
log.Debug("Opening source file for copy", "path", src)
sourceFile, err := sm.fs.Open(src)
if err != nil {
return err
}
defer func() {
log.Debug("Closing source file", "path", src)
if err := sourceFile.Close(); err != nil {
err := sourceFile.Close()
if err != nil {
log.Debug("Failed to close source file", "path", src, "error", err)
}
}()
log.Debug("Creating destination file", "path", dst)
destFile, err := sm.fs.Create(dst)
if err != nil {
return err
}
defer func() {
log.Debug("Closing destination file", "path", dst)
if err := destFile.Close(); err != nil {
err := destFile.Close()
if err != nil {
log.Debug("Failed to close destination file", "path", dst, "error", err)
}
}()
log.Debug("Copying file data")
n, err := io.Copy(destFile, sourceFile)
if err != nil {
return err
}
log.Debug("File copy complete", "bytes_copied", n)
return nil
@@ -576,7 +621,6 @@ func (sm *SnapshotManager) copyFile(src, dst string) error {
// generateBlobManifest creates a compressed JSON list of all blobs in the snapshot
func (sm *SnapshotManager) generateBlobManifest(ctx context.Context, dbPath string, snapshotID string) ([]byte, error) {
// Open the cleaned database using the database package
db, err := database.New(ctx, dbPath)
if err != nil {
@@ -589,10 +633,12 @@ func (sm *SnapshotManager) generateBlobManifest(ctx context.Context, dbPath stri
// Get all blobs for this snapshot
log.Debug("Querying blobs for snapshot", "snapshot_id", snapshotID)
blobHashes, err := repos.Snapshots.GetBlobHashes(ctx, snapshotID)
if err != nil {
return nil, fmt.Errorf("getting snapshot blobs: %w", err)
}
log.Debug("Found blobs", "count", len(blobHashes))
// Get blob details including sizes
@@ -603,8 +649,10 @@ func (sm *SnapshotManager) generateBlobManifest(ctx context.Context, dbPath stri
blob, err := repos.Blobs.GetByHash(ctx, hash)
if err != nil {
log.Warn("Failed to get blob details", "hash", hash, "error", err)
continue
}
if blob != nil {
blobs = append(blobs, BlobInfo{
Hash: hash,
@@ -648,6 +696,7 @@ func (sm *SnapshotManager) getFileSize(path string) int64 {
if err != nil {
return -1
}
return info.Size()
}
@@ -663,6 +712,7 @@ type BackupStats struct {
// ExtendedBackupStats contains additional statistics for comprehensive tracking
type ExtendedBackupStats struct {
BackupStats
BlobUncompressedSize int64 // Total uncompressed size of all referenced blobs
CompressionLevel int // Compression level used for this snapshot
UploadDurationMs int64 // Total milliseconds spent uploading to S3
@@ -682,6 +732,7 @@ func (sm *SnapshotManager) CleanupIncompleteSnapshots(ctx context.Context, hostn
if len(incompleteSnapshots) == 0 {
log.Debug("No incomplete snapshots found")
return nil
}
@@ -692,14 +743,15 @@ func (sm *SnapshotManager) CleanupIncompleteSnapshots(ctx context.Context, hostn
// Check if metadata exists in storage (paths use the hashed
// remote key so we don't leak host info to the listing).
metadataKey := fmt.Sprintf("metadata/%s/db.zst", RemoteSnapshotKey(snapshot.ID.String()))
_, err := sm.storage.Stat(ctx, metadataKey)
_, err := sm.storage.Stat(ctx, metadataKey)
if err != nil {
// Metadata doesn't exist in S3 - this is an incomplete snapshot
log.Info("Cleaning up incomplete snapshot record", "snapshot_id", snapshot.ID, "started_at", snapshot.StartedAt)
// Delete the snapshot and all its associations
if err := sm.deleteSnapshot(ctx, snapshot.ID.String()); err != nil {
err := sm.deleteSnapshot(ctx, snapshot.ID.String())
if err != nil {
return fmt.Errorf("deleting incomplete snapshot %s: %w", snapshot.ID, err)
}
@@ -708,7 +760,9 @@ func (sm *SnapshotManager) CleanupIncompleteSnapshots(ctx context.Context, hostn
// Metadata exists - this snapshot was completed but database wasn't updated
// This shouldn't happen in normal operation, but mark it complete
log.Warn("Found snapshot with remote metadata but incomplete in database", "snapshot_id", snapshot.ID)
if err := sm.repos.Snapshots.MarkComplete(ctx, nil, snapshot.ID.String()); err != nil {
err := sm.repos.Snapshots.MarkComplete(ctx, nil, snapshot.ID.String())
if err != nil {
log.Error("Failed to mark snapshot as complete in database", "snapshot_id", snapshot.ID, "error", err)
}
}
@@ -720,28 +774,34 @@ func (sm *SnapshotManager) CleanupIncompleteSnapshots(ctx context.Context, hostn
// deleteSnapshot removes a snapshot and all its associations from the database
func (sm *SnapshotManager) deleteSnapshot(ctx context.Context, snapshotID string) error {
// Delete snapshot_files entries
if err := sm.repos.Snapshots.DeleteSnapshotFiles(ctx, snapshotID); err != nil {
err := sm.repos.Snapshots.DeleteSnapshotFiles(ctx, snapshotID)
if err != nil {
return fmt.Errorf("deleting snapshot files: %w", err)
}
// Delete snapshot_blobs entries
if err := sm.repos.Snapshots.DeleteSnapshotBlobs(ctx, snapshotID); err != nil {
err = sm.repos.Snapshots.DeleteSnapshotBlobs(ctx, snapshotID)
if err != nil {
return fmt.Errorf("deleting snapshot blobs: %w", err)
}
// Delete uploads entries (has foreign key to snapshots without CASCADE)
if err := sm.repos.Snapshots.DeleteSnapshotUploads(ctx, snapshotID); err != nil {
err = sm.repos.Snapshots.DeleteSnapshotUploads(ctx, snapshotID)
if err != nil {
return fmt.Errorf("deleting snapshot uploads: %w", err)
}
// Delete the snapshot itself
if err := sm.repos.Snapshots.Delete(ctx, snapshotID); err != nil {
err = sm.repos.Snapshots.Delete(ctx, snapshotID)
if err != nil {
return fmt.Errorf("deleting snapshot: %w", err)
}
// Clean up orphaned data
log.Debug("Cleaning up orphaned records in main database")
if err := sm.CleanupOrphanedData(ctx); err != nil {
err = sm.CleanupOrphanedData(ctx)
if err != nil {
return fmt.Errorf("cleaning up orphaned data: %w", err)
}
@@ -759,28 +819,36 @@ func (sm *SnapshotManager) CleanupOrphanedData(ctx context.Context) error {
// Delete orphaned files (files not in any snapshot)
log.Debug("Deleting orphaned file records from database")
if err := sm.repos.Files.DeleteOrphaned(ctx); err != nil {
err := sm.repos.Files.DeleteOrphaned(ctx)
if err != nil {
return fmt.Errorf("deleting orphaned files: %w", err)
}
// Delete orphaned blobs (blobs not in any snapshot)
// This will cascade delete blob_chunks for deleted blobs
log.Debug("Deleting orphaned blob records from database")
if err := sm.repos.Blobs.DeleteOrphaned(ctx); err != nil {
err = sm.repos.Blobs.DeleteOrphaned(ctx)
if err != nil {
return fmt.Errorf("deleting orphaned blobs: %w", err)
}
// Delete orphaned blob_chunks entries
// This handles cases where the blob still exists but chunks were deleted
log.Debug("Deleting orphaned blob_chunks associations from database")
if err := sm.repos.BlobChunks.DeleteOrphaned(ctx); err != nil {
err = sm.repos.BlobChunks.DeleteOrphaned(ctx)
if err != nil {
return fmt.Errorf("deleting orphaned blob_chunks: %w", err)
}
// Delete orphaned chunks (chunks not referenced by any file)
// This must come after cleaning up blob_chunks to avoid foreign key violations
log.Debug("Deleting orphaned chunk records from database")
if err := sm.repos.Chunks.DeleteOrphaned(ctx); err != nil {
err = sm.repos.Chunks.DeleteOrphaned(ctx)
if err != nil {
return fmt.Errorf("deleting orphaned chunks: %w", err)
}
@@ -793,21 +861,26 @@ func (sm *SnapshotManager) deleteOtherSnapshots(ctx context.Context, tx *sql.Tx,
// First delete uploads that reference other snapshots (no CASCADE DELETE on this FK)
database.LogSQL("Execute", "DELETE FROM uploads WHERE snapshot_id != ?", currentSnapshotID)
uploadResult, err := tx.ExecContext(ctx, "DELETE FROM uploads WHERE snapshot_id != ?", currentSnapshotID)
if err != nil {
return fmt.Errorf("deleting uploads for other snapshots: %w", err)
}
uploadsDeleted, _ := uploadResult.RowsAffected()
log.Debug("[Temp DB Cleanup] Deleted upload records", "count", uploadsDeleted)
// Now we can safely delete the snapshots
database.LogSQL("Execute", "DELETE FROM snapshots WHERE id != ?", currentSnapshotID)
result, err := tx.ExecContext(ctx, "DELETE FROM snapshots WHERE id != ?", currentSnapshotID)
if err != nil {
return fmt.Errorf("deleting other snapshots: %w", err)
}
rowsAffected, _ := result.RowsAffected()
log.Debug("[Temp DB Cleanup] Deleted snapshot records from database", "count", rowsAffected)
return nil
}
@@ -816,22 +889,27 @@ func (sm *SnapshotManager) deleteOrphanedSnapshotAssociations(ctx context.Contex
// Delete orphaned snapshot_files
log.Debug("[Temp DB Cleanup] Deleting orphaned snapshot_files associations")
database.LogSQL("Execute", "DELETE FROM snapshot_files WHERE snapshot_id != ?", currentSnapshotID)
result, err := tx.ExecContext(ctx, "DELETE FROM snapshot_files WHERE snapshot_id != ?", currentSnapshotID)
if err != nil {
return fmt.Errorf("deleting orphaned snapshot_files: %w", err)
}
rowsAffected, _ := result.RowsAffected()
log.Debug("[Temp DB Cleanup] Deleted snapshot_files associations", "count", rowsAffected)
// Delete orphaned snapshot_blobs
log.Debug("[Temp DB Cleanup] Deleting orphaned snapshot_blobs associations")
database.LogSQL("Execute", "DELETE FROM snapshot_blobs WHERE snapshot_id != ?", currentSnapshotID)
result, err = tx.ExecContext(ctx, "DELETE FROM snapshot_blobs WHERE snapshot_id != ?", currentSnapshotID)
if err != nil {
return fmt.Errorf("deleting orphaned snapshot_blobs: %w", err)
}
rowsAffected, _ = result.RowsAffected()
log.Debug("[Temp DB Cleanup] Deleted snapshot_blobs associations", "count", rowsAffected)
return nil
}
@@ -839,6 +917,7 @@ func (sm *SnapshotManager) deleteOrphanedSnapshotAssociations(ctx context.Contex
func (sm *SnapshotManager) deleteOrphanedFiles(ctx context.Context, tx *sql.Tx, currentSnapshotID string) error {
log.Debug("[Temp DB Cleanup] Deleting file records not referenced by current snapshot")
database.LogSQL("Execute", `DELETE FROM files WHERE NOT EXISTS (SELECT 1 FROM snapshot_files WHERE snapshot_files.file_id = files.id AND snapshot_files.snapshot_id = ?)`, currentSnapshotID)
result, err := tx.ExecContext(ctx, `
DELETE FROM files
WHERE NOT EXISTS (
@@ -849,11 +928,13 @@ func (sm *SnapshotManager) deleteOrphanedFiles(ctx context.Context, tx *sql.Tx,
if err != nil {
return fmt.Errorf("deleting orphaned files: %w", err)
}
rowsAffected, _ := result.RowsAffected()
log.Debug("[Temp DB Cleanup] Deleted file records from database", "count", rowsAffected)
// Note: file_chunks will be deleted via CASCADE
log.Debug("[Temp DB Cleanup] file_chunks associations deleted via CASCADE")
return nil
}
@@ -861,6 +942,7 @@ func (sm *SnapshotManager) deleteOrphanedFiles(ctx context.Context, tx *sql.Tx,
func (sm *SnapshotManager) deleteOrphanedChunkToFileMappings(ctx context.Context, tx *sql.Tx) error {
log.Debug("[Temp DB Cleanup] Deleting orphaned chunk_files associations")
database.LogSQL("Execute", `DELETE FROM chunk_files WHERE NOT EXISTS (SELECT 1 FROM files WHERE files.id = chunk_files.file_id)`)
result, err := tx.ExecContext(ctx, `
DELETE FROM chunk_files
WHERE NOT EXISTS (
@@ -870,8 +952,10 @@ func (sm *SnapshotManager) deleteOrphanedChunkToFileMappings(ctx context.Context
if err != nil {
return fmt.Errorf("deleting orphaned chunk_files: %w", err)
}
rowsAffected, _ := result.RowsAffected()
log.Debug("[Temp DB Cleanup] Deleted chunk_files associations", "count", rowsAffected)
return nil
}
@@ -879,6 +963,7 @@ func (sm *SnapshotManager) deleteOrphanedChunkToFileMappings(ctx context.Context
func (sm *SnapshotManager) deleteOrphanedBlobs(ctx context.Context, tx *sql.Tx, currentSnapshotID string) error {
log.Debug("[Temp DB Cleanup] Deleting blob records not referenced by current snapshot")
database.LogSQL("Execute", `DELETE FROM blobs WHERE NOT EXISTS (SELECT 1 FROM snapshot_blobs WHERE snapshot_blobs.blob_hash = blobs.blob_hash AND snapshot_blobs.snapshot_id = ?)`, currentSnapshotID)
result, err := tx.ExecContext(ctx, `
DELETE FROM blobs
WHERE NOT EXISTS (
@@ -889,8 +974,10 @@ func (sm *SnapshotManager) deleteOrphanedBlobs(ctx context.Context, tx *sql.Tx,
if err != nil {
return fmt.Errorf("deleting orphaned blobs: %w", err)
}
rowsAffected, _ := result.RowsAffected()
log.Debug("[Temp DB Cleanup] Deleted blob records from database", "count", rowsAffected)
return nil
}
@@ -898,6 +985,7 @@ func (sm *SnapshotManager) deleteOrphanedBlobs(ctx context.Context, tx *sql.Tx,
func (sm *SnapshotManager) deleteOrphanedBlobToChunkMappings(ctx context.Context, tx *sql.Tx) error {
log.Debug("[Temp DB Cleanup] Deleting orphaned blob_chunks associations")
database.LogSQL("Execute", `DELETE FROM blob_chunks WHERE NOT EXISTS (SELECT 1 FROM blobs WHERE blobs.id = blob_chunks.blob_id)`)
result, err := tx.ExecContext(ctx, `
DELETE FROM blob_chunks
WHERE NOT EXISTS (
@@ -907,14 +995,17 @@ func (sm *SnapshotManager) deleteOrphanedBlobToChunkMappings(ctx context.Context
if err != nil {
return fmt.Errorf("deleting orphaned blob_chunks: %w", err)
}
rowsAffected, _ := result.RowsAffected()
log.Debug("[Temp DB Cleanup] Deleted blob_chunks associations", "count", rowsAffected)
return nil
}
// deleteOrphanedChunks deletes chunks not referenced by any file or blob
func (sm *SnapshotManager) deleteOrphanedChunks(ctx context.Context, tx *sql.Tx) error {
log.Debug("[Temp DB Cleanup] Deleting orphaned chunk records")
query := `
DELETE FROM chunks
WHERE NOT EXISTS (
@@ -926,11 +1017,14 @@ func (sm *SnapshotManager) deleteOrphanedChunks(ctx context.Context, tx *sql.Tx)
WHERE blob_chunks.chunk_hash = chunks.chunk_hash
)`
database.LogSQL("Execute", query)
result, err := tx.ExecContext(ctx, query)
if err != nil {
return fmt.Errorf("deleting orphaned chunks: %w", err)
}
rowsAffected, _ := result.RowsAffected()
log.Debug("[Temp DB Cleanup] Deleted chunk records from database", "count", rowsAffected)
return nil
}

13
internal/snapshot/snapshot_test.go
View File

@@ -33,6 +33,7 @@ func copyFile(fs afero.Fs, src, dst string) error {
defer func() { _ = destFile.Close() }()
_, err = io.Copy(destFile, sourceFile)
return err
}
@@ -46,6 +47,7 @@ func TestCleanSnapshotDBEmptySnapshot(t *testing.T) {
// Create a test database
tempDir := t.TempDir()
dbPath := filepath.Join(tempDir, "test.db")
db, err := database.New(ctx, dbPath)
if err != nil {
t.Fatalf("failed to create database: %v", err)
@@ -71,9 +73,11 @@ func TestCleanSnapshotDBEmptySnapshot(t *testing.T) {
chunk := &database.Chunk{ChunkHash: "orphan-chunk", Size: 500}
err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
if err := repos.Files.Create(ctx, tx, file); err != nil {
err := repos.Files.Create(ctx, tx, file)
if err != nil {
return err
}
return repos.Chunks.Create(ctx, tx, chunk)
})
if err != nil {
@@ -111,7 +115,8 @@ func TestCleanSnapshotDBEmptySnapshot(t *testing.T) {
t.Fatalf("failed to open cleaned database: %v", err)
}
defer func() {
if err := cleanedDB.Close(); err != nil {
err := cleanedDB.Close()
if err != nil {
t.Errorf("failed to close database: %v", err)
}
}()
@@ -123,6 +128,7 @@ func TestCleanSnapshotDBEmptySnapshot(t *testing.T) {
if err != nil {
t.Fatalf("failed to get snapshot: %v", err)
}
if verifySnapshot == nil {
t.Error("snapshot should exist")
}
@@ -132,6 +138,7 @@ func TestCleanSnapshotDBEmptySnapshot(t *testing.T) {
if err != nil {
t.Fatalf("failed to check file: %v", err)
}
if f != nil {
t.Error("orphan file should not exist")
}
@@ -141,6 +148,7 @@ func TestCleanSnapshotDBEmptySnapshot(t *testing.T) {
if err != nil {
t.Fatalf("failed to check chunk: %v", err)
}
if c != nil {
t.Error("orphan chunk should not exist")
}
@@ -156,6 +164,7 @@ func TestCleanSnapshotDBNonExistentSnapshot(t *testing.T) {
// Create a test database
tempDir := t.TempDir()
dbPath := filepath.Join(tempDir, "test.db")
db, err := database.New(ctx, dbPath)
if err != nil {
t.Fatalf("failed to create database: %v", err)

25
internal/storage/file.go
View File

@@ -102,26 +102,32 @@ func (f *FileStorer) PutWithProgress(ctx context.Context, key string, data io.Re
// Get retrieves data from the specified key.
func (f *FileStorer) Get(ctx context.Context, key string) (io.ReadCloser, error) {
path := f.fullPath(key)
file, err := f.fs.Open(path)
if err != nil {
if os.IsNotExist(err) {
return nil, ErrNotFound
}
return nil, fmt.Errorf("opening file: %w", err)
}
return file, nil
}
// Stat returns metadata about an object without retrieving its contents.
func (f *FileStorer) Stat(ctx context.Context, key string) (*ObjectInfo, error) {
path := f.fullPath(key)
info, err := f.fs.Stat(path)
if err != nil {
if os.IsNotExist(err) {
return nil, ErrNotFound
}
return nil, fmt.Errorf("stat file: %w", err)
}
return &ObjectInfo{
Key: key,
Size: info.Size(),
@@ -131,19 +137,23 @@ func (f *FileStorer) Stat(ctx context.Context, key string) (*ObjectInfo, error)
// Delete removes an object.
func (f *FileStorer) Delete(ctx context.Context, key string) error {
path := f.fullPath(key)
err := f.fs.Remove(path)
if os.IsNotExist(err) {
return nil // Match S3 behavior: no error if doesn't exist
}
if err != nil {
return fmt.Errorf("removing file: %w", err)
}
return nil
}
// List returns all keys with the given prefix.
func (f *FileStorer) List(ctx context.Context, prefix string) ([]string, error) {
var keys []string
basePath := f.fullPath(prefix)
// Check if base path exists
@@ -151,6 +161,7 @@ func (f *FileStorer) List(ctx context.Context, prefix string) ([]string, error)
if err != nil {
return nil, fmt.Errorf("checking path: %w", err)
}
if !exists {
return keys, nil // Empty list for non-existent prefix
}
@@ -177,9 +188,9 @@ func (f *FileStorer) List(ctx context.Context, prefix string) ([]string, error)
relPath = strings.ReplaceAll(relPath, string(filepath.Separator), "/")
keys = append(keys, relPath)
}
return nil
})
if err != nil {
return nil, fmt.Errorf("walking directory: %w", err)
}
@@ -192,14 +203,17 @@ func (f *FileStorer) ListStream(ctx context.Context, prefix string) <-chan Objec
ch := make(chan ObjectInfo)
go func() {
defer close(ch)
basePath := f.fullPath(prefix)
// Check if base path exists
exists, err := afero.Exists(f.fs, basePath)
if err != nil {
ch <- ObjectInfo{Err: fmt.Errorf("checking path: %w", err)}
return
}
if !exists {
return // Empty channel for non-existent prefix
}
@@ -209,12 +223,14 @@ func (f *FileStorer) ListStream(ctx context.Context, prefix string) <-chan Objec
select {
case <-ctx.Done():
ch <- ObjectInfo{Err: ctx.Err()}
return ctx.Err()
default:
}
if err != nil {
ch <- ObjectInfo{Err: err}
return nil // Continue walking despite errors
}
@@ -222,6 +238,7 @@ func (f *FileStorer) ListStream(ctx context.Context, prefix string) <-chan Objec
relPath, err := filepath.Rel(f.basePath, path)
if err != nil {
ch <- ObjectInfo{Err: fmt.Errorf("computing relative path: %w", err)}
return nil
}
// Normalize path separators
@@ -231,9 +248,11 @@ func (f *FileStorer) ListStream(ctx context.Context, prefix string) <-chan Objec
Size: info.Size(),
}
}
return nil
})
}()
return ch
}
@@ -257,10 +276,12 @@ func (pw *progressWriter) Write(p []byte) (int, error) {
if n > 0 {
pw.written += int64(n)
if pw.callback != nil {
if callbackErr := pw.callback(pw.written); callbackErr != nil {
callbackErr := pw.callback(pw.written)
if callbackErr != nil {
return n, callbackErr
}
}
}
return n, err
}

3
internal/storage/module.go
View File

@@ -24,6 +24,7 @@ func NewStorer(cfg *config.Config) (Storer, error) {
if cfg.StorageURL != "" {
return storerFromURL(cfg.StorageURL, cfg)
}
return storerFromLegacyS3Config(cfg)
}
@@ -71,6 +72,7 @@ func storerFromURL(rawURL string, cfg *config.Config) (Storer, error) {
if err != nil {
return nil, fmt.Errorf("creating S3 client: %w", err)
}
return NewS3Storer(client), nil
case "rclone":
@@ -109,5 +111,6 @@ func storerFromLegacyS3Config(cfg *config.Config) (Storer, error) {
if err != nil {
return nil, fmt.Errorf("creating S3 client: %w", err)
}
return NewS3Storer(client), nil
}

12
internal/storage/rclone.go
View File

@@ -49,6 +49,7 @@ func NewRcloneStorer(ctx context.Context, remote, path string) (*RcloneStorer, e
strings.Contains(err.Error(), "failed to find remote") {
return nil, fmt.Errorf("%w: %s", ErrRemoteNotFound, remote)
}
return nil, fmt.Errorf("creating rclone filesystem: %w", err)
}
@@ -101,9 +102,11 @@ func (r *RcloneStorer) Get(ctx context.Context, key string) (io.ReadCloser, erro
if errors.Is(err, fs.ErrorObjectNotFound) {
return nil, ErrNotFound
}
if errors.Is(err, fs.ErrorDirNotFound) {
return nil, ErrNotFound
}
return nil, fmt.Errorf("getting object: %w", err)
}
@@ -123,9 +126,11 @@ func (r *RcloneStorer) Stat(ctx context.Context, key string) (*ObjectInfo, error
if errors.Is(err, fs.ErrorObjectNotFound) {
return nil, ErrNotFound
}
if errors.Is(err, fs.ErrorDirNotFound) {
return nil, ErrNotFound
}
return nil, fmt.Errorf("getting object: %w", err)
}
@@ -142,9 +147,11 @@ func (r *RcloneStorer) Delete(ctx context.Context, key string) error {
if errors.Is(err, fs.ErrorObjectNotFound) {
return nil // Match S3 behavior: no error if doesn't exist
}
if errors.Is(err, fs.ErrorDirNotFound) {
return nil
}
return fmt.Errorf("getting object: %w", err)
}
@@ -209,6 +216,7 @@ func (r *RcloneStorer) Info() StorageInfo {
if r.path != "" {
location += ":" + r.path
}
return StorageInfo{
Type: "rclone",
Location: location,
@@ -227,10 +235,12 @@ func (pr *progressReader) Read(p []byte) (int, error) {
if n > 0 {
pr.read += int64(n)
if pr.callback != nil {
if callbackErr := pr.callback(pr.read); callbackErr != nil {
callbackErr := pr.callback(pr.read)
if callbackErr != nil {
return n, callbackErr
}
}
}
return n, err
}

4
internal/storage/s3.go
View File

@@ -30,6 +30,7 @@ func (s *S3Storer) PutWithProgress(ctx context.Context, key string, data io.Read
if progress != nil {
s3Progress = s3.ProgressCallback(progress)
}
return s.client.PutObjectWithProgress(ctx, key, data, size, s3Progress)
}
@@ -44,6 +45,7 @@ func (s *S3Storer) Stat(ctx context.Context, key string) (*ObjectInfo, error) {
if err != nil {
return nil, err
}
return &ObjectInfo{
Key: info.Key,
Size: info.Size,
@@ -65,6 +67,7 @@ func (s *S3Storer) ListStream(ctx context.Context, prefix string) <-chan ObjectI
ch := make(chan ObjectInfo)
go func() {
defer close(ch)
for info := range s.client.ListObjectsStream(ctx, prefix, false) {
ch <- ObjectInfo{
Key: info.Key,
@@ -73,6 +76,7 @@ func (s *S3Storer) ListStream(ctx context.Context, prefix string) <-chan ObjectI
}
}
}()
return ch
}

26
internal/storage/url.go
View File

@@ -1,6 +1,7 @@
package storage
import (
"errors"
"fmt"
"net/url"
"strings"
@@ -24,15 +25,16 @@ type StorageURL struct {
// - rclone://remote/path/to/backups
func ParseStorageURL(rawURL string) (*StorageURL, error) {
if rawURL == "" {
return nil, fmt.Errorf("storage URL is empty")
return nil, errors.New("storage URL is empty")
}
// Handle file:// URLs
if strings.HasPrefix(rawURL, "file://") {
path := strings.TrimPrefix(rawURL, "file://")
if after, ok := strings.CutPrefix(rawURL, "file://"); ok {
path := after
if path == "" {
return nil, fmt.Errorf("file URL path is empty")
return nil, errors.New("file URL path is empty")
}
return &StorageURL{
Scheme: "file",
Prefix: path,
@@ -48,12 +50,13 @@ func ParseStorageURL(rawURL string) (*StorageURL, error) {
bucket := u.Host
if bucket == "" {
return nil, fmt.Errorf("s3 URL missing bucket name")
return nil, errors.New("s3 URL missing bucket name")
}
prefix := strings.TrimPrefix(u.Path, "/")
query := u.Query()
useSSL := true
if query.Get("ssl") == "false" {
useSSL = false
@@ -78,7 +81,7 @@ func ParseStorageURL(rawURL string) (*StorageURL, error) {
remote := u.Host
if remote == "" {
return nil, fmt.Errorf("rclone URL missing remote name")
return nil, errors.New("rclone URL missing remote name")
}
path := strings.TrimPrefix(u.Path, "/")
@@ -90,29 +93,32 @@ func ParseStorageURL(rawURL string) (*StorageURL, error) {
}, nil
}
return nil, fmt.Errorf("unsupported URL scheme: must start with s3://, file://, or rclone://")
return nil, errors.New("unsupported URL scheme: must start with s3://, file://, or rclone://")
}
// String returns a human-readable representation of the storage URL.
func (u *StorageURL) String() string {
switch u.Scheme {
case "file":
return fmt.Sprintf("file://%s", u.Prefix)
return "file://" + u.Prefix
case "s3":
endpoint := u.Endpoint
if endpoint == "" {
endpoint = "s3.amazonaws.com"
}
if u.Prefix != "" {
return fmt.Sprintf("s3://%s/%s (endpoint: %s)", u.Bucket, u.Prefix, endpoint)
}
return fmt.Sprintf("s3://%s (endpoint: %s)", u.Bucket, endpoint)
case "rclone":
if u.Prefix != "" {
return fmt.Sprintf("rclone://%s/%s", u.RcloneRemote, u.Prefix)
}
return fmt.Sprintf("rclone://%s", u.RcloneRemote)
return "rclone://" + u.RcloneRemote
default:
return fmt.Sprintf("%s://?", u.Scheme)
return u.Scheme + "://?"
}
}

14
internal/types/types.go
View File

@@ -24,6 +24,7 @@ func ParseFileID(s string) (FileID, error) {
if err != nil {
return FileID{}, err
}
return FileID(id), nil
}
@@ -38,13 +39,15 @@ func (id FileID) Value() (driver.Value, error) {
}
// Scan implements sql.Scanner for database deserialization.
func (id *FileID) Scan(src interface{}) error {
func (id *FileID) Scan(src any) error {
if src == nil {
*id = FileID{}
return nil
}
var s string
switch v := src.(type) {
case string:
s = v
@@ -58,7 +61,9 @@ func (id *FileID) Scan(src interface{}) error {
if err != nil {
return fmt.Errorf("invalid FileID: %w", err)
}
*id = FileID(parsed)
return nil
}
@@ -77,6 +82,7 @@ func ParseBlobID(s string) (BlobID, error) {
if err != nil {
return BlobID{}, err
}
return BlobID(id), nil
}
@@ -91,13 +97,15 @@ func (id BlobID) Value() (driver.Value, error) {
}
// Scan implements sql.Scanner for database deserialization.
func (id *BlobID) Scan(src interface{}) error {
func (id *BlobID) Scan(src any) error {
if src == nil {
*id = BlobID{}
return nil
}
var s string
switch v := src.(type) {
case string:
s = v
@@ -111,7 +119,9 @@ func (id *BlobID) Scan(src interface{}) error {
if err != nil {
return fmt.Errorf("invalid BlobID: %w", err)
}
*id = BlobID(parsed)
return nil
}

57
internal/ui/ui.go
View File

@@ -94,10 +94,12 @@ func shouldColor(w io.Writer) bool {
if os.Getenv("NO_COLOR") != "" {
return false
}
f, ok := w.(*os.File)
if !ok {
return false
}
return term.IsTerminal(int(f.Fd()))
}
@@ -106,67 +108,73 @@ func (w *Writer) paint(color, s string) string {
if !w.color {
return s
}
return color + s + ansiReset
}
// ───────────────────────── message methods ─────────────────────────
// Begin prints an operation-start line, left-aligned with a white marker.
func (w *Writer) Begin(format string, args ...any) {
// Beginf prints an operation-start line, left-aligned with a white marker.
func (w *Writer) Beginf(format string, args ...any) {
if w.quiet {
return
}
w.emit(ansiWhite, Marker, "", format, args)
}
// Complete prints an operation-completion line in green, left-aligned.
func (w *Writer) Complete(format string, args ...any) {
// Completef prints an operation-completion line in green, left-aligned.
func (w *Writer) Completef(format string, args ...any) {
if w.quiet {
return
}
w.emit(ansiGreen, Marker, ansiGreen, format, args)
}
// Info prints a neutral status line, left-aligned with a white marker.
func (w *Writer) Info(format string, args ...any) {
// Infof prints a neutral status line, left-aligned with a white marker.
func (w *Writer) Infof(format string, args ...any) {
if w.quiet {
return
}
w.emit(ansiWhite, Marker, "", format, args)
}
// Notice prints an attention-worthy informational line, marker in cyan.
func (w *Writer) Notice(format string, args ...any) {
// Noticef prints an attention-worthy informational line, marker in cyan.
func (w *Writer) Noticef(format string, args ...any) {
if w.quiet {
return
}
w.emit(ansiCyan, Marker, "", format, args)
}
// Warning prints "⚠️ Warning: " in orange/yellow followed by the message.
func (w *Writer) Warning(format string, args ...any) {
// Warningf prints "⚠️ Warning: " in orange/yellow followed by the message.
func (w *Writer) Warningf(format string, args ...any) {
w.warnings++
prefix := "⚠️ " + w.paint(ansiYellow+ansiBold, "Warning: ")
_, _ = fmt.Fprintln(w.out, prefix+fmt.Sprintf(format, args...))
}
// Error prints "🛑 ERROR: " in red followed by the message. Goes to the
// Errorf prints "🛑 ERROR: " in red followed by the message. Goes to the
// same writer as everything else; callers that want stderr should
// construct a separate Writer for it.
func (w *Writer) Error(format string, args ...any) {
func (w *Writer) Errorf(format string, args ...any) {
w.errors++
prefix := "🛑 " + w.paint(ansiRed+ansiBold, "ERROR: ")
_, _ = fmt.Fprintln(w.out, prefix+fmt.Sprintf(format, args...))
}
// Detail prints an indented continuation line under a preceding Complete
// Detailf prints an indented continuation line under a preceding Completef
// (or other top-level message). Marker " 》" (white) at column 2.
// Distinct from Progress (semantically a "heartbeat") in usage but
// Distinct from Progressf (semantically a "heartbeat") in usage but
// visually identical.
func (w *Writer) Detail(format string, args ...any) {
func (w *Writer) Detailf(format string, args ...any) {
if w.quiet {
return
}
w.emit(ansiWhite, " "+Marker, "", format, args)
}
@@ -176,24 +184,27 @@ func (w *Writer) WarningCount() int { return w.warnings }
// ErrorCount returns the number of Error() calls this writer has emitted.
func (w *Writer) ErrorCount() int { return w.errors }
// Progress prints an indented heartbeat / per-item update, marker in white.
func (w *Writer) Progress(format string, args ...any) {
// Progressf prints an indented heartbeat / per-item update, marker in white.
func (w *Writer) Progressf(format string, args ...any) {
if w.quiet {
return
}
w.emit(ansiWhite, " "+Marker, "", format, args)
}
// Banner prints a line with no marker, left-aligned. Bold when color
// Bannerf prints a line with no marker, left-aligned. Bold when color
// is enabled. Used for the application startup banner only.
func (w *Writer) Banner(format string, args ...any) {
func (w *Writer) Bannerf(format string, args ...any) {
if w.quiet {
return
}
body := fmt.Sprintf(format, args...)
if w.color {
body = ansiBold + body + ansiReset
}
_, _ = fmt.Fprintln(w.out, body)
}
@@ -204,6 +215,7 @@ func (w *Writer) emit(prefixColor, prefix, bodyColor, format string, args []any)
if bodyColor != "" {
body = w.paint(bodyColor, body)
}
_, _ = fmt.Fprintln(w.out, w.paint(prefixColor, prefix)+" "+body)
}
@@ -219,6 +231,7 @@ func (w *Writer) Hex(s string) string {
if len(s) > 12 {
short = s[:12] + "..."
}
return w.paint(ansiCyan, short)
}
@@ -244,8 +257,11 @@ func (w *Writer) Speed(bytesPerSec float64) string {
if bytesPerSec <= 0 {
return w.paint(ansiMagenta, "N/A")
}
bitsPerSec := bytesPerSec * 8
var s string
switch {
case bitsPerSec >= 1e9:
s = fmt.Sprintf("%.1f Gbit/sec", bitsPerSec/1e9)
@@ -256,6 +272,7 @@ func (w *Writer) Speed(bytesPerSec float64) string {
default:
s = fmt.Sprintf("%.0f bit/sec", bitsPerSec)
}
return w.paint(ansiMagenta, s)
}
@@ -270,10 +287,12 @@ func (w *Writer) Duration(d time.Duration) string {
// displayed in the process's local zone.
func (w *Writer) Time(t time.Time) string {
t = t.Local()
now := time.Now()
if t.Year() == now.Year() && t.YearDay() == now.YearDay() {
return w.paint(ansiYellow, t.Format("15:04:05"))
}
return w.paint(ansiYellow, t.Format("2006-01-02 15:04:05"))
}

48
internal/ui/ui_test.go
View File

@@ -9,6 +9,7 @@ import (
func newTestWriter(color bool) (*Writer, *bytes.Buffer) {
buf := &bytes.Buffer{}
return NewWithColor(buf, color), buf
}
@@ -18,21 +19,22 @@ func TestMessageMethodsPlain(t *testing.T) {
fn func(*Writer)
want string
}{
{"Begin", func(w *Writer) { w.Begin("starting %s", "thing") }, "》 starting thing\n"},
{"Complete", func(w *Writer) { w.Complete("done %s", "thing") }, "》 done thing\n"},
{"Info", func(w *Writer) { w.Info("status") }, "》 status\n"},
{"Notice", func(w *Writer) { w.Notice("note") }, "》 note\n"},
{"Warning", func(w *Writer) { w.Warning("oops") }, "⚠️ Warning: oops\n"},
{"Error", func(w *Writer) { w.Error("boom") }, "🛑 ERROR: boom\n"},
{"Progress", func(w *Writer) { w.Progress("p") }, " 》 p\n"},
{"Detail", func(w *Writer) { w.Detail("d") }, " 》 d\n"},
{"Banner", func(w *Writer) { w.Banner("hello") }, "hello\n"}, // plain mode, no bold
{"Begin", func(w *Writer) { w.Beginf("starting %s", "thing") }, "》 starting thing\n"},
{"Complete", func(w *Writer) { w.Completef("done %s", "thing") }, "》 done thing\n"},
{"Info", func(w *Writer) { w.Infof("status") }, "》 status\n"},
{"Notice", func(w *Writer) { w.Noticef("note") }, "》 note\n"},
{"Warning", func(w *Writer) { w.Warningf("oops") }, "⚠️ Warning: oops\n"},
{"Error", func(w *Writer) { w.Errorf("boom") }, "🛑 ERROR: boom\n"},
{"Progress", func(w *Writer) { w.Progressf("p") }, " 》 p\n"},
{"Detail", func(w *Writer) { w.Detailf("d") }, " 》 d\n"},
{"Banner", func(w *Writer) { w.Bannerf("hello") }, "hello\n"}, // plain mode, no bold
}
for _, tt := range tests {
t.Run(tt.method, func(t *testing.T) {
w, buf := newTestWriter(false)
tt.fn(w)
if got := buf.String(); got != tt.want {
t.Errorf("got %q, want %q", got, tt.want)
}
@@ -45,13 +47,16 @@ func TestWarningErrorCounters(t *testing.T) {
if w.WarningCount() != 0 || w.ErrorCount() != 0 {
t.Fatalf("expected fresh writer to have zero counts")
}
w.Info("normal")
w.Warning("first warn")
w.Warning("second warn")
w.Error("only error")
w.Infof("normal")
w.Warningf("first warn")
w.Warningf("second warn")
w.Errorf("only error")
if got, want := w.WarningCount(), 2; got != want {
t.Errorf("WarningCount: got %d, want %d", got, want)
}
if got, want := w.ErrorCount(), 1; got != want {
t.Errorf("ErrorCount: got %d, want %d", got, want)
}
@@ -59,11 +64,13 @@ func TestWarningErrorCounters(t *testing.T) {
func TestColorOutputContainsANSI(t *testing.T) {
w, buf := newTestWriter(true)
w.Error("boom")
w.Errorf("boom")
out := buf.String()
if !strings.Contains(out, "\033[") {
t.Errorf("expected ANSI escapes in color output, got %q", out)
}
if !strings.Contains(out, "ERROR: ") {
t.Errorf("expected 'ERROR: ' text in output, got %q", out)
}
@@ -71,7 +78,8 @@ func TestColorOutputContainsANSI(t *testing.T) {
func TestBannerBoldWhenColor(t *testing.T) {
w, buf := newTestWriter(true)
w.Banner("hello")
w.Bannerf("hello")
out := buf.String()
if !strings.Contains(out, "\033[1m") {
t.Errorf("expected bold ANSI escape in colored Banner output, got %q", out)
@@ -84,18 +92,23 @@ func TestValueFormattersPlain(t *testing.T) {
if got := w.Hex("0123456789abcdef0123"); got != "0123456789ab..." {
t.Errorf("Hex long: got %q", got)
}
if got := w.Hex("short"); got != "short" {
t.Errorf("Hex short: got %q", got)
}
if got := w.Size(1024); got != "1.0 kB" {
t.Errorf("Size: got %q", got)
}
if got := w.Duration(90 * time.Second); got != "1m30s" {
t.Errorf("Duration: got %q", got)
}
if got := w.Count(12345); got != "12,345" {
t.Errorf("Count: got %q", got)
}
if got := w.Percent(12.34); got != "12.3%" {
t.Errorf("Percent: got %q", got)
}
@@ -104,9 +117,11 @@ func TestValueFormattersPlain(t *testing.T) {
if got := w.Speed(0); got != "N/A" {
t.Errorf("Speed(0): got %q, want N/A", got)
}
if got := w.Speed(125_000_000); got != "1.0 Gbit/sec" { // 1 Gbit/s = 125 MB/s
t.Errorf("Speed(125e6): got %q", got)
}
if got := w.Speed(125_000); got != "1 Mbit/sec" {
t.Errorf("Speed(125e3): got %q", got)
}
@@ -116,6 +131,7 @@ func TestValueFormattersPlain(t *testing.T) {
if got := w.Time(today); got != "14:30:45" {
t.Errorf("Time today: got %q, want 14:30:45", got)
}
other := time.Date(2030, 1, 2, 3, 4, 5, 0, time.Local)
if got := w.Time(other); got != "2030-01-02 03:04:05" {
t.Errorf("Time other day: got %q", got)
@@ -124,10 +140,12 @@ func TestValueFormattersPlain(t *testing.T) {
func TestValueFormattersColored(t *testing.T) {
w, _ := newTestWriter(true)
hex := w.Hex("0123456789abcdef0123")
if !strings.Contains(hex, "\033[") {
t.Errorf("expected ANSI in colored Hex output, got %q", hex)
}
if !strings.Contains(hex, "0123456789ab") {
t.Errorf("expected hex content in output, got %q", hex)
}

10
internal/vaultik/blob_fetch.go
View File

@@ -4,6 +4,7 @@ import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io"
"time"
@@ -26,9 +27,10 @@ type hashVerifyReader struct {
func (h *hashVerifyReader) Read(p []byte) (int, error) {
n, err := h.reader.Read(p)
if err == io.EOF {
if errors.Is(err, io.EOF) {
h.done = true
}
return n, err
}
@@ -41,6 +43,7 @@ func (h *hashVerifyReader) Close() error {
firstHash := h.reader.Sum256()
secondHasher := sha256.New()
secondHasher.Write(firstHash)
actualHashHex := hex.EncodeToString(secondHasher.Sum(nil))
if actualHashHex != h.blobHash {
return fmt.Errorf("blob hash mismatch: expected %s, got %s", h.blobHash[:16], actualHashHex[:16])
@@ -50,6 +53,7 @@ func (h *hashVerifyReader) Close() error {
if readerErr != nil {
return readerErr
}
return fetcherErr
}
@@ -66,6 +70,7 @@ func (v *Vaultik) FetchAndDecryptBlob(ctx context.Context, blobHash string, expe
reader, err := blobgen.NewReader(rc, identity)
if err != nil {
_ = rc.Close()
return nil, fmt.Errorf("creating blob reader: %w", err)
}
@@ -86,6 +91,7 @@ func (v *Vaultik) FetchBlob(ctx context.Context, blobHash string, expectedSize i
t0 := time.Now()
rc, err := v.Storage.Get(ctx, blobPath)
getDur := time.Since(t0)
if err != nil {
return nil, 0, fmt.Errorf("downloading blob %s: %w", blobHash[:16], err)
}
@@ -93,8 +99,10 @@ func (v *Vaultik) FetchBlob(ctx context.Context, blobHash string, expectedSize i
t0 = time.Now()
info, err := v.Storage.Stat(ctx, blobPath)
statDur := time.Since(t0)
if err != nil {
_ = rc.Close()
return nil, 0, fmt.Errorf("stat blob %s: %w", blobHash[:16], err)
}

12
internal/vaultik/blob_fetch_hash_test.go
View File

@@ -24,17 +24,22 @@ func TestFetchAndDecryptBlobVerifiesHash(t *testing.T) {
// Create test data and encrypt it using blobgen.Writer
plaintext := []byte("hello world test data for blob hash verification")
var encBuf bytes.Buffer
writer, err := blobgen.NewWriter(&encBuf, 1, []string{identity.Recipient().String()})
if err != nil {
t.Fatalf("creating blobgen writer: %v", err)
}
if _, err := writer.Write(plaintext); err != nil {
t.Fatalf("writing plaintext: %v", err)
}
if err := writer.Close(); err != nil {
t.Fatalf("closing writer: %v", err)
}
encryptedData := encBuf.Bytes()
// Compute correct double-SHA-256 hash of the plaintext (matches blobgen.Writer.Sum256)
@@ -51,6 +56,7 @@ func TestFetchAndDecryptBlobVerifiesHash(t *testing.T) {
// Set up mock storage with the blob at the correct path
mockStorage := NewMockStorer()
blobPath := "blobs/" + correctHash[:2] + "/" + correctHash[2:4] + "/" + correctHash
mockStorage.mu.Lock()
mockStorage.data[blobPath] = encryptedData
mockStorage.mu.Unlock()
@@ -63,13 +69,16 @@ func TestFetchAndDecryptBlobVerifiesHash(t *testing.T) {
if err != nil {
t.Fatalf("expected success, got error: %v", err)
}
data, err := io.ReadAll(rc)
if err != nil {
t.Fatalf("reading stream: %v", err)
}
if err := rc.Close(); err != nil {
t.Fatalf("close (hash verification) failed: %v", err)
}
if !bytes.Equal(data, plaintext) {
t.Fatalf("decrypted data mismatch: got %q, want %q", data, plaintext)
}
@@ -79,6 +88,7 @@ func TestFetchAndDecryptBlobVerifiesHash(t *testing.T) {
// Use a fake hash that doesn't match the actual plaintext
fakeHash := strings.Repeat("ab", 32) // 64 hex chars
fakePath := "blobs/" + fakeHash[:2] + "/" + fakeHash[2:4] + "/" + fakeHash
mockStorage.mu.Lock()
mockStorage.data[fakePath] = encryptedData
mockStorage.mu.Unlock()
@@ -89,10 +99,12 @@ func TestFetchAndDecryptBlobVerifiesHash(t *testing.T) {
}
// Read all data — hash is verified on Close
_, _ = io.ReadAll(rc)
err = rc.Close()
if err == nil {
t.Fatal("expected error for mismatched hash, got nil")
}
if !strings.Contains(err.Error(), "hash mismatch") {
t.Fatalf("expected hash mismatch error, got: %v", err)
}

39
internal/vaultik/blobcache.go
View File

@@ -53,6 +53,7 @@ func newBlobDiskCache(maxBytes int64) (*blobDiskCache, error) {
if err != nil {
return nil, fmt.Errorf("creating blob cache dir: %w", err)
}
return &blobDiskCache{
dir: dir,
maxBytes: maxBytes,
@@ -70,21 +71,25 @@ func (c *blobDiskCache) unlink(e *blobDiskCacheEntry) {
} else {
c.head = e.next
}
if e.next != nil {
e.next.prev = e.prev
} else {
c.tail = e.prev
}
e.prev = nil
e.next = nil
}
func (c *blobDiskCache) pushFront(e *blobDiskCacheEntry) {
e.prev = nil
e.next = c.head
if c.head != nil {
c.head.prev = e
}
c.head = e
if c.tail == nil {
c.tail = e
@@ -95,6 +100,7 @@ func (c *blobDiskCache) evictLRU() {
if c.tail == nil {
return
}
victim := c.tail
c.unlink(victim)
delete(c.items, victim.key)
@@ -121,7 +127,8 @@ func (c *blobDiskCache) Put(key string, data []byte) error {
delete(c.items, key)
}
if err := os.WriteFile(c.path(key), data, 0600); err != nil {
err := os.WriteFile(c.path(key), data, 0600)
if err != nil {
return fmt.Errorf("writing blob to cache: %w", err)
}
@@ -163,14 +170,19 @@ func (c *blobDiskCache) PutFromReader(key string, r io.Reader) (int64, error) {
if err != nil {
return 0, fmt.Errorf("creating cache file: %w", err)
}
written, copyErr := io.Copy(f, r)
closeErr := f.Close()
if copyErr != nil {
_ = os.Remove(c.path(key))
return written, fmt.Errorf("streaming to cache file: %w", copyErr)
}
if closeErr != nil {
_ = os.Remove(c.path(key))
return written, fmt.Errorf("closing cache file: %w", closeErr)
}
@@ -182,6 +194,7 @@ func (c *blobDiskCache) PutFromReader(key string, r io.Reader) (int64, error) {
// so this branch is effectively unreachable there.
if written > c.maxBytes {
_ = os.Remove(c.path(key))
return written, nil
}
@@ -205,11 +218,14 @@ func (c *blobDiskCache) PutFromReader(key string, r io.Reader) (int64, error) {
func (c *blobDiskCache) Get(key string) ([]byte, bool) {
c.mu.Lock()
c.getCalls++
e, ok := c.items[key]
if !ok {
c.mu.Unlock()
return nil, false
}
c.unlink(e)
c.pushFront(e)
c.mu.Unlock()
@@ -223,8 +239,10 @@ func (c *blobDiskCache) Get(key string) ([]byte, bool) {
c.curBytes -= e.size
}
c.mu.Unlock()
return nil, false
}
return data, true
}
@@ -232,15 +250,20 @@ func (c *blobDiskCache) Get(key string) ([]byte, bool) {
func (c *blobDiskCache) ReadAt(key string, offset, length int64) ([]byte, error) {
c.mu.Lock()
c.readAtCalls++
e, ok := c.items[key]
if !ok {
c.mu.Unlock()
return nil, fmt.Errorf("key %q not in cache", key)
}
if offset+length > e.size {
c.mu.Unlock()
return nil, fmt.Errorf("read beyond blob size: offset=%d length=%d size=%d", offset, length, e.size)
}
c.unlink(e)
c.pushFront(e)
c.mu.Unlock()
@@ -255,6 +278,7 @@ func (c *blobDiskCache) ReadAt(key string, offset, length int64) ([]byte, error)
if _, err := f.ReadAt(buf, offset); err != nil {
return nil, err
}
return buf, nil
}
@@ -262,7 +286,9 @@ func (c *blobDiskCache) ReadAt(key string, offset, length int64) ([]byte, error)
func (c *blobDiskCache) Has(key string) bool {
c.mu.Lock()
defer c.mu.Unlock()
_, ok := c.items[key]
return ok
}
@@ -272,10 +298,12 @@ func (c *blobDiskCache) Has(key string) bool {
func (c *blobDiskCache) Delete(key string) {
c.mu.Lock()
defer c.mu.Unlock()
e, ok := c.items[key]
if !ok {
return
}
c.unlink(e)
delete(c.items, key)
c.curBytes -= e.size
@@ -287,10 +315,12 @@ func (c *blobDiskCache) Delete(key string) {
func (c *blobDiskCache) Keys() []string {
c.mu.Lock()
defer c.mu.Unlock()
keys := make([]string, 0, len(c.items))
for k := range c.items {
keys = append(keys, k)
}
return keys
}
@@ -298,6 +328,7 @@ func (c *blobDiskCache) Keys() []string {
func (c *blobDiskCache) Size() int64 {
c.mu.Lock()
defer c.mu.Unlock()
return c.curBytes
}
@@ -305,6 +336,7 @@ func (c *blobDiskCache) Size() int64 {
func (c *blobDiskCache) Len() int {
c.mu.Lock()
defer c.mu.Unlock()
return len(c.items)
}
@@ -312,6 +344,7 @@ func (c *blobDiskCache) Len() int {
func (c *blobDiskCache) GetCalls() int {
c.mu.Lock()
defer c.mu.Unlock()
return c.getCalls
}
@@ -319,6 +352,7 @@ func (c *blobDiskCache) GetCalls() int {
func (c *blobDiskCache) ReadAtCalls() int {
c.mu.Lock()
defer c.mu.Unlock()
return c.readAtCalls
}
@@ -327,6 +361,7 @@ func (c *blobDiskCache) ReadAtCalls() int {
func (c *blobDiskCache) PeakLen() int {
c.mu.Lock()
defer c.mu.Unlock()
return c.peakLen
}
@@ -334,9 +369,11 @@ func (c *blobDiskCache) PeakLen() int {
func (c *blobDiskCache) Close() error {
c.mu.Lock()
defer c.mu.Unlock()
c.items = nil
c.head = nil
c.tail = nil
c.curBytes = 0
return os.RemoveAll(c.dir)
}

20
internal/vaultik/blobcache_test.go
View File

@@ -23,6 +23,7 @@ func TestBlobDiskCache_BasicGetPut(t *testing.T) {
if !ok {
t.Fatal("expected cache hit")
}
if !bytes.Equal(got, data) {
t.Fatalf("got %q, want %q", got, data)
}
@@ -35,15 +36,19 @@ func TestBlobDiskCache_BasicGetPut(t *testing.T) {
func TestBlobDiskCache_EvictionUnderPressure(t *testing.T) {
maxBytes := int64(1000)
cache, err := newBlobDiskCache(maxBytes)
if err != nil {
t.Fatal(err)
}
defer func() { _ = cache.Close() }()
for i := 0; i < 5; i++ {
for i := range 5 {
data := make([]byte, 300)
if err := cache.Put(fmt.Sprintf("key%d", i), data); err != nil {
err := cache.Put(fmt.Sprintf("key%d", i), data)
if err != nil {
t.Fatal(err)
}
}
@@ -55,6 +60,7 @@ func TestBlobDiskCache_EvictionUnderPressure(t *testing.T) {
if !cache.Has("key4") {
t.Fatal("expected key4 to be cached")
}
if cache.Has("key0") {
t.Fatal("expected key0 to be evicted")
}
@@ -87,6 +93,7 @@ func TestBlobDiskCache_UpdateInPlace(t *testing.T) {
if err := cache.Put("key1", []byte("v1")); err != nil {
t.Fatal(err)
}
if err := cache.Put("key1", []byte("version2")); err != nil {
t.Fatal(err)
}
@@ -95,12 +102,15 @@ func TestBlobDiskCache_UpdateInPlace(t *testing.T) {
if !ok {
t.Fatal("expected hit")
}
if string(got) != "version2" {
t.Fatalf("got %q, want %q", got, "version2")
}
if cache.Len() != 1 {
t.Fatalf("expected 1 entry, got %d", cache.Len())
}
if cache.Size() != int64(len("version2")) {
t.Fatalf("expected size %d, got %d", len("version2"), cache.Size())
}
@@ -117,6 +127,7 @@ func TestBlobDiskCache_ReadAt(t *testing.T) {
if _, err := rand.Read(data); err != nil {
t.Fatal(err)
}
if err := cache.Put("blob1", data); err != nil {
t.Fatal(err)
}
@@ -125,6 +136,7 @@ func TestBlobDiskCache_ReadAt(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(chunk, data[100:300]) {
t.Fatal("ReadAt returned wrong data")
}
@@ -149,6 +161,7 @@ func TestBlobDiskCache_Close(t *testing.T) {
if err := cache.Put("key1", []byte("data")); err != nil {
t.Fatal(err)
}
if err := cache.Close(); err != nil {
t.Fatal(err)
}
@@ -165,6 +178,7 @@ func TestBlobDiskCache_LRUOrder(t *testing.T) {
if err := cache.Put("a", d); err != nil {
t.Fatal(err)
}
if err := cache.Put("b", d); err != nil {
t.Fatal(err)
}
@@ -180,9 +194,11 @@ func TestBlobDiskCache_LRUOrder(t *testing.T) {
if !cache.Has("a") {
t.Fatal("expected 'a' to survive")
}
if !cache.Has("c") {
t.Fatal("expected 'c' to be present")
}
if cache.Has("b") {
t.Fatal("expected 'b' to be evicted")
}

10
internal/vaultik/helpers.go
View File

@@ -1,6 +1,7 @@
package vaultik
import (
"errors"
"fmt"
"regexp"
"strconv"
@@ -29,11 +30,13 @@ func formatBytes(bytes int64) string {
if bytes < unit {
return fmt.Sprintf("%d B", bytes)
}
div, exp := int64(unit), 0
for n := bytes / unit; n >= unit; n /= unit {
div *= unit
exp++
}
return fmt.Sprintf("%.1f %cB", float64(bytes)/float64(div), "KMGTPE"[exp])
}
@@ -42,11 +45,12 @@ func formatBytes(bytes int64) string {
func parseSnapshotTimestamp(snapshotID string) (time.Time, error) {
parts := strings.Split(snapshotID, "_")
if len(parts) < 2 {
return time.Time{}, fmt.Errorf("invalid snapshot ID format: expected hostname_snapshotname_timestamp")
return time.Time{}, errors.New("invalid snapshot ID format: expected hostname_snapshotname_timestamp")
}
// Last part is the RFC3339 timestamp
timestampStr := parts[len(parts)-1]
timestamp, err := time.Parse(time.RFC3339, timestampStr)
if err != nil {
return time.Time{}, fmt.Errorf("invalid timestamp: %w", err)
@@ -80,17 +84,20 @@ func parseDuration(s string) (time.Duration, error) {
}
re := regexp.MustCompile(`(\d+)\s*([a-zA-Z]+)`)
matches := re.FindAllStringSubmatch(s, -1)
if len(matches) == 0 {
return 0, fmt.Errorf("invalid duration: %q", s)
}
var total time.Duration
for _, match := range matches {
n, err := strconv.Atoi(match[1])
if err != nil {
return 0, fmt.Errorf("invalid number %q: %w", match[1], err)
}
unit := strings.ToLower(match[2])
switch unit {
case "d", "day", "days":
@@ -105,5 +112,6 @@ func parseDuration(s string) (time.Duration, error) {
return 0, fmt.Errorf("unknown time unit %q", unit)
}
}
return total, nil
}

3
internal/vaultik/helpers_test.go
View File

@@ -61,11 +61,14 @@ func TestParseDuration(t *testing.T) {
if err == nil {
t.Fatalf("expected error for %q, got %v", tt.input, got)
}
return
}
if err != nil {
t.Fatalf("unexpected error for %q: %v", tt.input, err)
}
if got != tt.want {
t.Errorf("parseDuration(%q) = %v, want %v", tt.input, got, tt.want)
}

149
internal/vaultik/info.go
View File

@@ -15,102 +15,122 @@ import (
// ShowInfo displays system and configuration information
func (v *Vaultik) ShowInfo() error {
// System Information
v.printfStdout("=== System Information ===\n")
v.printfStdout("OS/Architecture: %s/%s\n", runtime.GOOS, runtime.GOARCH)
v.printfStdout("Version: %s\n", v.Globals.Version)
v.printfStdout("Commit: %s\n", v.Globals.Commit)
v.printfStdout("Go Version: %s\n", runtime.Version())
v.stdoutf("=== System Information ===\n")
v.stdoutf("OS/Architecture: %s/%s\n", runtime.GOOS, runtime.GOARCH)
v.stdoutf("Version: %s\n", v.Globals.Version)
v.stdoutf("Commit: %s\n", v.Globals.Commit)
v.stdoutf("Go Version: %s\n", runtime.Version())
v.printlnStdout()
// Storage Configuration. The backend is selected by storage_url
// (s3://, file://, rclone://); the legacy s3.* fields are only
// printed when they're actually populated, since the URL scheme
// is the primary configuration.
v.printfStdout("=== Storage Configuration ===\n")
v.stdoutf("=== Storage Configuration ===\n")
storageInfo := v.Storage.Info()
v.printfStdout("Type: %s\n", storageInfo.Type)
v.printfStdout("Location: %s\n", storageInfo.Location)
v.stdoutf("Type: %s\n", storageInfo.Type)
v.stdoutf("Location: %s\n", storageInfo.Location)
if v.Config.StorageURL != "" {
v.printfStdout("Storage URL: %s\n", v.Config.StorageURL)
v.stdoutf("Storage URL: %s\n", v.Config.StorageURL)
}
if v.Config.S3.Bucket != "" {
v.printfStdout("S3 Bucket: %s\n", v.Config.S3.Bucket)
v.stdoutf("S3 Bucket: %s\n", v.Config.S3.Bucket)
}
if v.Config.S3.Prefix != "" {
v.printfStdout("S3 Prefix: %s\n", v.Config.S3.Prefix)
v.stdoutf("S3 Prefix: %s\n", v.Config.S3.Prefix)
}
if v.Config.S3.Endpoint != "" {
v.printfStdout("S3 Endpoint: %s\n", v.Config.S3.Endpoint)
v.stdoutf("S3 Endpoint: %s\n", v.Config.S3.Endpoint)
}
if v.Config.S3.Region != "" {
v.printfStdout("S3 Region: %s\n", v.Config.S3.Region)
v.stdoutf("S3 Region: %s\n", v.Config.S3.Region)
}
v.printlnStdout()
// Backup Settings
v.printfStdout("=== Backup Settings ===\n")
v.stdoutf("=== Backup Settings ===\n")
// Show configured snapshots
v.printfStdout("Snapshots:\n")
v.stdoutf("Snapshots:\n")
for _, name := range v.Config.SnapshotNames() {
snap := v.Config.Snapshots[name]
v.printfStdout(" %s:\n", name)
v.stdoutf(" %s:\n", name)
for _, path := range snap.Paths {
v.printfStdout(" - %s\n", path)
v.stdoutf(" - %s\n", path)
}
if len(snap.Exclude) > 0 {
v.printfStdout(" exclude: %s\n", strings.Join(snap.Exclude, ", "))
v.stdoutf(" exclude: %s\n", strings.Join(snap.Exclude, ", "))
}
}
// Global exclude patterns
if len(v.Config.Exclude) > 0 {
v.printfStdout("Global Exclude: %s\n", strings.Join(v.Config.Exclude, ", "))
v.stdoutf("Global Exclude: %s\n", strings.Join(v.Config.Exclude, ", "))
}
v.printfStdout("Compression: zstd level %d\n", v.Config.CompressionLevel)
v.printfStdout("Chunk Size: %s\n", humanize.Bytes(uint64(v.Config.ChunkSize)))
v.printfStdout("Blob Size Limit: %s\n", humanize.Bytes(uint64(v.Config.BlobSizeLimit)))
v.stdoutf("Compression: zstd level %d\n", v.Config.CompressionLevel)
v.stdoutf("Chunk Size: %s\n", humanize.Bytes(uint64(v.Config.ChunkSize)))
v.stdoutf("Blob Size Limit: %s\n", humanize.Bytes(uint64(v.Config.BlobSizeLimit)))
v.printlnStdout()
// Encryption Configuration
v.printfStdout("=== Encryption Configuration ===\n")
v.printfStdout("Recipients:\n")
v.stdoutf("=== Encryption Configuration ===\n")
v.stdoutf("Recipients:\n")
for _, recipient := range v.Config.AgeRecipients {
v.printfStdout(" - %s\n", recipient)
v.stdoutf(" - %s\n", recipient)
}
v.printlnStdout()
// Local Database
v.printfStdout("=== Local Database ===\n")
v.printfStdout("Index Path: %s\n", v.Config.IndexPath)
v.stdoutf("=== Local Database ===\n")
v.stdoutf("Index Path: %s\n", v.Config.IndexPath)
// Check if index file exists and get its size
if info, err := v.Fs.Stat(v.Config.IndexPath); err == nil {
v.printfStdout("Index Size: %s\n", humanize.Bytes(uint64(info.Size())))
v.stdoutf("Index Size: %s\n", humanize.Bytes(uint64(info.Size())))
// Get snapshot count from database
query := `SELECT COUNT(*) FROM snapshots WHERE completed_at IS NOT NULL`
var snapshotCount int
if err := v.DB.Conn().QueryRowContext(v.ctx, query).Scan(&snapshotCount); err == nil {
v.printfStdout("Snapshots: %d\n", snapshotCount)
err := v.DB.Conn().QueryRowContext(v.ctx, query).Scan(&snapshotCount)
if err == nil {
v.stdoutf("Snapshots: %d\n", snapshotCount)
}
// Get blob count from database
query = `SELECT COUNT(*) FROM blobs`
var blobCount int
if err := v.DB.Conn().QueryRowContext(v.ctx, query).Scan(&blobCount); err == nil {
v.printfStdout("Blobs: %d\n", blobCount)
err = v.DB.Conn().QueryRowContext(v.ctx, query).Scan(&blobCount)
if err == nil {
v.stdoutf("Blobs: %d\n", blobCount)
}
// Get file count from database
query = `SELECT COUNT(*) FROM files`
var fileCount int
if err := v.DB.Conn().QueryRowContext(v.ctx, query).Scan(&fileCount); err == nil {
v.printfStdout("Files: %d\n", fileCount)
err = v.DB.Conn().QueryRowContext(v.ctx, query).Scan(&fileCount)
if err == nil {
v.stdoutf("Files: %d\n", fileCount)
}
} else {
v.printfStdout("Index Size: (not created)\n")
v.stdoutf("Index Size: (not created)\n")
}
return nil
@@ -153,6 +173,7 @@ type RemoteInfoResult struct {
// RemoteInfo displays information about remote storage
func (v *Vaultik) RemoteInfo(jsonOutput bool) error {
log.Info("Starting remote storage info gathering")
result := &RemoteInfoResult{}
storageInfo := v.Storage.Info()
@@ -160,11 +181,11 @@ func (v *Vaultik) RemoteInfo(jsonOutput bool) error {
result.StorageLocation = storageInfo.Location
if !jsonOutput {
v.printfStdout("=== Remote Storage ===\n")
v.printfStdout("Type: %s\n", storageInfo.Type)
v.printfStdout("Location: %s\n", storageInfo.Location)
v.stdoutf("=== Remote Storage ===\n")
v.stdoutf("Type: %s\n", storageInfo.Type)
v.stdoutf("Location: %s\n", storageInfo.Location)
v.printlnStdout()
v.printfStdout("Scanning snapshot metadata...\n")
v.stdoutf("Scanning snapshot metadata...\n")
}
snapshotMetadata, snapshotIDs, err := v.collectSnapshotMetadata()
@@ -173,7 +194,7 @@ func (v *Vaultik) RemoteInfo(jsonOutput bool) error {
}
if !jsonOutput {
v.printfStdout("Downloading %d manifest(s)...\n", len(snapshotIDs))
v.stdoutf("Downloading %d manifest(s)...\n", len(snapshotIDs))
}
referencedBlobs := v.collectReferencedBlobsFromManifests(snapshotIDs, snapshotMetadata)
@@ -193,10 +214,12 @@ func (v *Vaultik) RemoteInfo(jsonOutput bool) error {
if jsonOutput {
enc := json.NewEncoder(v.Stdout)
enc.SetIndent("", " ")
return enc.Encode(result)
}
v.printRemoteInfoTable(result)
return nil
}
@@ -214,6 +237,7 @@ func (v *Vaultik) collectSnapshotMetadata() (map[string]*SnapshotMetadataInfo, [
if len(parts) < 3 {
continue
}
snapshotID := parts[1]
if _, exists := snapshotMetadata[snapshotID]; !exists {
@@ -221,12 +245,14 @@ func (v *Vaultik) collectSnapshotMetadata() (map[string]*SnapshotMetadataInfo, [
}
info := snapshotMetadata[snapshotID]
filename := parts[2]
if strings.HasPrefix(filename, "manifest") {
info.ManifestSize = obj.Size
} else if strings.HasPrefix(filename, "db") {
info.DatabaseSize = obj.Size
}
info.TotalSize = info.ManifestSize + info.DatabaseSize
}
@@ -234,6 +260,7 @@ func (v *Vaultik) collectSnapshotMetadata() (map[string]*SnapshotMetadataInfo, [
for id := range snapshotMetadata {
snapshotIDs = append(snapshotIDs, id)
}
sort.Strings(snapshotIDs)
return snapshotMetadata, snapshotIDs, nil
@@ -245,26 +272,33 @@ func (v *Vaultik) collectReferencedBlobsFromManifests(snapshotIDs []string, snap
for _, snapshotID := range snapshotIDs {
manifestKey := fmt.Sprintf("metadata/%s/manifest.json.zst", snapshotID)
reader, err := v.Storage.Get(v.ctx, manifestKey)
if err != nil {
log.Warn("Failed to get manifest", "snapshot", snapshotID, "error", err)
continue
}
manifest, err := snapshot.DecodeManifest(reader)
_ = reader.Close()
if err != nil {
log.Warn("Failed to decode manifest", "snapshot", snapshotID, "error", err)
continue
}
info := snapshotMetadata[snapshotID]
info.BlobCount = manifest.BlobCount
var blobsSize int64
for _, blob := range manifest.Blobs {
referencedBlobs[blob.Hash] = blob.CompressedSize
blobsSize += blob.CompressedSize
}
info.BlobsSize = blobsSize
}
@@ -274,11 +308,13 @@ func (v *Vaultik) collectReferencedBlobsFromManifests(snapshotIDs []string, snap
// populateRemoteInfoResult fills in the result's snapshot and referenced blob stats
func (v *Vaultik) populateRemoteInfoResult(result *RemoteInfoResult, snapshotMetadata map[string]*SnapshotMetadataInfo, snapshotIDs []string, referencedBlobs map[string]int64) {
var totalMetadataSize int64
for _, id := range snapshotIDs {
info := snapshotMetadata[id]
result.Snapshots = append(result.Snapshots, *info)
totalMetadataSize += info.TotalSize
}
result.TotalMetadataSize = totalMetadataSize
result.TotalMetadataCount = len(snapshotIDs)
@@ -291,7 +327,7 @@ func (v *Vaultik) populateRemoteInfoResult(result *RemoteInfoResult, snapshotMet
// scanRemoteBlobStorage lists all blobs on remote and computes orphan stats
func (v *Vaultik) scanRemoteBlobStorage(result *RemoteInfoResult, referencedBlobs map[string]int64, jsonOutput bool) error {
if !jsonOutput {
v.printfStdout("Scanning blobs...\n")
v.stdoutf("Scanning blobs...\n")
}
blobCh := v.Storage.ListStream(v.ctx, "blobs/")
@@ -301,10 +337,12 @@ func (v *Vaultik) scanRemoteBlobStorage(result *RemoteInfoResult, referencedBlob
if obj.Err != nil {
return fmt.Errorf("listing blobs: %w", obj.Err)
}
parts := strings.Split(obj.Key, "/")
if len(parts) < 4 {
continue
}
hash := parts[3]
allBlobs[hash] = obj.Size
result.TotalBlobCount++
@@ -323,14 +361,16 @@ func (v *Vaultik) scanRemoteBlobStorage(result *RemoteInfoResult, referencedBlob
// printRemoteInfoTable renders the human-readable remote info output
func (v *Vaultik) printRemoteInfoTable(result *RemoteInfoResult) {
v.printfStdout("\n=== Snapshot Metadata ===\n")
v.stdoutf("\n=== Snapshot Metadata ===\n")
if len(result.Snapshots) == 0 {
v.printfStdout("No snapshots found\n")
v.stdoutf("No snapshots found\n")
} else {
v.printfStdout("%-45s %12s %12s %12s %10s %12s\n", "SNAPSHOT", "MANIFEST", "DATABASE", "TOTAL", "BLOBS", "BLOB SIZE")
v.printfStdout("%-45s %12s %12s %12s %10s %12s\n", strings.Repeat("-", 45), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 10), strings.Repeat("-", 12))
v.stdoutf("%-45s %12s %12s %12s %10s %12s\n", "SNAPSHOT", "MANIFEST", "DATABASE", "TOTAL", "BLOBS", "BLOB SIZE")
v.stdoutf("%-45s %12s %12s %12s %10s %12s\n", strings.Repeat("-", 45), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 10), strings.Repeat("-", 12))
for _, info := range result.Snapshots {
v.printfStdout("%-45s %12s %12s %12s %10s %12s\n",
v.stdoutf("%-45s %12s %12s %12s %10s %12s\n",
truncateString(info.SnapshotID, 45),
humanize.Bytes(uint64(info.ManifestSize)),
humanize.Bytes(uint64(info.DatabaseSize)),
@@ -339,20 +379,21 @@ func (v *Vaultik) printRemoteInfoTable(result *RemoteInfoResult) {
humanize.Bytes(uint64(info.BlobsSize)),
)
}
v.printfStdout("%-45s %12s %12s %12s %10s %12s\n", strings.Repeat("-", 45), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 10), strings.Repeat("-", 12))
v.printfStdout("%-45s %12s %12s %12s\n", fmt.Sprintf("Total (%d snapshots)", result.TotalMetadataCount), "", "", humanize.Bytes(uint64(result.TotalMetadataSize)))
v.stdoutf("%-45s %12s %12s %12s %10s %12s\n", strings.Repeat("-", 45), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 10), strings.Repeat("-", 12))
v.stdoutf("%-45s %12s %12s %12s\n", fmt.Sprintf("Total (%d snapshots)", result.TotalMetadataCount), "", "", humanize.Bytes(uint64(result.TotalMetadataSize)))
}
v.printfStdout("\n=== Blob Storage ===\n")
v.printfStdout("Total blobs on remote: %s (%s)\n",
v.stdoutf("\n=== Blob Storage ===\n")
v.stdoutf("Total blobs on remote: %s (%s)\n",
humanize.Comma(int64(result.TotalBlobCount)), humanize.Bytes(uint64(result.TotalBlobSize)))
v.printfStdout("Referenced by snapshots: %s (%s)\n",
v.stdoutf("Referenced by snapshots: %s (%s)\n",
humanize.Comma(int64(result.ReferencedBlobCount)), humanize.Bytes(uint64(result.ReferencedBlobSize)))
v.printfStdout("Orphaned (unreferenced): %s (%s)\n",
v.stdoutf("Orphaned (unreferenced): %s (%s)\n",
humanize.Comma(int64(result.OrphanedBlobCount)), humanize.Bytes(uint64(result.OrphanedBlobSize)))
if result.OrphanedBlobCount > 0 {
v.printfStdout("\nRun 'vaultik prune' to remove orphaned blobs.\n")
v.stdoutf("\nRun 'vaultik prune' to remove orphaned blobs.\n")
}
}
@@ -361,8 +402,10 @@ func truncateString(s string, maxLen int) string {
if len(s) <= maxLen {
return s
}
if maxLen <= 3 {
return s[:maxLen]
}
return s[:maxLen-3] + "..."
}

77
internal/vaultik/integration_test.go
View File

@@ -43,11 +43,14 @@ func (m *MockStorer) Put(ctx context.Context, key string, reader io.Reader) erro
defer m.mu.Unlock()
m.calls = append(m.calls, "Put:"+key)
data, err := io.ReadAll(reader)
if err != nil {
return err
}
m.data[key] = data
return nil
}
@@ -60,10 +63,12 @@ func (m *MockStorer) Get(ctx context.Context, key string) (io.ReadCloser, error)
defer m.mu.Unlock()
m.calls = append(m.calls, "Get:"+key)
data, exists := m.data[key]
if !exists {
return nil, storage.ErrNotFound
}
return io.NopCloser(bytes.NewReader(data)), nil
}
@@ -72,10 +77,12 @@ func (m *MockStorer) Stat(ctx context.Context, key string) (*storage.ObjectInfo,
defer m.mu.Unlock()
m.calls = append(m.calls, "Stat:"+key)
data, exists := m.data[key]
if !exists {
return nil, storage.ErrNotFound
}
return &storage.ObjectInfo{
Key: key,
Size: int64(len(data)),
@@ -88,6 +95,7 @@ func (m *MockStorer) Delete(ctx context.Context, key string) error {
m.calls = append(m.calls, "Delete:"+key)
delete(m.data, key)
return nil
}
@@ -96,12 +104,15 @@ func (m *MockStorer) List(ctx context.Context, prefix string) ([]string, error)
defer m.mu.Unlock()
m.calls = append(m.calls, "List:"+prefix)
var keys []string
for key := range m.data {
if len(prefix) == 0 || (len(key) >= len(prefix) && key[:len(prefix)] == prefix) {
keys = append(keys, key)
}
}
return keys, nil
}
@@ -109,6 +120,7 @@ func (m *MockStorer) ListStream(ctx context.Context, prefix string) <-chan stora
ch := make(chan storage.ObjectInfo)
go func() {
defer close(ch)
m.mu.Lock()
defer m.mu.Unlock()
@@ -121,6 +133,7 @@ func (m *MockStorer) ListStream(ctx context.Context, prefix string) <-chan stora
}
}
}()
return ch
}
@@ -138,6 +151,7 @@ func (m *MockStorer) GetCalls() []string {
calls := make([]string, len(m.calls))
copy(calls, m.calls)
return calls
}
@@ -172,14 +186,16 @@ func TestEndToEndBackup(t *testing.T) {
"/home/user/code",
}
for _, dir := range dirs {
if err := fs.MkdirAll(dir, 0755); err != nil {
err := fs.MkdirAll(dir, 0755)
if err != nil {
t.Fatalf("failed to create directory %s: %v", dir, err)
}
}
// Create test files
for path, content := range testFiles {
if err := afero.WriteFile(fs, path, []byte(content), 0644); err != nil {
err := afero.WriteFile(fs, path, []byte(content), 0644)
if err != nil {
t.Fatalf("failed to create test file %s: %v", path, err)
}
}
@@ -216,9 +232,11 @@ func TestEndToEndBackup(t *testing.T) {
// Create in-memory database
db, err := database.New(ctx, ":memory:")
require.NoError(t, err)
defer func() {
if err := db.Close(); err != nil {
err := db.Close()
if err != nil {
t.Errorf("failed to close database: %v", err)
}
}()
@@ -246,6 +264,7 @@ func TestEndToEndBackup(t *testing.T) {
VaultikVersion: "test-version",
StartedAt: time.Now(),
}
return repos.Snapshots.Create(ctx, tx, snapshot)
})
require.NoError(t, err)
@@ -258,9 +277,9 @@ func TestEndToEndBackup(t *testing.T) {
// The scanner counts both files and directories, so we have:
// 4 files + 4 directories (/home, /home/user, /home/user/documents, /home/user/pictures, /home/user/code)
assert.GreaterOrEqual(t, result.FilesScanned, 4, "Should scan at least 4 files")
assert.Greater(t, result.BytesScanned, int64(0), "Should scan some bytes")
assert.Greater(t, result.ChunksCreated, 0, "Should create chunks")
assert.Greater(t, result.BlobsCreated, 0, "Should create blobs")
assert.Positive(t, result.BytesScanned, "Should scan some bytes")
assert.Positive(t, result.ChunksCreated, "Should create chunks")
assert.Positive(t, result.BlobsCreated, "Should create blobs")
// Verify storage operations
calls := mockStorage.GetCalls()
@@ -268,6 +287,7 @@ func TestEndToEndBackup(t *testing.T) {
// Should have uploaded at least one blob
blobUploads := 0
for _, call := range calls {
if len(call) > 4 && call[:4] == "Put:" {
if len(call) > 10 && call[4:10] == "blobs/" {
@@ -275,27 +295,30 @@ func TestEndToEndBackup(t *testing.T) {
}
}
}
assert.Greater(t, blobUploads, 0, "Should upload at least one blob")
assert.Positive(t, blobUploads, "Should upload at least one blob")
// Verify files in database
files, err := repos.Files.ListByPrefix(ctx, "/home/user")
require.NoError(t, err)
// Count only regular files (not directories)
regularFiles := 0
for _, f := range files {
if f.Mode&0x80000000 == 0 { // Check if regular file (not directory)
regularFiles++
}
}
assert.Equal(t, 4, regularFiles, "Should have 4 regular files in database")
// Verify chunks were created by checking a specific file
fileChunks, err := repos.FileChunks.GetByPath(ctx, "/home/user/documents/file1.txt")
require.NoError(t, err)
assert.Greater(t, len(fileChunks), 0, "Should have chunks for file1.txt")
assert.NotEmpty(t, fileChunks, "Should have chunks for file1.txt")
// Verify blobs were uploaded to storage
assert.Greater(t, mockStorage.GetStorageSize(), 0, "Should have blobs in storage")
assert.Positive(t, mockStorage.GetStorageSize(), "Should have blobs in storage")
// Complete the snapshot - just verify we got results
// In a real integration test, we'd update the snapshot record
@@ -337,9 +360,11 @@ func TestBackupAndVerify(t *testing.T) {
// Create test database
ctx := context.Background()
db, err := database.New(ctx, ":memory:")
require.NoError(t, err)
defer func() {
if err := db.Close(); err != nil {
err := db.Close()
if err != nil {
t.Errorf("failed to close database: %v", err)
}
}()
@@ -366,6 +391,7 @@ func TestBackupAndVerify(t *testing.T) {
VaultikVersion: "test-version",
StartedAt: time.Now(),
}
return repos.Snapshots.Create(ctx, tx, snapshot)
})
require.NoError(t, err)
@@ -375,7 +401,7 @@ func TestBackupAndVerify(t *testing.T) {
require.NoError(t, err)
// Verify backup created blobs
assert.Greater(t, result.BlobsCreated, 0, "Should create at least one blob")
assert.Positive(t, result.BlobsCreated, "Should create at least one blob")
assert.Equal(t, mockStorage.GetStorageSize(), result.BlobsCreated, "Storage should have the blobs")
// Verify we can retrieve the blob from storage
@@ -391,18 +417,19 @@ func TestBackupAndVerify(t *testing.T) {
// Get blob info
blobInfo, err := mockStorage.Stat(ctx, blobKey)
require.NoError(t, err)
assert.Greater(t, blobInfo.Size, int64(0), "Blob should have content")
assert.Positive(t, blobInfo.Size, "Blob should have content")
// Get blob content
reader, err := mockStorage.Get(ctx, blobKey)
require.NoError(t, err)
defer func() { _ = reader.Close() }()
// Verify blob data is encrypted (should not contain plaintext)
blobData, err := io.ReadAll(reader)
require.NoError(t, err)
assert.NotContains(t, string(blobData), testContent, "Blob should be encrypted")
assert.Greater(t, len(blobData), 0, "Blob should have data")
assert.NotEmpty(t, blobData, "Blob should have data")
}
t.Logf("Backup and verify test completed successfully")
@@ -418,6 +445,7 @@ func TestBackupAndRestore(t *testing.T) {
// Create real temp directory for the database (SQLite needs real filesystem)
realTempDir, err := os.MkdirTemp("", "vaultik-test-")
require.NoError(t, err)
defer func() { _ = os.RemoveAll(realTempDir) }()
// Use real OS filesystem for this test
@@ -434,10 +462,14 @@ func TestBackupAndRestore(t *testing.T) {
// Create directories and files
for path, content := range testFiles {
dir := filepath.Dir(path)
if err := fs.MkdirAll(dir, 0755); err != nil {
err := fs.MkdirAll(dir, 0755)
if err != nil {
t.Fatalf("failed to create directory %s: %v", dir, err)
}
if err := afero.WriteFile(fs, path, []byte(content), 0644); err != nil {
err = afero.WriteFile(fs, path, []byte(content), 0644)
if err != nil {
t.Fatalf("failed to create test file %s: %v", path, err)
}
}
@@ -455,6 +487,7 @@ func TestBackupAndRestore(t *testing.T) {
dbPath := filepath.Join(realTempDir, "test.db")
db, err := database.New(ctx, dbPath)
require.NoError(t, err)
defer func() { _ = db.Close() }()
repos := database.NewRepositories(db)
@@ -558,6 +591,7 @@ func TestEndToEndFileStorage(t *testing.T) {
fs := afero.NewOsFs()
tempDir, err := os.MkdirTemp("", "vaultik-e2e-")
require.NoError(t, err)
defer func() { _ = os.RemoveAll(tempDir) }()
dataDir := filepath.Join(tempDir, "source")
@@ -618,6 +652,7 @@ func TestEndToEndFileStorage(t *testing.T) {
db, err := database.New(ctx, dbPath)
require.NoError(t, err)
defer func() { _ = db.Close() }()
repos := database.NewRepositories(db)
@@ -644,8 +679,8 @@ func TestEndToEndFileStorage(t *testing.T) {
scanResult, err := scanner.Scan(ctx, dataDir, snapshotID)
require.NoError(t, err)
require.Greater(t, scanResult.FilesScanned, 0)
require.Greater(t, scanResult.BlobsCreated, 0)
require.Positive(t, scanResult.FilesScanned)
require.Positive(t, scanResult.BlobsCreated)
require.NoError(t, sm.CompleteSnapshot(ctx, snapshotID))
require.NoError(t, sm.ExportSnapshotMetadata(ctx, dbPath, snapshotID))
@@ -656,6 +691,7 @@ func TestEndToEndFileStorage(t *testing.T) {
blobInfo, err := os.Stat(filepath.Join(storeDir, "blobs"))
require.NoError(t, err)
require.True(t, blobInfo.IsDir())
metaInfo, err := os.Stat(filepath.Join(storeDir, "metadata", snapshot.RemoteSnapshotKey(snapshotID)))
require.NoError(t, err)
require.True(t, metaInfo.IsDir())
@@ -721,6 +757,7 @@ func TestDedupOnlySnapshotRestores(t *testing.T) {
fs := afero.NewOsFs()
tempDir, err := os.MkdirTemp("", "vaultik-dedup-")
require.NoError(t, err)
defer func() { _ = os.RemoveAll(tempDir) }()
dataDir := filepath.Join(tempDir, "source")
@@ -756,7 +793,9 @@ func TestDedupOnlySnapshotRestores(t *testing.T) {
ctx := context.Background()
db, err := database.New(ctx, dbPath)
require.NoError(t, err)
defer func() { _ = db.Close() }()
repos := database.NewRepositories(db)
makeScanner := func() *snapshot.Scanner {
@@ -780,13 +819,14 @@ func TestDedupOnlySnapshotRestores(t *testing.T) {
require.NoError(t, err)
r1, err := makeScanner().Scan(ctx, dataDir, id1)
require.NoError(t, err)
require.Greater(t, r1.BlobsCreated, 0, "first snapshot should upload at least one blob")
require.Positive(t, r1.BlobsCreated, "first snapshot should upload at least one blob")
require.NoError(t, sm.CompleteSnapshot(ctx, id1))
require.NoError(t, sm.ExportSnapshotMetadata(ctx, dbPath, id1))
// Second snapshot — same data, every chunk dedups. Sleep past the
// second-precision timestamp so the snapshot IDs differ.
time.Sleep(1100 * time.Millisecond)
id2, err := sm.CreateSnapshotWithName(ctx, cfg.Hostname, "dedup", "v", "g")
require.NoError(t, err)
r2, err := makeScanner().Scan(ctx, dataDir, id2)
@@ -833,5 +873,6 @@ func bytesPattern(tag string, n int) []byte {
for i := range out {
out[i] = byte(tag[i%len(tag)] ^ byte(i&0xff))
}
return out
}

65
internal/vaultik/prune.go
View File

@@ -2,6 +2,7 @@ package vaultik
import (
"encoding/json"
"errors"
"fmt"
"strings"
@@ -23,20 +24,24 @@ type PruneOptions struct {
// confirming with the user.
func (v *Vaultik) NukeRemote(force bool) error {
if !force {
return fmt.Errorf("nuke requires --force (this deletes ALL remote snapshots and blobs)")
return errors.New("nuke requires --force (this deletes ALL remote snapshots and blobs)")
}
v.UI.Begin("Removing all snapshot metadata from backup destination store.")
v.UI.Beginf("Removing all snapshot metadata from backup destination store.")
if _, err := v.RemoveAllSnapshots(&RemoveOptions{Force: true}); err != nil {
return fmt.Errorf("removing all snapshots: %w", err)
}
v.UI.Begin("Removing any blobs still present in backup destination store.")
if err := v.PruneBlobs(&PruneOptions{Force: true}); err != nil {
v.UI.Beginf("Removing any blobs still present in backup destination store.")
err := v.PruneBlobs(&PruneOptions{Force: true})
if err != nil {
return fmt.Errorf("pruning blobs: %w", err)
}
v.UI.Complete("Backup destination store is now empty.")
v.UI.Completef("Backup destination store is now empty.")
return nil
}
@@ -55,7 +60,8 @@ type PruneBlobsResult struct {
// prefer this method over PruneDatabase or PruneBlobs individually
// unless it specifically wants one half.
func (v *Vaultik) Prune(opts *PruneOptions) error {
if err := v.EnsureStorageBinding(); err != nil {
err := v.EnsureStorageBinding()
if err != nil {
return err
}
// First reconcile local snapshot records against remote metadata:
@@ -63,12 +69,15 @@ func (v *Vaultik) Prune(opts *PruneOptions) error {
// store is treated as gone. This used to be the separate 'snapshot
// cleanup' command and is now folded in so a single 'vaultik prune'
// gets the local index fully back in sync with the destination.
if err := v.CleanupLocalSnapshots(); err != nil {
err = v.CleanupLocalSnapshots()
if err != nil {
return fmt.Errorf("reconciling local snapshots with remote: %w", err)
}
if _, err := v.PruneDatabase(); err != nil {
return fmt.Errorf("pruning local database: %w", err)
}
return v.PruneBlobs(opts)
}
@@ -92,27 +101,35 @@ func (v *Vaultik) PruneBlobs(opts *PruneOptions) error {
if len(unreferencedBlobs) == 0 {
log.Info("No unreferenced blobs found")
if opts.JSON {
return v.outputPruneBlobsJSON(result)
}
v.printlnStdout("No unreferenced blobs to remove.")
return nil
}
log.Info("Found unreferenced blobs", "count", len(unreferencedBlobs), "total_size", humanize.Bytes(uint64(totalSize)))
if !opts.JSON {
v.printfStdout("Found %d unreferenced blob(s) totaling %s\n", len(unreferencedBlobs), humanize.Bytes(uint64(totalSize)))
v.stdoutf("Found %d unreferenced blob(s) totaling %s\n", len(unreferencedBlobs), humanize.Bytes(uint64(totalSize)))
}
if !opts.Force && !opts.JSON {
v.printfStdout("\nDelete %d unreferenced blob(s)? [y/N] ", len(unreferencedBlobs))
v.stdoutf("\nDelete %d unreferenced blob(s)? [y/N] ", len(unreferencedBlobs))
var confirm string
if _, err := v.scanStdin(&confirm); err != nil {
v.printlnStdout("Cancelled")
return nil
}
if strings.ToLower(confirm) != "y" {
v.printlnStdout("Cancelled")
return nil
}
}
@@ -123,9 +140,10 @@ func (v *Vaultik) PruneBlobs(opts *PruneOptions) error {
return v.outputPruneBlobsJSON(result)
}
v.printfStdout("\nDeleted %d blob(s) totaling %s\n", result.BlobsDeleted, humanize.Bytes(uint64(result.BytesFreed)))
v.stdoutf("\nDeleted %d blob(s) totaling %s\n", result.BlobsDeleted, humanize.Bytes(uint64(result.BytesFreed)))
if result.BlobsFailed > 0 {
v.printfStdout("Failed to delete %d blob(s)\n", result.BlobsFailed)
v.stdoutf("Failed to delete %d blob(s)\n", result.BlobsFailed)
}
return nil
@@ -140,6 +158,7 @@ func (v *Vaultik) collectReferencedBlobs() (map[string]bool, error) {
if err != nil {
return nil, fmt.Errorf("listing snapshot keys: %w", err)
}
log.Info("Found manifests in remote storage", "count", len(remoteKeys))
allBlobsReferenced := make(map[string]bool)
@@ -147,18 +166,23 @@ func (v *Vaultik) collectReferencedBlobs() (map[string]bool, error) {
for _, remoteKey := range remoteKeys {
log.Debug("Processing manifest", "remote_key", remoteKey)
manifest, err := v.downloadManifestByKey(remoteKey)
if err != nil {
log.Error("Failed to download manifest", "remote_key", remoteKey, "error", err)
continue
}
for _, blob := range manifest.Blobs {
allBlobsReferenced[blob.Hash] = true
}
manifestCount++
}
log.Info("Processed manifests", "count", manifestCount, "unique_blobs_referenced", len(allBlobsReferenced))
return allBlobsReferenced, nil
}
@@ -166,12 +190,14 @@ func (v *Vaultik) collectReferencedBlobs() (map[string]bool, error) {
func (v *Vaultik) listUniqueSnapshotIDs() ([]string, error) {
objectCh := v.Storage.ListStream(v.ctx, "metadata/")
seen := make(map[string]bool)
var snapshotIDs []string
for object := range objectCh {
if object.Err != nil {
return nil, fmt.Errorf("listing metadata objects: %w", object.Err)
}
parts := strings.Split(object.Key, "/")
if len(parts) >= 2 && parts[0] == "metadata" && parts[1] != "" {
if strings.HasSuffix(object.Key, "/") || strings.Contains(object.Key, "/manifest.json.zst") {
@@ -183,12 +209,14 @@ func (v *Vaultik) listUniqueSnapshotIDs() ([]string, error) {
}
}
}
return snapshotIDs, nil
}
// listAllRemoteBlobs returns a map of all blob hashes to their sizes in remote storage
func (v *Vaultik) listAllRemoteBlobs() (map[string]int64, error) {
log.Info("Listing all blobs in storage")
allBlobs := make(map[string]int64)
blobObjectCh := v.Storage.ListStream(v.ctx, "blobs/")
@@ -196,6 +224,7 @@ func (v *Vaultik) listAllRemoteBlobs() (map[string]int64, error) {
if object.Err != nil {
return nil, fmt.Errorf("listing blobs: %w", object.Err)
}
parts := strings.Split(object.Key, "/")
if len(parts) == 4 && parts[0] == "blobs" {
allBlobs[parts[3]] = object.Size
@@ -203,19 +232,24 @@ func (v *Vaultik) listAllRemoteBlobs() (map[string]int64, error) {
}
log.Info("Found blobs in storage", "count", len(allBlobs))
return allBlobs, nil
}
// findUnreferencedBlobs returns blob hashes not referenced by any manifest and their total size
func (v *Vaultik) findUnreferencedBlobs(allBlobs map[string]int64, referenced map[string]bool) ([]string, int64) {
var unreferenced []string
var totalSize int64
var (
unreferenced []string
totalSize int64
)
for hash, size := range allBlobs {
if !referenced[hash] {
unreferenced = append(unreferenced, hash)
totalSize += size
}
}
return unreferenced, totalSize
}
@@ -226,8 +260,10 @@ func (v *Vaultik) deleteUnreferencedBlobs(unreferencedBlobs []string, allBlobs m
for i, hash := range unreferencedBlobs {
blobPath := fmt.Sprintf("blobs/%s/%s/%s", hash[:2], hash[2:4], hash)
if err := v.Storage.Delete(v.ctx, blobPath); err != nil {
err := v.Storage.Delete(v.ctx, blobPath)
if err != nil {
log.Error("Failed to delete blob", "hash", hash, "error", err)
continue
}
@@ -256,5 +292,6 @@ func (v *Vaultik) deleteUnreferencedBlobs(unreferencedBlobs []string, allBlobs m
func (v *Vaultik) outputPruneBlobsJSON(result *PruneBlobsResult) error {
encoder := json.NewEncoder(v.Stdout)
encoder.SetIndent("", " ")
return encoder.Encode(result)
}

3
internal/vaultik/purge_per_name_test.go
View File

@@ -79,16 +79,19 @@ func setupPurgeTest(t *testing.T, snapshotIDs []string) *vaultik.Vaultik {
// listRemainingSnapshots returns IDs of all completed snapshots in the database.
func listRemainingSnapshots(t *testing.T, v *vaultik.Vaultik) []string {
t.Helper()
ctx := context.Background()
dbSnaps, err := v.Repositories.Snapshots.ListRecent(ctx, 10000)
require.NoError(t, err)
var ids []string
for _, s := range dbSnaps {
if s.CompletedAt != nil {
ids = append(ids, s.ID.String())
}
}
return ids
}

14
internal/vaultik/remove_snapshot_test.go
View File

@@ -37,7 +37,9 @@ func (s *testStorer) Put(ctx context.Context, key string, reader io.Reader) erro
if err != nil {
return err
}
s.data[key] = data
return nil
}
@@ -53,6 +55,7 @@ func (s *testStorer) Get(ctx context.Context, key string) (io.ReadCloser, error)
if !exists {
return nil, storage.ErrNotFound
}
return io.NopCloser(bytes.NewReader(data)), nil
}
@@ -64,6 +67,7 @@ func (s *testStorer) Stat(ctx context.Context, key string) (*storage.ObjectInfo,
if !exists {
return nil, storage.ErrNotFound
}
return &storage.ObjectInfo{
Key: key,
Size: int64(len(data)),
@@ -75,6 +79,7 @@ func (s *testStorer) Delete(ctx context.Context, key string) error {
defer s.mu.Unlock()
delete(s.data, key)
return nil
}
@@ -83,11 +88,13 @@ func (s *testStorer) List(ctx context.Context, prefix string) ([]string, error)
defer s.mu.Unlock()
var keys []string
for key := range s.data {
if prefix == "" || strings.HasPrefix(key, prefix) {
keys = append(keys, key)
}
}
return keys, nil
}
@@ -96,6 +103,7 @@ func (s *testStorer) ListStream(ctx context.Context, prefix string) <-chan stora
go func() {
defer close(ch)
s.mu.Lock()
defer s.mu.Unlock()
@@ -115,13 +123,16 @@ func (s *testStorer) ListStream(ctx context.Context, prefix string) <-chan stora
func (s *testStorer) hasKey(key string) bool {
s.mu.Lock()
defer s.mu.Unlock()
_, exists := s.data[key]
return exists
}
func (s *testStorer) keyCount() int {
s.mu.Lock()
defer s.mu.Unlock()
return len(s.data)
}
@@ -175,6 +186,7 @@ func addBlob(t *testing.T, store *testStorer, hash string) {
// Create zstd compressed data
var buf bytes.Buffer
writer, _ := zstd.NewWriter(&buf)
_, _ = writer.Write([]byte("blob data"))
_ = writer.Close()
@@ -366,7 +378,7 @@ func TestRemoveAllSnapshots_NoSnapshots(t *testing.T) {
result, err := tv.RemoveAllSnapshots(opts)
require.NoError(t, err)
assert.Len(t, result.SnapshotsRemoved, 0)
assert.Empty(t, result.SnapshotsRemoved)
// Verify output
assert.Contains(t, tv.Stdout.String(), "No snapshots found")

Some files were not shown because too many files have changed in this diff Show More