Apply mechanical lint fixes for golangci-lint v2.12.2 rollout
Auto-remediate style-only findings (wsl_v5, nlreturn, noinlineerr, modernize, intrange, perfsprint, usetesting, unconvert, errorlint, gocritic, testifylint) and rename printf-style helpers to f-suffixed names (goprintffuncname): ui.Writer message methods, cli.ReportErrorf, database.Fatalf, vaultik stdoutf.
This commit is contained in:
@@ -1,5 +1,9 @@
|
||||
version: "2"
|
||||
|
||||
# Config schema uses the golangci-lint v2 layout (settings live under
|
||||
# linters.settings, not top-level linters-settings) so that the
|
||||
# thresholds below are actually applied by golangci-lint >= v2.
|
||||
|
||||
run:
|
||||
timeout: 5m
|
||||
modules-download-mode: readonly
|
||||
@@ -14,19 +18,17 @@ linters:
|
||||
- wsl # Deprecated, replaced by wsl_v5
|
||||
- wrapcheck # Too verbose for internal packages
|
||||
- varnamelen # Short names like db, id are idiomatic Go
|
||||
|
||||
linters-settings:
|
||||
lll:
|
||||
line-length: 88
|
||||
funlen:
|
||||
lines: 80
|
||||
statements: 50
|
||||
cyclop:
|
||||
max-complexity: 15
|
||||
dupl:
|
||||
threshold: 100
|
||||
settings:
|
||||
lll:
|
||||
line-length: 88
|
||||
funlen:
|
||||
lines: 80
|
||||
statements: 50
|
||||
cyclop:
|
||||
max-complexity: 15
|
||||
dupl:
|
||||
threshold: 100
|
||||
|
||||
issues:
|
||||
exclude-use-default: false
|
||||
max-issues-per-linter: 0
|
||||
max-same-issues: 0
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Lint stage
|
||||
# golangci/golangci-lint:v2.11.3-alpine, 2026-03-17
|
||||
FROM golangci/golangci-lint:v2.11.3-alpine@sha256:b1c3de5862ad0a95b4e45a993b0f00415835d687e4f12c845c7493b86c13414e AS lint
|
||||
# golangci/golangci-lint:v2.12.2-alpine, 2026-08-07
|
||||
FROM golangci/golangci-lint:v2.12.2-alpine@sha256:91b27804074a0bacea298707f016911e60cf0cdbc6c7bf5ccacb5f0606d18d60 AS lint
|
||||
|
||||
RUN apk add --no-cache make build-base
|
||||
|
||||
|
||||
2
Makefile
2
Makefile
@@ -55,7 +55,7 @@ clean:
|
||||
# Install dependencies.
|
||||
deps:
|
||||
go mod download
|
||||
go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest
|
||||
go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.12.2
|
||||
|
||||
# Run tests with coverage.
|
||||
test-coverage:
|
||||
|
||||
@@ -16,9 +16,11 @@ func main() {
|
||||
panic("could not create CPU profile: " + err.Error())
|
||||
}
|
||||
defer func() { _ = f.Close() }()
|
||||
|
||||
if err := pprof.StartCPUProfile(f); err != nil {
|
||||
panic("could not start CPU profile: " + err.Error())
|
||||
}
|
||||
|
||||
defer pprof.StopCPUProfile()
|
||||
}
|
||||
|
||||
@@ -30,7 +32,9 @@ func main() {
|
||||
panic("could not create memory profile: " + err.Error())
|
||||
}
|
||||
defer func() { _ = f.Close() }()
|
||||
|
||||
runtime.GC() // get up-to-date statistics
|
||||
|
||||
if err := pprof.WriteHeapProfile(f); err != nil {
|
||||
panic("could not write memory profile: " + err.Error())
|
||||
}
|
||||
|
||||
@@ -18,6 +18,7 @@ import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"sync"
|
||||
@@ -124,6 +125,7 @@ type BlobChunkRef struct {
|
||||
// BlobWithReader wraps a FinishedBlob with its data reader
|
||||
type BlobWithReader struct {
|
||||
*FinishedBlob
|
||||
|
||||
Reader io.ReadSeeker
|
||||
TempFile afero.File // Optional, only set for disk-based blobs
|
||||
InsertedChunkHashes []string // Chunk hashes that were inserted to DB with this blob
|
||||
@@ -134,14 +136,17 @@ type BlobWithReader struct {
|
||||
// Returns an error if required configuration fields are missing or invalid.
|
||||
func NewPacker(cfg PackerConfig) (*Packer, error) {
|
||||
if len(cfg.Recipients) == 0 {
|
||||
return nil, fmt.Errorf("recipients are required - blobs must be encrypted")
|
||||
return nil, errors.New("recipients are required - blobs must be encrypted")
|
||||
}
|
||||
|
||||
if cfg.MaxBlobSize <= 0 {
|
||||
return nil, fmt.Errorf("max blob size must be positive")
|
||||
return nil, errors.New("max blob size must be positive")
|
||||
}
|
||||
|
||||
if cfg.Fs == nil {
|
||||
return nil, fmt.Errorf("filesystem is required")
|
||||
return nil, errors.New("filesystem is required")
|
||||
}
|
||||
|
||||
return &Packer{
|
||||
maxBlobSize: cfg.MaxBlobSize,
|
||||
compressionLevel: cfg.CompressionLevel,
|
||||
@@ -160,6 +165,7 @@ func NewPacker(cfg PackerConfig) (*Packer, error) {
|
||||
func (p *Packer) SetBlobHandler(handler BlobHandler) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
|
||||
p.blobHandler = handler
|
||||
}
|
||||
|
||||
@@ -169,6 +175,7 @@ func (p *Packer) SetBlobHandler(handler BlobHandler) {
|
||||
func (p *Packer) AddPendingChunk(hash string, size int64) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
|
||||
p.pendingChunks = append(p.pendingChunks, PendingChunk{Hash: hash, Size: size})
|
||||
}
|
||||
|
||||
@@ -183,7 +190,8 @@ func (p *Packer) AddChunk(chunk *ChunkRef) error {
|
||||
|
||||
// Initialize new blob if needed
|
||||
if p.currentBlob == nil {
|
||||
if err := p.startNewBlob(); err != nil {
|
||||
err := p.startNewBlob()
|
||||
if err != nil {
|
||||
return fmt.Errorf("starting new blob: %w", err)
|
||||
}
|
||||
}
|
||||
@@ -202,7 +210,8 @@ func (p *Packer) AddChunk(chunk *ChunkRef) error {
|
||||
}
|
||||
|
||||
// Add chunk to current blob
|
||||
if err := p.addChunkToCurrentBlob(chunk); err != nil {
|
||||
err := p.addChunkToCurrentBlob(chunk)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -218,7 +227,8 @@ func (p *Packer) Flush() error {
|
||||
defer p.mu.Unlock()
|
||||
|
||||
if p.currentBlob != nil && len(p.currentBlob.chunks) > 0 {
|
||||
if err := p.finalizeCurrentBlob(); err != nil {
|
||||
err := p.finalizeCurrentBlob()
|
||||
if err != nil {
|
||||
return fmt.Errorf("finalizing blob: %w", err)
|
||||
}
|
||||
}
|
||||
@@ -253,6 +263,7 @@ func (p *Packer) GetFinishedBlobs() []*FinishedBlob {
|
||||
|
||||
blobs := p.finishedBlobs
|
||||
p.finishedBlobs = make([]*FinishedBlob, 0)
|
||||
|
||||
return blobs
|
||||
}
|
||||
|
||||
@@ -267,6 +278,7 @@ func (p *Packer) startNewBlob() error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("parsing blob ID: %w", err)
|
||||
}
|
||||
|
||||
blob := &database.Blob{
|
||||
ID: blobIDTyped,
|
||||
Hash: types.BlobHash("temp-placeholder-" + blobID), // Temporary placeholder until finalized
|
||||
@@ -276,6 +288,7 @@ func (p *Packer) startNewBlob() error {
|
||||
CompressedSize: 0,
|
||||
UploadedTS: nil,
|
||||
}
|
||||
|
||||
if err := p.repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error {
|
||||
return p.repos.Blobs.Create(ctx, tx, blob)
|
||||
}); err != nil {
|
||||
@@ -294,6 +307,7 @@ func (p *Packer) startNewBlob() error {
|
||||
if err != nil {
|
||||
_ = tempFile.Close()
|
||||
_ = p.fs.Remove(tempFile.Name())
|
||||
|
||||
return fmt.Errorf("creating blobgen writer: %w", err)
|
||||
}
|
||||
|
||||
@@ -308,6 +322,7 @@ func (p *Packer) startNewBlob() error {
|
||||
}
|
||||
|
||||
log.Debug("Created new blob container", "blob_id", blobID, "temp_file", tempFile.Name())
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -316,6 +331,7 @@ func (p *Packer) addChunkToCurrentBlob(chunk *ChunkRef) error {
|
||||
// Skip if chunk already in current blob
|
||||
if p.currentBlob.chunkSet[chunk.Hash] {
|
||||
log.Debug("Skipping duplicate chunk already in current blob", "chunk_hash", chunk.Hash)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -403,24 +419,31 @@ func (p *Packer) finalizeCurrentBlob() error {
|
||||
func (p *Packer) closeBlobWriter() (string, int64, error) {
|
||||
if err := p.currentBlob.writer.Close(); err != nil {
|
||||
p.cleanupTempFile()
|
||||
|
||||
return "", 0, fmt.Errorf("closing blobgen writer: %w", err)
|
||||
}
|
||||
|
||||
if err := p.currentBlob.tempFile.Sync(); err != nil {
|
||||
p.cleanupTempFile()
|
||||
|
||||
return "", 0, fmt.Errorf("syncing temp file: %w", err)
|
||||
}
|
||||
|
||||
finalSize, err := p.currentBlob.tempFile.Seek(0, io.SeekCurrent)
|
||||
if err != nil {
|
||||
p.cleanupTempFile()
|
||||
|
||||
return "", 0, fmt.Errorf("getting file size: %w", err)
|
||||
}
|
||||
|
||||
if _, err := p.currentBlob.tempFile.Seek(0, io.SeekStart); err != nil {
|
||||
p.cleanupTempFile()
|
||||
|
||||
return "", 0, fmt.Errorf("seeking to start: %w", err)
|
||||
}
|
||||
|
||||
finalHash := p.currentBlob.writer.Sum256()
|
||||
|
||||
return hex.EncodeToString(finalHash), finalSize, nil
|
||||
}
|
||||
|
||||
@@ -432,6 +455,7 @@ func (p *Packer) buildChunkRefs() []*BlobChunkRef {
|
||||
ChunkHash: chunk.Hash, Offset: chunk.Offset, Length: chunk.Size,
|
||||
})
|
||||
}
|
||||
|
||||
return refs
|
||||
}
|
||||
|
||||
@@ -444,13 +468,16 @@ func (p *Packer) commitBlobToDatabase(blobHash string, finalSize int64, chunksTo
|
||||
blobIDTyped, parseErr := types.ParseBlobID(p.currentBlob.id)
|
||||
if parseErr != nil {
|
||||
p.cleanupTempFile()
|
||||
|
||||
return fmt.Errorf("parsing blob ID: %w", parseErr)
|
||||
}
|
||||
|
||||
err := p.repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error {
|
||||
for _, chunk := range chunksToInsert {
|
||||
dbChunk := &database.Chunk{ChunkHash: types.ChunkHash(chunk.Hash), Size: chunk.Size}
|
||||
if err := p.repos.Chunks.Create(ctx, tx, dbChunk); err != nil {
|
||||
|
||||
err := p.repos.Chunks.Create(ctx, tx, dbChunk)
|
||||
if err != nil {
|
||||
return fmt.Errorf("creating chunk: %w", err)
|
||||
}
|
||||
}
|
||||
@@ -460,7 +487,9 @@ func (p *Packer) commitBlobToDatabase(blobHash string, finalSize int64, chunksTo
|
||||
BlobID: blobIDTyped, ChunkHash: types.ChunkHash(chunk.Hash),
|
||||
Offset: chunk.Offset, Length: chunk.Size,
|
||||
}
|
||||
if err := p.repos.BlobChunks.Create(ctx, tx, blobChunk); err != nil {
|
||||
|
||||
err := p.repos.BlobChunks.Create(ctx, tx, blobChunk)
|
||||
if err != nil {
|
||||
return fmt.Errorf("creating blob_chunk: %w", err)
|
||||
}
|
||||
}
|
||||
@@ -469,11 +498,13 @@ func (p *Packer) commitBlobToDatabase(blobHash string, finalSize int64, chunksTo
|
||||
})
|
||||
if err != nil {
|
||||
p.cleanupTempFile()
|
||||
|
||||
return fmt.Errorf("finalizing blob transaction: %w", err)
|
||||
}
|
||||
|
||||
log.Debug("Committed blob transaction",
|
||||
"chunks_inserted", len(chunksToInsert), "blob_chunks_inserted", len(p.currentBlob.chunks))
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -482,6 +513,7 @@ func (p *Packer) deliverFinishedBlob(finished *FinishedBlob, insertedChunkHashes
|
||||
if p.blobHandler != nil {
|
||||
if _, err := p.currentBlob.tempFile.Seek(0, io.SeekStart); err != nil {
|
||||
p.cleanupTempFile()
|
||||
|
||||
return fmt.Errorf("seeking for handler: %w", err)
|
||||
}
|
||||
|
||||
@@ -492,30 +524,39 @@ func (p *Packer) deliverFinishedBlob(finished *FinishedBlob, insertedChunkHashes
|
||||
InsertedChunkHashes: insertedChunkHashes,
|
||||
}
|
||||
|
||||
if err := p.blobHandler(blobWithReader); err != nil {
|
||||
err := p.blobHandler(blobWithReader)
|
||||
if err != nil {
|
||||
p.cleanupTempFile()
|
||||
|
||||
return fmt.Errorf("blob handler failed: %w", err)
|
||||
}
|
||||
|
||||
p.currentBlob = nil
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// No handler - read data for legacy behavior
|
||||
log.Debug("No blob handler callback configured", "blob_hash", finished.Hash[:8]+"...")
|
||||
|
||||
if _, err := p.currentBlob.tempFile.Seek(0, io.SeekStart); err != nil {
|
||||
p.cleanupTempFile()
|
||||
|
||||
return fmt.Errorf("seeking to read data: %w", err)
|
||||
}
|
||||
|
||||
data, err := io.ReadAll(p.currentBlob.tempFile)
|
||||
if err != nil {
|
||||
p.cleanupTempFile()
|
||||
|
||||
return fmt.Errorf("reading blob data: %w", err)
|
||||
}
|
||||
|
||||
finished.Data = data
|
||||
p.finishedBlobs = append(p.finishedBlobs, finished)
|
||||
p.cleanupTempFile()
|
||||
p.currentBlob = nil
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -532,13 +573,15 @@ func (p *Packer) cleanupTempFile() {
|
||||
func (p *Packer) PackChunks(chunks []*ChunkRef) error {
|
||||
for _, chunk := range chunks {
|
||||
err := p.AddChunk(chunk)
|
||||
if err == ErrBlobSizeLimitExceeded {
|
||||
if errors.Is(err, ErrBlobSizeLimitExceeded) {
|
||||
// Finalize current blob and retry
|
||||
if err := p.FinalizeBlob(); err != nil {
|
||||
err := p.FinalizeBlob()
|
||||
if err != nil {
|
||||
return fmt.Errorf("finalizing blob before retry: %w", err)
|
||||
}
|
||||
// Retry the chunk
|
||||
if err := p.AddChunk(chunk); err != nil {
|
||||
err = p.AddChunk(chunk)
|
||||
if err != nil {
|
||||
return fmt.Errorf("adding chunk %s after finalize: %w", chunk.Hash, err)
|
||||
}
|
||||
} else if err != nil {
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
@@ -40,6 +41,7 @@ func TestPacker(t *testing.T) {
|
||||
t.Fatalf("failed to create test db: %v", err)
|
||||
}
|
||||
defer func() { _ = db.Close() }()
|
||||
|
||||
repos := database.NewRepositories(db)
|
||||
|
||||
cfg := PackerConfig{
|
||||
@@ -49,6 +51,7 @@ func TestPacker(t *testing.T) {
|
||||
Repositories: repos,
|
||||
Fs: afero.NewMemMapFs(),
|
||||
}
|
||||
|
||||
packer, err := NewPacker(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create packer: %v", err)
|
||||
@@ -64,6 +67,7 @@ func TestPacker(t *testing.T) {
|
||||
ChunkHash: types.ChunkHash(hashStr),
|
||||
Size: int64(len(data)),
|
||||
}
|
||||
|
||||
err = repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error {
|
||||
return repos.Chunks.Create(ctx, tx, dbChunk)
|
||||
})
|
||||
@@ -130,6 +134,7 @@ func TestPacker(t *testing.T) {
|
||||
t.Fatalf("failed to create test db: %v", err)
|
||||
}
|
||||
defer func() { _ = db.Close() }()
|
||||
|
||||
repos := database.NewRepositories(db)
|
||||
|
||||
cfg := PackerConfig{
|
||||
@@ -139,6 +144,7 @@ func TestPacker(t *testing.T) {
|
||||
Repositories: repos,
|
||||
Fs: afero.NewMemMapFs(),
|
||||
}
|
||||
|
||||
packer, err := NewPacker(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create packer: %v", err)
|
||||
@@ -146,7 +152,8 @@ func TestPacker(t *testing.T) {
|
||||
|
||||
// Create multiple small chunks
|
||||
chunks := make([]*ChunkRef, 10)
|
||||
for i := 0; i < 10; i++ {
|
||||
|
||||
for i := range 10 {
|
||||
data := bytes.Repeat([]byte{byte(i)}, 1000)
|
||||
hash := sha256.Sum256(data)
|
||||
hashStr := hex.EncodeToString(hash[:])
|
||||
@@ -156,6 +163,7 @@ func TestPacker(t *testing.T) {
|
||||
ChunkHash: types.ChunkHash(hashStr),
|
||||
Size: int64(len(data)),
|
||||
}
|
||||
|
||||
err = repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error {
|
||||
return repos.Chunks.Create(ctx, tx, dbChunk)
|
||||
})
|
||||
@@ -198,9 +206,11 @@ func TestPacker(t *testing.T) {
|
||||
if chunkRef.Offset != expectedOffset {
|
||||
t.Errorf("chunk %d: expected offset %d, got %d", i, expectedOffset, chunkRef.Offset)
|
||||
}
|
||||
|
||||
if chunkRef.Length != 1000 {
|
||||
t.Errorf("chunk %d: expected length 1000, got %d", i, chunkRef.Length)
|
||||
}
|
||||
|
||||
expectedOffset += chunkRef.Length
|
||||
}
|
||||
})
|
||||
@@ -212,6 +222,7 @@ func TestPacker(t *testing.T) {
|
||||
t.Fatalf("failed to create test db: %v", err)
|
||||
}
|
||||
defer func() { _ = db.Close() }()
|
||||
|
||||
repos := database.NewRepositories(db)
|
||||
|
||||
// Small blob size limit to force multiple blobs
|
||||
@@ -222,6 +233,7 @@ func TestPacker(t *testing.T) {
|
||||
Repositories: repos,
|
||||
Fs: afero.NewMemMapFs(),
|
||||
}
|
||||
|
||||
packer, err := NewPacker(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create packer: %v", err)
|
||||
@@ -229,7 +241,8 @@ func TestPacker(t *testing.T) {
|
||||
|
||||
// Create chunks that will exceed the limit
|
||||
chunks := make([]*ChunkRef, 10)
|
||||
for i := 0; i < 10; i++ {
|
||||
|
||||
for i := range 10 {
|
||||
data := bytes.Repeat([]byte{byte(i)}, 1000) // 1KB each
|
||||
hash := sha256.Sum256(data)
|
||||
hashStr := hex.EncodeToString(hash[:])
|
||||
@@ -239,6 +252,7 @@ func TestPacker(t *testing.T) {
|
||||
ChunkHash: types.ChunkHash(hashStr),
|
||||
Size: int64(len(data)),
|
||||
}
|
||||
|
||||
err = repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error {
|
||||
return repos.Chunks.Create(ctx, tx, dbChunk)
|
||||
})
|
||||
@@ -257,14 +271,17 @@ func TestPacker(t *testing.T) {
|
||||
// Add chunks and handle size limit errors
|
||||
for _, chunk := range chunks {
|
||||
err := packer.AddChunk(chunk)
|
||||
if err == ErrBlobSizeLimitExceeded {
|
||||
if errors.Is(err, ErrBlobSizeLimitExceeded) {
|
||||
// Finalize current blob
|
||||
if err := packer.FinalizeBlob(); err != nil {
|
||||
err := packer.FinalizeBlob()
|
||||
if err != nil {
|
||||
t.Fatalf("failed to finalize blob: %v", err)
|
||||
}
|
||||
|
||||
blobCount++
|
||||
// Retry adding the chunk
|
||||
if err := packer.AddChunk(chunk); err != nil {
|
||||
err = packer.AddChunk(chunk)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to add chunk after finalize: %v", err)
|
||||
}
|
||||
} else if err != nil {
|
||||
@@ -301,6 +318,7 @@ func TestPacker(t *testing.T) {
|
||||
t.Fatalf("failed to create test db: %v", err)
|
||||
}
|
||||
defer func() { _ = db.Close() }()
|
||||
|
||||
repos := database.NewRepositories(db)
|
||||
|
||||
// Generate test identity (using the one from parent test)
|
||||
@@ -311,6 +329,7 @@ func TestPacker(t *testing.T) {
|
||||
Repositories: repos,
|
||||
Fs: afero.NewMemMapFs(),
|
||||
}
|
||||
|
||||
packer, err := NewPacker(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create packer: %v", err)
|
||||
@@ -326,6 +345,7 @@ func TestPacker(t *testing.T) {
|
||||
ChunkHash: types.ChunkHash(hashStr),
|
||||
Size: int64(len(data)),
|
||||
}
|
||||
|
||||
err = repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error {
|
||||
return repos.Chunks.Create(ctx, tx, dbChunk)
|
||||
})
|
||||
@@ -342,6 +362,7 @@ func TestPacker(t *testing.T) {
|
||||
if err := packer.AddChunk(chunk); err != nil {
|
||||
t.Fatalf("failed to add chunk: %v", err)
|
||||
}
|
||||
|
||||
if err := packer.Flush(); err != nil {
|
||||
t.Fatalf("failed to flush: %v", err)
|
||||
}
|
||||
|
||||
@@ -28,6 +28,7 @@ func CompressData(data []byte, compressionLevel int, recipients []string) (*Comp
|
||||
// Write data
|
||||
if _, err := w.Write(data); err != nil {
|
||||
_ = w.Close()
|
||||
|
||||
return nil, fmt.Errorf("writing data: %w", err)
|
||||
}
|
||||
|
||||
@@ -68,6 +69,7 @@ func CompressStream(dst io.Writer, src io.Reader, compressionLevel int, recipien
|
||||
if err := w.Close(); err != nil {
|
||||
return 0, "", fmt.Errorf("closing writer: %w", err)
|
||||
}
|
||||
|
||||
closed = true
|
||||
|
||||
return w.BytesWritten(), hex.EncodeToString(w.Sum256()), nil
|
||||
|
||||
@@ -20,13 +20,14 @@ const testRecipient = "age1cplgrwj77ta54dnmydvvmzn64ltk83ankxl5sww04mrtmu62kv3s8
|
||||
// cause a double close.
|
||||
func TestCompressStreamNoDoubleClose(t *testing.T) {
|
||||
input := []byte("regression test data for issue #28 double-close fix")
|
||||
|
||||
var buf bytes.Buffer
|
||||
|
||||
written, hash, err := CompressStream(&buf, bytes.NewReader(input), 3, []string{testRecipient})
|
||||
require.NoError(t, err, "CompressStream should not return an error")
|
||||
assert.True(t, written > 0, "expected bytes written > 0")
|
||||
assert.Positive(t, written, "expected bytes written > 0")
|
||||
assert.NotEmpty(t, hash, "expected non-empty hash")
|
||||
assert.True(t, buf.Len() > 0, "expected non-empty output")
|
||||
assert.Positive(t, buf.Len(), "expected non-empty output")
|
||||
}
|
||||
|
||||
// TestCompressStreamLargeInput exercises CompressStream with a larger payload
|
||||
@@ -37,9 +38,10 @@ func TestCompressStreamLargeInput(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
|
||||
var buf bytes.Buffer
|
||||
|
||||
written, hash, err := CompressStream(&buf, bytes.NewReader(data), 3, []string{testRecipient})
|
||||
require.NoError(t, err)
|
||||
assert.True(t, written > 0)
|
||||
assert.Positive(t, written)
|
||||
assert.NotEmpty(t, hash)
|
||||
}
|
||||
|
||||
@@ -47,6 +49,7 @@ func TestCompressStreamLargeInput(t *testing.T) {
|
||||
// without double-close issues.
|
||||
func TestCompressStreamEmptyInput(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
|
||||
_, hash, err := CompressStream(&buf, strings.NewReader(""), 3, []string{testRecipient})
|
||||
require.NoError(t, err)
|
||||
assert.NotEmpty(t, hash)
|
||||
@@ -58,7 +61,7 @@ func TestCompressDataNoDoubleClose(t *testing.T) {
|
||||
input := []byte("CompressData regression test for double-close")
|
||||
result, err := CompressData(input, 3, []string{testRecipient})
|
||||
require.NoError(t, err)
|
||||
assert.True(t, result.CompressedSize > 0)
|
||||
assert.True(t, result.UncompressedSize == int64(len(input)))
|
||||
assert.Positive(t, result.CompressedSize)
|
||||
assert.Equal(t, result.UncompressedSize, int64(len(input)))
|
||||
assert.NotEmpty(t, result.SHA256)
|
||||
}
|
||||
|
||||
@@ -53,12 +53,14 @@ func NewReader(r io.Reader, identity age.Identity) (*Reader, error) {
|
||||
func (r *Reader) Read(p []byte) (n int, err error) {
|
||||
n, err = r.teeReader.Read(p)
|
||||
r.bytesRead += int64(n)
|
||||
|
||||
return n, err
|
||||
}
|
||||
|
||||
// Close closes the decompressor
|
||||
func (r *Reader) Close() error {
|
||||
r.decompressor.Close()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -36,11 +36,13 @@ func NewWriter(w io.Writer, compressionLevel int, recipients []string) (*Writer,
|
||||
|
||||
// Parse recipients
|
||||
var ageRecipients []age.Recipient
|
||||
|
||||
for _, recipient := range recipients {
|
||||
r, err := age.ParseX25519Recipient(recipient)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parsing recipient %s: %w", recipient, err)
|
||||
}
|
||||
|
||||
ageRecipients = append(ageRecipients, r)
|
||||
}
|
||||
|
||||
@@ -51,10 +53,7 @@ func NewWriter(w io.Writer, compressionLevel int, recipients []string) (*Writer,
|
||||
}
|
||||
|
||||
// Calculate compression concurrency: CPUs - 2, minimum 1
|
||||
concurrency := runtime.NumCPU() - 2
|
||||
if concurrency < 1 {
|
||||
concurrency = 1
|
||||
}
|
||||
concurrency := max(runtime.NumCPU()-2, 1)
|
||||
|
||||
// Create compression writer with encryption as destination
|
||||
compressor, err := zstd.NewWriter(encWriter,
|
||||
@@ -63,6 +62,7 @@ func NewWriter(w io.Writer, compressionLevel int, recipients []string) (*Writer,
|
||||
)
|
||||
if err != nil {
|
||||
_ = encWriter.Close()
|
||||
|
||||
return nil, fmt.Errorf("creating compression writer: %w", err)
|
||||
}
|
||||
|
||||
@@ -82,18 +82,21 @@ func NewWriter(w io.Writer, compressionLevel int, recipients []string) (*Writer,
|
||||
func (w *Writer) Write(p []byte) (n int, err error) {
|
||||
n, err = w.teeWriter.Write(p)
|
||||
w.bytesWritten += int64(n)
|
||||
|
||||
return n, err
|
||||
}
|
||||
|
||||
// Close closes all layers and returns any errors
|
||||
func (w *Writer) Close() error {
|
||||
// Close compressor first
|
||||
if err := w.compressor.Close(); err != nil {
|
||||
err := w.compressor.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("closing compressor: %w", err)
|
||||
}
|
||||
|
||||
// Then close encryptor
|
||||
if err := w.encryptor.Close(); err != nil {
|
||||
err = w.encryptor.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("closing encryptor: %w", err)
|
||||
}
|
||||
|
||||
@@ -109,6 +112,7 @@ func (w *Writer) Sum256() []byte {
|
||||
firstHash := w.hasher.Sum(nil)
|
||||
// Second hash: SHA256(firstHash) - this is the blob ID
|
||||
secondHash := sha256.Sum256(firstHash)
|
||||
|
||||
return secondHash[:]
|
||||
}
|
||||
|
||||
@@ -123,5 +127,6 @@ func validateCompressionLevel(level int) error {
|
||||
if level < 1 || level > 19 {
|
||||
return fmt.Errorf("invalid compression level %d: must be between 1 and 19", level)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package chunker
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
@@ -50,13 +51,15 @@ func (c *Chunker) ChunkReader(r io.Reader) ([]Chunk, error) {
|
||||
defer chunker.Release()
|
||||
|
||||
var chunks []Chunk
|
||||
|
||||
offset := int64(0)
|
||||
|
||||
for {
|
||||
chunk, err := chunker.Next()
|
||||
if err == io.EOF {
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading chunk: %w", err)
|
||||
}
|
||||
@@ -104,9 +107,10 @@ func (c *Chunker) ChunkReaderStreaming(r io.Reader, callback ChunkCallback) (str
|
||||
|
||||
for {
|
||||
chunk, err := chunker.Next()
|
||||
if err == io.EOF {
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("reading chunk: %w", err)
|
||||
}
|
||||
@@ -143,7 +147,8 @@ func (c *Chunker) ChunkFile(path string) ([]Chunk, error) {
|
||||
return nil, fmt.Errorf("opening file: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
if err := file.Close(); err != nil && err.Error() != "invalid argument" {
|
||||
err := file.Close()
|
||||
if err != nil && err.Error() != "invalid argument" {
|
||||
// Log error or handle as needed
|
||||
_ = err
|
||||
}
|
||||
|
||||
@@ -42,7 +42,7 @@ func TestChunkerExpectedChunkCount(t *testing.T) {
|
||||
|
||||
// Create data with some variation to trigger chunk boundaries
|
||||
data := make([]byte, tt.fileSize)
|
||||
for i := 0; i < len(data); i++ {
|
||||
for i := range data {
|
||||
// Use a pattern that should create boundaries
|
||||
data[i] = byte((i * 17) ^ (i >> 5))
|
||||
}
|
||||
@@ -59,6 +59,7 @@ func TestChunkerExpectedChunkCount(t *testing.T) {
|
||||
t.Errorf("too few chunks: got %d, expected at least %d",
|
||||
len(chunks), tt.minExpected)
|
||||
}
|
||||
|
||||
if len(chunks) > tt.maxExpected {
|
||||
t.Errorf("too many chunks: got %d, expected at most %d",
|
||||
len(chunks), tt.maxExpected)
|
||||
@@ -69,6 +70,7 @@ func TestChunkerExpectedChunkCount(t *testing.T) {
|
||||
for _, chunk := range chunks {
|
||||
reconstructed = append(reconstructed, chunk.Data...)
|
||||
}
|
||||
|
||||
if !bytes.Equal(data, reconstructed) {
|
||||
t.Error("reconstructed data doesn't match original")
|
||||
}
|
||||
|
||||
@@ -60,6 +60,7 @@ func TestChunker(t *testing.T) {
|
||||
if chunk.Offset != expectedOffset {
|
||||
t.Errorf("chunk %d: expected offset %d, got %d", i, expectedOffset, chunk.Offset)
|
||||
}
|
||||
|
||||
expectedOffset += chunk.Size
|
||||
}
|
||||
})
|
||||
@@ -90,6 +91,7 @@ func TestChunker(t *testing.T) {
|
||||
if chunks1[i].Hash != chunks2[i].Hash {
|
||||
t.Errorf("chunk %d: different hashes", i)
|
||||
}
|
||||
|
||||
if chunks1[i].Size != chunks2[i].Size {
|
||||
t.Errorf("chunk %d: different sizes", i)
|
||||
}
|
||||
@@ -121,6 +123,7 @@ func TestChunkBoundaries(t *testing.T) {
|
||||
if i < len(chunks)-1 && chunk.Size < minSize {
|
||||
t.Errorf("chunk %d size %d is below minimum %d", i, chunk.Size, minSize)
|
||||
}
|
||||
|
||||
if chunk.Size > maxSize {
|
||||
t.Errorf("chunk %d size %d exceeds maximum %d", i, chunk.Size, maxSize)
|
||||
}
|
||||
|
||||
@@ -28,7 +28,7 @@ type ReusableChunker struct {
|
||||
|
||||
// reusableChunkerPool pools ReusableChunker instances to avoid allocations.
|
||||
var reusableChunkerPool = sync.Pool{
|
||||
New: func() interface{} {
|
||||
New: func() any {
|
||||
return &ReusableChunker{}
|
||||
},
|
||||
}
|
||||
@@ -39,17 +39,20 @@ var bufferPools = sync.Map{}
|
||||
|
||||
func getBuffer(size int) []byte {
|
||||
poolI, _ := bufferPools.LoadOrStore(size, &sync.Pool{
|
||||
New: func() interface{} {
|
||||
New: func() any {
|
||||
buf := make([]byte, size)
|
||||
|
||||
return &buf
|
||||
},
|
||||
})
|
||||
pool := poolI.(*sync.Pool)
|
||||
|
||||
return *pool.Get().(*[]byte)
|
||||
}
|
||||
|
||||
func putBuffer(buf []byte) {
|
||||
size := cap(buf)
|
||||
|
||||
poolI, ok := bufferPools.Load(size)
|
||||
if ok {
|
||||
pool := poolI.(*sync.Pool)
|
||||
@@ -77,6 +80,7 @@ func AcquireReusableChunker(rd io.Reader, minSize, avgSize, maxSize int) *Reusab
|
||||
if c.buf != nil {
|
||||
putBuffer(c.buf)
|
||||
}
|
||||
|
||||
c.buf = getBuffer(bufSize)
|
||||
} else {
|
||||
// Restore buffer to full capacity (may have been truncated by previous EOF)
|
||||
@@ -120,6 +124,7 @@ func (c *ReusableChunker) fillBuffer() error {
|
||||
|
||||
if c.eof {
|
||||
c.buf = c.buf[:n]
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -134,15 +139,18 @@ func (c *ReusableChunker) fillBuffer() error {
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Next returns the next chunk or io.EOF when done.
|
||||
// The returned Data slice is only valid until the next call to Next.
|
||||
func (c *ReusableChunker) Next() (FastCDCChunk, error) {
|
||||
if err := c.fillBuffer(); err != nil {
|
||||
err := c.fillBuffer()
|
||||
if err != nil {
|
||||
return FastCDCChunk{}, err
|
||||
}
|
||||
|
||||
if len(c.buf) == 0 {
|
||||
return FastCDCChunk{}, io.EOF
|
||||
}
|
||||
@@ -189,13 +197,6 @@ func (c *ReusableChunker) nextChunk(data []byte) (int, uint64) {
|
||||
return i, fp
|
||||
}
|
||||
|
||||
func min(a, b int) int {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// 256 random uint64s for the rolling hash function (from FastCDC paper)
|
||||
var table = [256]uint64{
|
||||
0xe80e8d55032474b3, 0x11b25b61f5924e15, 0x03aa5bd82a9eb669, 0xc45a153ef107a38c,
|
||||
|
||||
@@ -44,9 +44,11 @@ func setupGlobals(lc fx.Lifecycle, g *globals.Globals, v *vaultik.Vaultik, opts
|
||||
lc.Append(fx.Hook{
|
||||
OnStart: func(ctx context.Context) error {
|
||||
g.StartTime = time.Now().UTC()
|
||||
|
||||
if opts.Cron || opts.Quiet {
|
||||
v.UI.SetQuiet(true)
|
||||
}
|
||||
|
||||
return nil
|
||||
},
|
||||
})
|
||||
@@ -56,12 +58,12 @@ func setupGlobals(lc fx.Lifecycle, g *globals.Globals, v *vaultik.Vaultik, opts
|
||||
// blank line. Used both from the fx hook (for subcommand invocations) and
|
||||
// from the root cobra Run handler (for `vaultik` with no subcommand).
|
||||
func writeStartupBanner(w *ui.Writer, startTime time.Time, shortCommit string) {
|
||||
w.Banner("%s %s by %s (commit %s, built on %s) starting up at %s.",
|
||||
w.Bannerf("%s %s by %s (commit %s, built on %s) starting up at %s.",
|
||||
globals.Appname, globals.Version, globals.Author,
|
||||
shortCommit, globals.CommitDate,
|
||||
startTime.Format(time.RFC3339))
|
||||
w.Banner("%s", globals.Homepage)
|
||||
w.Banner("")
|
||||
w.Bannerf("%s", globals.Homepage)
|
||||
w.Bannerf("")
|
||||
}
|
||||
|
||||
// NewApp creates a new fx application with common modules.
|
||||
@@ -105,6 +107,7 @@ func cleanStartupError(err error) error {
|
||||
if idx := strings.LastIndex(msg, "): "); idx >= 0 {
|
||||
msg = msg[idx+3:]
|
||||
}
|
||||
|
||||
return errors.New(msg)
|
||||
}
|
||||
|
||||
@@ -122,7 +125,8 @@ func RunApp(ctx context.Context, app *fx.App) error {
|
||||
defer cancel()
|
||||
|
||||
// Start the app
|
||||
if err := app.Start(ctx); err != nil {
|
||||
err := app.Start(ctx)
|
||||
if err != nil {
|
||||
return cleanStartupError(err)
|
||||
}
|
||||
|
||||
@@ -130,6 +134,7 @@ func RunApp(ctx context.Context, app *fx.App) error {
|
||||
shutdownComplete := make(chan struct{})
|
||||
go func() {
|
||||
defer close(shutdownComplete)
|
||||
|
||||
<-sigChan
|
||||
log.Notice("Received interrupt signal, shutting down gracefully...")
|
||||
|
||||
@@ -137,7 +142,8 @@ func RunApp(ctx context.Context, app *fx.App) error {
|
||||
shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer shutdownCancel()
|
||||
|
||||
if err := app.Stop(shutdownCtx); err != nil {
|
||||
err := app.Stop(shutdownCtx)
|
||||
if err != nil {
|
||||
log.Error("Error during shutdown", "error", err)
|
||||
}
|
||||
}()
|
||||
@@ -149,9 +155,11 @@ func RunApp(ctx context.Context, app *fx.App) error {
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
// Context cancelled (shouldn't happen in normal operation)
|
||||
if err := app.Stop(context.Background()); err != nil {
|
||||
err := app.Stop(context.Background())
|
||||
if err != nil {
|
||||
log.Error("Error stopping app", "error", err)
|
||||
}
|
||||
|
||||
return ctx.Err()
|
||||
case <-app.Done():
|
||||
// App finished running (e.g., backup completed)
|
||||
@@ -166,19 +174,24 @@ func RunApp(ctx context.Context, app *fx.App) error {
|
||||
func RunWithApp(ctx context.Context, opts AppOptions) error {
|
||||
// Acquire PID lock to prevent concurrent instances
|
||||
lockDir := filepath.Join(xdg.DataHome, "vaultik")
|
||||
|
||||
lock, err := pidlock.Acquire(lockDir)
|
||||
if err != nil {
|
||||
if errors.Is(err, pidlock.ErrAlreadyRunning) {
|
||||
return fmt.Errorf("cannot start: %w", err)
|
||||
}
|
||||
|
||||
return fmt.Errorf("failed to acquire lock: %w", err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if err := lock.Release(); err != nil {
|
||||
err := lock.Release()
|
||||
if err != nil {
|
||||
log.Warn("Failed to release PID lock", "error", err)
|
||||
}
|
||||
}()
|
||||
|
||||
app := NewApp(opts)
|
||||
|
||||
return RunApp(ctx, app)
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
@@ -240,16 +241,20 @@ on macOS, ~/.config/ on Linux, /etc/vaultik/ as root).`,
|
||||
}
|
||||
|
||||
dir := filepath.Dir(path)
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
|
||||
err := os.MkdirAll(dir, 0o755)
|
||||
if err != nil {
|
||||
return fmt.Errorf("creating config directory %s: %w", dir, err)
|
||||
}
|
||||
|
||||
if err := os.WriteFile(path, []byte(defaultConfigTemplate), 0o600); err != nil {
|
||||
err = os.WriteFile(path, []byte(defaultConfigTemplate), 0o600)
|
||||
if err != nil {
|
||||
return fmt.Errorf("writing config file: %w", err)
|
||||
}
|
||||
|
||||
fmt.Printf("Config written to %s\n", path)
|
||||
fmt.Println("Edit it to set your age_recipients, snapshots, and storage_url.")
|
||||
|
||||
return nil
|
||||
},
|
||||
}
|
||||
@@ -276,6 +281,7 @@ func newConfigEditCommand() *cobra.Command {
|
||||
ed.Stdin = os.Stdin
|
||||
ed.Stdout = os.Stdout
|
||||
ed.Stderr = os.Stderr
|
||||
|
||||
return ed.Run()
|
||||
},
|
||||
}
|
||||
@@ -305,6 +311,7 @@ func newConfigGetCommand() *cobra.Command {
|
||||
|
||||
if node.Kind == yaml.ScalarNode {
|
||||
fmt.Println(node.Value)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -312,7 +319,9 @@ func newConfigGetCommand() *cobra.Command {
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshaling value: %w", err)
|
||||
}
|
||||
|
||||
fmt.Print(string(out))
|
||||
|
||||
return nil
|
||||
},
|
||||
}
|
||||
@@ -363,6 +372,7 @@ Examples:
|
||||
}
|
||||
|
||||
fmt.Printf("%s = %s\n", args[0], args[1])
|
||||
|
||||
return nil
|
||||
},
|
||||
}
|
||||
@@ -399,8 +409,9 @@ func yamlPathGet(root *yaml.Node, keys []string) (*yaml.Node, error) {
|
||||
node := root
|
||||
if node.Kind == yaml.DocumentNode {
|
||||
if len(node.Content) == 0 {
|
||||
return nil, fmt.Errorf("empty config file")
|
||||
return nil, errors.New("empty config file")
|
||||
}
|
||||
|
||||
node = node.Content[0]
|
||||
}
|
||||
|
||||
@@ -408,13 +419,16 @@ func yamlPathGet(root *yaml.Node, keys []string) (*yaml.Node, error) {
|
||||
switch node.Kind {
|
||||
case yaml.MappingNode:
|
||||
found := false
|
||||
|
||||
for j := 0; j+1 < len(node.Content); j += 2 {
|
||||
if node.Content[j].Value == key {
|
||||
node = node.Content[j+1]
|
||||
found = true
|
||||
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if !found {
|
||||
return nil, fmt.Errorf("key not found: %s", strings.Join(keys[:i+1], "."))
|
||||
}
|
||||
@@ -423,9 +437,11 @@ func yamlPathGet(root *yaml.Node, keys []string) (*yaml.Node, error) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("key %q is a list; use a numeric index", strings.Join(keys[:i], "."))
|
||||
}
|
||||
|
||||
if idx < 0 || idx >= len(node.Content) {
|
||||
return nil, fmt.Errorf("index %d out of range for %s (len %d)", idx, strings.Join(keys[:i], "."), len(node.Content))
|
||||
}
|
||||
|
||||
node = node.Content[idx]
|
||||
default:
|
||||
return nil, fmt.Errorf("key %q is not a map or list", strings.Join(keys[:i], "."))
|
||||
@@ -445,6 +461,7 @@ func yamlPathSet(root *yaml.Node, keys []string, value string) error {
|
||||
if len(node.Content) == 0 {
|
||||
node.Content = []*yaml.Node{{Kind: yaml.MappingNode}}
|
||||
}
|
||||
|
||||
node = node.Content[0]
|
||||
}
|
||||
|
||||
@@ -454,19 +471,23 @@ func yamlPathSet(root *yaml.Node, keys []string, value string) error {
|
||||
switch node.Kind {
|
||||
case yaml.MappingNode:
|
||||
var valueNode *yaml.Node
|
||||
|
||||
for j := 0; j+1 < len(node.Content); j += 2 {
|
||||
if node.Content[j].Value == key {
|
||||
valueNode = node.Content[j+1]
|
||||
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if valueNode == nil {
|
||||
keyNode := &yaml.Node{Kind: yaml.ScalarNode, Value: key}
|
||||
|
||||
valueNode = &yaml.Node{Kind: yaml.MappingNode}
|
||||
if last {
|
||||
valueNode = &yaml.Node{Kind: yaml.ScalarNode, Value: value}
|
||||
}
|
||||
|
||||
node.Content = append(node.Content, keyNode, valueNode)
|
||||
} else if last {
|
||||
setScalar(valueNode, value)
|
||||
@@ -479,18 +500,22 @@ func yamlPathSet(root *yaml.Node, keys []string, value string) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("key %q is a list; use a numeric index", strings.Join(keys[:i], "."))
|
||||
}
|
||||
|
||||
if idx < 0 || idx > len(node.Content) {
|
||||
return fmt.Errorf("index %d out of range for %s (len %d)", idx, strings.Join(keys[:i], "."), len(node.Content))
|
||||
}
|
||||
|
||||
if idx == len(node.Content) {
|
||||
newNode := &yaml.Node{Kind: yaml.MappingNode}
|
||||
if last {
|
||||
newNode = &yaml.Node{Kind: yaml.ScalarNode, Value: value}
|
||||
}
|
||||
|
||||
node.Content = append(node.Content, newNode)
|
||||
} else if last {
|
||||
setScalar(node.Content[idx], value)
|
||||
}
|
||||
|
||||
node = node.Content[idx]
|
||||
|
||||
default:
|
||||
@@ -516,8 +541,10 @@ func configPathForInit() string {
|
||||
if rootFlags.ConfigPath != "" {
|
||||
return rootFlags.ConfigPath
|
||||
}
|
||||
|
||||
if envPath := os.Getenv("VAULTIK_CONFIG"); envPath != "" {
|
||||
return envPath
|
||||
}
|
||||
|
||||
return DefaultConfigPath()
|
||||
}
|
||||
|
||||
@@ -12,7 +12,9 @@ import (
|
||||
// that unmarshals into the Config struct with the expected snapshots.
|
||||
func TestDefaultConfigTemplateParses(t *testing.T) {
|
||||
var cfg config.Config
|
||||
if err := yaml.Unmarshal([]byte(defaultConfigTemplate), &cfg); err != nil {
|
||||
|
||||
err := yaml.Unmarshal([]byte(defaultConfigTemplate), &cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("default config template is not valid YAML: %v", err)
|
||||
}
|
||||
|
||||
@@ -24,9 +26,11 @@ func TestDefaultConfigTemplateParses(t *testing.T) {
|
||||
if !ok {
|
||||
t.Fatal("expected 'home' snapshot in default config")
|
||||
}
|
||||
|
||||
if len(home.Paths) == 0 {
|
||||
t.Error("home snapshot should have at least one path")
|
||||
}
|
||||
|
||||
if len(home.Exclude) == 0 {
|
||||
t.Error("home snapshot should have exclude patterns")
|
||||
}
|
||||
@@ -35,9 +39,11 @@ func TestDefaultConfigTemplateParses(t *testing.T) {
|
||||
if !ok {
|
||||
t.Fatal("expected 'apps' snapshot in default config")
|
||||
}
|
||||
|
||||
if len(apps.Paths) != 1 || apps.Paths[0] != "/Applications" {
|
||||
t.Errorf("apps snapshot should back up /Applications, got %v", apps.Paths)
|
||||
}
|
||||
|
||||
if len(apps.Exclude) == 0 {
|
||||
t.Error("apps snapshot should have exclude patterns")
|
||||
}
|
||||
@@ -58,10 +64,14 @@ snapshots:
|
||||
|
||||
func parseTestYAML(t *testing.T) *yaml.Node {
|
||||
t.Helper()
|
||||
|
||||
var root yaml.Node
|
||||
if err := yaml.Unmarshal([]byte(testYAML), &root); err != nil {
|
||||
|
||||
err := yaml.Unmarshal([]byte(testYAML), &root)
|
||||
if err != nil {
|
||||
t.Fatalf("parsing test yaml: %v", err)
|
||||
}
|
||||
|
||||
return &root
|
||||
}
|
||||
|
||||
@@ -91,11 +101,14 @@ func TestYAMLPathGet(t *testing.T) {
|
||||
if err == nil {
|
||||
t.Fatalf("expected error for %q", tt.path)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if node.Value != tt.want {
|
||||
t.Errorf("get %q = %q, want %q", tt.path, node.Value, tt.want)
|
||||
}
|
||||
@@ -115,6 +128,7 @@ func TestYAMLPathSet(t *testing.T) {
|
||||
if err := yamlPathSet(root, splitPath("s3.endpoint"), "s3.example.com"); err != nil {
|
||||
t.Fatalf("set s3.endpoint: %v", err)
|
||||
}
|
||||
|
||||
if err := yamlPathSet(root, splitPath("newmap.newkey"), "val"); err != nil {
|
||||
t.Fatalf("set newmap.newkey: %v", err)
|
||||
}
|
||||
@@ -123,9 +137,11 @@ func TestYAMLPathSet(t *testing.T) {
|
||||
if err := yamlPathSet(root, splitPath("age_recipients.0"), "age1bbb"); err != nil {
|
||||
t.Fatalf("set age_recipients.0: %v", err)
|
||||
}
|
||||
|
||||
if err := yamlPathSet(root, splitPath("age_recipients.1"), "age1ccc"); err != nil {
|
||||
t.Fatalf("append age_recipients.1: %v", err)
|
||||
}
|
||||
|
||||
if err := yamlPathSet(root, splitPath("age_recipients.5"), "age1ddd"); err == nil {
|
||||
t.Error("expected out-of-range append to fail")
|
||||
}
|
||||
@@ -135,6 +151,7 @@ func TestYAMLPathSet(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("marshal: %v", err)
|
||||
}
|
||||
|
||||
text := string(out)
|
||||
|
||||
for _, want := range []string{"newbucket", "s3.example.com", "newkey: val", "# top comment", "# inline comment", "age1bbb", "age1ccc"} {
|
||||
@@ -147,6 +164,7 @@ func TestYAMLPathSet(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("get after set: %v", err)
|
||||
}
|
||||
|
||||
if got.Value != "newbucket" {
|
||||
t.Errorf("s3.bucket = %q after set, want newbucket", got.Value)
|
||||
}
|
||||
|
||||
@@ -66,6 +66,7 @@ Use --force to skip the confirmation prompt.`,
|
||||
// Check if database exists
|
||||
if _, err := os.Stat(dbPath); os.IsNotExist(err) {
|
||||
fmt.Printf("Database does not exist: %s\n", dbPath)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -73,9 +74,11 @@ Use --force to skip the confirmation prompt.`,
|
||||
if !force {
|
||||
fmt.Printf("This will delete the local state database at:\n %s\n\n", dbPath)
|
||||
fmt.Print("Are you sure? Type 'yes' to confirm: ")
|
||||
|
||||
var confirm string
|
||||
if _, err := fmt.Scanln(&confirm); err != nil || confirm != "yes" {
|
||||
fmt.Println("Aborted.")
|
||||
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -97,6 +100,7 @@ Use --force to skip the confirmation prompt.`,
|
||||
}
|
||||
|
||||
log.Info("Local state database deleted", "path", dbPath)
|
||||
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strconv"
|
||||
@@ -25,7 +26,7 @@ func parseDuration(s string) (time.Duration, error) {
|
||||
// Extended duration parsing
|
||||
// Check for negative values
|
||||
if strings.HasPrefix(strings.TrimSpace(s), "-") {
|
||||
return 0, fmt.Errorf("negative durations are not supported")
|
||||
return 0, errors.New("negative durations are not supported")
|
||||
}
|
||||
|
||||
// Pattern matches: number + unit, repeated
|
||||
@@ -48,6 +49,7 @@ func parseDuration(s string) (time.Duration, error) {
|
||||
}
|
||||
|
||||
var d time.Duration
|
||||
|
||||
switch unit {
|
||||
// Standard time units
|
||||
case "ns", "nanosecond", "nanoseconds":
|
||||
@@ -75,7 +77,7 @@ func parseDuration(s string) (time.Duration, error) {
|
||||
d = time.Duration(value * float64(365*24*time.Hour))
|
||||
default:
|
||||
// Try parsing as standard Go duration unit
|
||||
testStr := fmt.Sprintf("1%s", unit)
|
||||
testStr := "1" + unit
|
||||
if _, err := time.ParseDuration(testStr); err == nil {
|
||||
// It's a valid Go duration unit, parse the full value
|
||||
fullStr := fmt.Sprintf("%g%s", value, unit)
|
||||
|
||||
@@ -185,6 +185,7 @@ func TestParseDuration(t *testing.T) {
|
||||
|
||||
if tt.wantErr {
|
||||
assert.Error(t, err, "expected error for input %q", tt.input)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
@@ -19,24 +19,26 @@ func CLIEntry() {
|
||||
if len(short) > 12 {
|
||||
short = short[:12]
|
||||
}
|
||||
|
||||
writeStartupBanner(ui.New(os.Stdout), time.Now().UTC(), short)
|
||||
}
|
||||
|
||||
rootCmd := NewRootCommand()
|
||||
rootCmd.SilenceErrors = true
|
||||
|
||||
if err := rootCmd.Execute(); err != nil {
|
||||
ReportError("%s", err.Error())
|
||||
err := rootCmd.Execute()
|
||||
if err != nil {
|
||||
ReportErrorf("%s", err.Error())
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// ReportError emits a user-facing error to stderr in the standard
|
||||
// ReportErrorf emits a user-facing error to stderr in the standard
|
||||
// 🛑 ERROR: format. Use it from goroutine error paths (where returning
|
||||
// an error to cobra isn't an option) and anywhere else a CLI command
|
||||
// must surface a failure outside the normal RunE return path.
|
||||
func ReportError(format string, args ...any) {
|
||||
ui.New(os.Stderr).Error(format, args...)
|
||||
func ReportErrorf(format string, args ...any) {
|
||||
ui.New(os.Stderr).Errorf(format, args...)
|
||||
}
|
||||
|
||||
// bannerSuppressedInArgs reports whether any of args is a flag that
|
||||
@@ -48,10 +50,12 @@ func bannerSuppressedInArgs(args []string) bool {
|
||||
if a == "--" {
|
||||
return false
|
||||
}
|
||||
|
||||
switch a {
|
||||
case "--quiet", "-q", "--cron":
|
||||
return true
|
||||
}
|
||||
|
||||
if strings.HasPrefix(a, "--quiet=") || strings.HasPrefix(a, "--cron=") {
|
||||
return true
|
||||
}
|
||||
@@ -64,5 +68,6 @@ func bannerSuppressedInArgs(args []string) bool {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -21,12 +21,15 @@ func TestCLIEntry(t *testing.T) {
|
||||
expectedCommands := []string{"config", "snapshot", "prune", "info", "version", "remote", "database"}
|
||||
for _, expected := range expectedCommands {
|
||||
found := false
|
||||
|
||||
for _, cmd := range cmd.Commands() {
|
||||
if cmd.Use == expected || cmd.Name() == expected {
|
||||
found = true
|
||||
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if !found {
|
||||
t.Errorf("Expected command '%s' not found", expected)
|
||||
}
|
||||
@@ -41,12 +44,15 @@ func TestCLIEntry(t *testing.T) {
|
||||
expectedSubCommands := []string{"create", "list", "purge", "verify", "remove", "restore"}
|
||||
for _, expected := range expectedSubCommands {
|
||||
found := false
|
||||
|
||||
for _, subcmd := range snapshotCmd.Commands() {
|
||||
if subcmd.Use == expected || subcmd.Name() == expected {
|
||||
found = true
|
||||
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if !found {
|
||||
t.Errorf("Expected snapshot subcommand '%s' not found", expected)
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
@@ -31,6 +32,7 @@ func NewInfoCommand() *cobra.Command {
|
||||
|
||||
// Use the app framework
|
||||
rootFlags := GetRootFlags()
|
||||
|
||||
return RunWithApp(cmd.Context(), AppOptions{
|
||||
ConfigPath: configPath,
|
||||
LogOptions: log.LogOptions{
|
||||
@@ -44,21 +46,26 @@ func NewInfoCommand() *cobra.Command {
|
||||
lc.Append(fx.Hook{
|
||||
OnStart: func(ctx context.Context) error {
|
||||
go func() {
|
||||
if err := v.ShowInfo(); err != nil {
|
||||
if err != context.Canceled {
|
||||
err := v.ShowInfo()
|
||||
if err != nil {
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
log.Error("Failed to show info", "error", err)
|
||||
ReportError("Failed to show info: %v", err)
|
||||
ReportErrorf("Failed to show info: %v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
if err := v.Shutdowner.Shutdown(); err != nil {
|
||||
|
||||
err = v.Shutdowner.Shutdown()
|
||||
if err != nil {
|
||||
log.Error("Failed to shutdown", "error", err)
|
||||
}
|
||||
}()
|
||||
|
||||
return nil
|
||||
},
|
||||
OnStop: func(ctx context.Context) error {
|
||||
v.Cancel()
|
||||
|
||||
return nil
|
||||
},
|
||||
})
|
||||
|
||||
@@ -2,6 +2,7 @@ package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
@@ -39,6 +40,7 @@ work (e.g. after a crashed backup or to reclaim storage).`,
|
||||
|
||||
// Use the app framework like other commands
|
||||
rootFlags := GetRootFlags()
|
||||
|
||||
return RunWithApp(cmd.Context(), AppOptions{
|
||||
ConfigPath: configPath,
|
||||
LogOptions: log.LogOptions{
|
||||
@@ -54,26 +56,31 @@ work (e.g. after a crashed backup or to reclaim storage).`,
|
||||
// Start the prune operation in a goroutine
|
||||
go func() {
|
||||
// Run the prune operation
|
||||
if err := v.Prune(opts); err != nil {
|
||||
if err != context.Canceled {
|
||||
err := v.Prune(opts)
|
||||
if err != nil {
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
if !opts.JSON {
|
||||
log.Error("Prune operation failed", "error", err)
|
||||
ReportError("Prune failed: %v", err)
|
||||
ReportErrorf("Prune failed: %v", err)
|
||||
}
|
||||
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// Shutdown the app when prune completes
|
||||
if err := v.Shutdowner.Shutdown(); err != nil {
|
||||
err = v.Shutdowner.Shutdown()
|
||||
if err != nil {
|
||||
log.Error("Failed to shutdown", "error", err)
|
||||
}
|
||||
}()
|
||||
|
||||
return nil
|
||||
},
|
||||
OnStop: func(ctx context.Context) error {
|
||||
log.Debug("Stopping prune operation")
|
||||
v.Cancel()
|
||||
|
||||
return nil
|
||||
},
|
||||
})
|
||||
|
||||
@@ -2,7 +2,7 @@ package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"errors"
|
||||
"os"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
@@ -41,7 +41,7 @@ This is destructive and irreversible. Requires --force.`,
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
if !force {
|
||||
return fmt.Errorf("remote nuke requires --force (this deletes ALL remote snapshots and blobs)")
|
||||
return errors.New("remote nuke requires --force (this deletes ALL remote snapshots and blobs)")
|
||||
}
|
||||
|
||||
configPath, err := ResolveConfigPath()
|
||||
@@ -50,6 +50,7 @@ This is destructive and irreversible. Requires --force.`,
|
||||
}
|
||||
|
||||
rootFlags := GetRootFlags()
|
||||
|
||||
return RunWithApp(cmd.Context(), AppOptions{
|
||||
ConfigPath: configPath,
|
||||
LogOptions: log.LogOptions{
|
||||
@@ -63,21 +64,26 @@ This is destructive and irreversible. Requires --force.`,
|
||||
lc.Append(fx.Hook{
|
||||
OnStart: func(ctx context.Context) error {
|
||||
go func() {
|
||||
if err := v.NukeRemote(true); err != nil {
|
||||
if err != context.Canceled {
|
||||
err := v.NukeRemote(true)
|
||||
if err != nil {
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
log.Error("Remote nuke failed", "error", err)
|
||||
ReportError("Remote nuke failed: %v", err)
|
||||
ReportErrorf("Remote nuke failed: %v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
if err := v.Shutdowner.Shutdown(); err != nil {
|
||||
|
||||
err = v.Shutdowner.Shutdown()
|
||||
if err != nil {
|
||||
log.Error("Failed to shutdown", "error", err)
|
||||
}
|
||||
}()
|
||||
|
||||
return nil
|
||||
},
|
||||
OnStop: func(ctx context.Context) error {
|
||||
v.Cancel()
|
||||
|
||||
return nil
|
||||
},
|
||||
})
|
||||
@@ -113,6 +119,7 @@ func newRemoteInfoCommand() *cobra.Command {
|
||||
}
|
||||
|
||||
rootFlags := GetRootFlags()
|
||||
|
||||
return RunWithApp(cmd.Context(), AppOptions{
|
||||
ConfigPath: configPath,
|
||||
LogOptions: log.LogOptions{
|
||||
@@ -126,23 +133,29 @@ func newRemoteInfoCommand() *cobra.Command {
|
||||
lc.Append(fx.Hook{
|
||||
OnStart: func(ctx context.Context) error {
|
||||
go func() {
|
||||
if err := v.RemoteInfo(jsonOutput); err != nil {
|
||||
if err != context.Canceled {
|
||||
err := v.RemoteInfo(jsonOutput)
|
||||
if err != nil {
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
if !jsonOutput {
|
||||
log.Error("Failed to get remote info", "error", err)
|
||||
ReportError("Failed to get remote info: %v", err)
|
||||
ReportErrorf("Failed to get remote info: %v", err)
|
||||
}
|
||||
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
if err := v.Shutdowner.Shutdown(); err != nil {
|
||||
|
||||
err = v.Shutdowner.Shutdown()
|
||||
if err != nil {
|
||||
log.Error("Failed to shutdown", "error", err)
|
||||
}
|
||||
}()
|
||||
|
||||
return nil
|
||||
},
|
||||
OnStop: func(ctx context.Context) error {
|
||||
v.Cancel()
|
||||
|
||||
return nil
|
||||
},
|
||||
})
|
||||
|
||||
@@ -78,6 +78,7 @@ func ResolveConfigPath() (string, error) {
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
return "", fmt.Errorf("config file from --config not found: %s (run 'vaultik config init --config %s' to create it)", path, path)
|
||||
}
|
||||
|
||||
return path, nil
|
||||
}
|
||||
|
||||
@@ -85,6 +86,7 @@ func ResolveConfigPath() (string, error) {
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
return "", fmt.Errorf("config file from $VAULTIK_CONFIG not found: %s (unset VAULTIK_CONFIG, point it at an existing file, or run 'vaultik config init')", path)
|
||||
}
|
||||
|
||||
return path, nil
|
||||
}
|
||||
|
||||
@@ -114,5 +116,6 @@ func DefaultConfigPath() string {
|
||||
if os.Getuid() == 0 {
|
||||
return "/etc/vaultik/config.yml"
|
||||
}
|
||||
|
||||
return filepath.Join(xdg.ConfigHome, "vaultik", "config.yml")
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
@@ -58,6 +59,7 @@ specifying a path using --config or by setting VAULTIK_CONFIG to a path.`,
|
||||
|
||||
// Use the backup functionality from cli package
|
||||
rootFlags := GetRootFlags()
|
||||
|
||||
return RunWithApp(cmd.Context(), AppOptions{
|
||||
ConfigPath: configPath,
|
||||
LogOptions: log.LogOptions{
|
||||
@@ -74,25 +76,29 @@ specifying a path using --config or by setting VAULTIK_CONFIG to a path.`,
|
||||
// Start the snapshot creation in a goroutine
|
||||
go func() {
|
||||
// --cron suppression is wired through v.UI by setupGlobals.
|
||||
if err := v.CreateSnapshot(opts); err != nil {
|
||||
if err != context.Canceled {
|
||||
err := v.CreateSnapshot(opts)
|
||||
if err != nil {
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
log.Error("Snapshot creation failed", "error", err)
|
||||
ReportError("Snapshot creation failed: %v", err)
|
||||
ReportErrorf("Snapshot creation failed: %v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// Shutdown the app when snapshot completes
|
||||
if err := v.Shutdowner.Shutdown(); err != nil {
|
||||
err = v.Shutdowner.Shutdown()
|
||||
if err != nil {
|
||||
log.Error("Failed to shutdown", "error", err)
|
||||
}
|
||||
}()
|
||||
|
||||
return nil
|
||||
},
|
||||
OnStop: func(ctx context.Context) error {
|
||||
log.Debug("Stopping snapshot creation")
|
||||
// Cancel the Vaultik context
|
||||
v.Cancel()
|
||||
|
||||
return nil
|
||||
},
|
||||
})
|
||||
@@ -127,6 +133,7 @@ func newSnapshotListCommand() *cobra.Command {
|
||||
}
|
||||
|
||||
rootFlags := GetRootFlags()
|
||||
|
||||
return RunWithApp(cmd.Context(), AppOptions{
|
||||
ConfigPath: configPath,
|
||||
LogOptions: log.LogOptions{
|
||||
@@ -140,21 +147,26 @@ func newSnapshotListCommand() *cobra.Command {
|
||||
lc.Append(fx.Hook{
|
||||
OnStart: func(ctx context.Context) error {
|
||||
go func() {
|
||||
if err := v.ListSnapshots(jsonOutput); err != nil {
|
||||
if err != context.Canceled {
|
||||
err := v.ListSnapshots(jsonOutput)
|
||||
if err != nil {
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
log.Error("Failed to list snapshots", "error", err)
|
||||
ReportError("Failed to list snapshots: %v", err)
|
||||
ReportErrorf("Failed to list snapshots: %v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
if err := v.Shutdowner.Shutdown(); err != nil {
|
||||
|
||||
err = v.Shutdowner.Shutdown()
|
||||
if err != nil {
|
||||
log.Error("Failed to shutdown", "error", err)
|
||||
}
|
||||
}()
|
||||
|
||||
return nil
|
||||
},
|
||||
OnStop: func(ctx context.Context) error {
|
||||
v.Cancel()
|
||||
|
||||
return nil
|
||||
},
|
||||
})
|
||||
@@ -185,10 +197,11 @@ restrict the operation to specific snapshot names.`,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
// Validate flags
|
||||
if !opts.KeepLatest && opts.OlderThan == "" {
|
||||
return fmt.Errorf("must specify either --keep-latest or --older-than")
|
||||
return errors.New("must specify either --keep-latest or --older-than")
|
||||
}
|
||||
|
||||
if opts.KeepLatest && opts.OlderThan != "" {
|
||||
return fmt.Errorf("cannot specify both --keep-latest and --older-than")
|
||||
return errors.New("cannot specify both --keep-latest and --older-than")
|
||||
}
|
||||
|
||||
// Use unified config resolution
|
||||
@@ -198,6 +211,7 @@ restrict the operation to specific snapshot names.`,
|
||||
}
|
||||
|
||||
rootFlags := GetRootFlags()
|
||||
|
||||
return RunWithApp(cmd.Context(), AppOptions{
|
||||
ConfigPath: configPath,
|
||||
LogOptions: log.LogOptions{
|
||||
@@ -211,21 +225,26 @@ restrict the operation to specific snapshot names.`,
|
||||
lc.Append(fx.Hook{
|
||||
OnStart: func(ctx context.Context) error {
|
||||
go func() {
|
||||
if err := v.PurgeSnapshotsWithOptions(opts); err != nil {
|
||||
if err != context.Canceled {
|
||||
err := v.PurgeSnapshotsWithOptions(opts)
|
||||
if err != nil {
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
log.Error("Failed to purge snapshots", "error", err)
|
||||
ReportError("Failed to purge snapshots: %v", err)
|
||||
ReportErrorf("Failed to purge snapshots: %v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
if err := v.Shutdowner.Shutdown(); err != nil {
|
||||
|
||||
err = v.Shutdowner.Shutdown()
|
||||
if err != nil {
|
||||
log.Error("Failed to shutdown", "error", err)
|
||||
}
|
||||
}()
|
||||
|
||||
return nil
|
||||
},
|
||||
OnStop: func(ctx context.Context) error {
|
||||
v.Cancel()
|
||||
|
||||
return nil
|
||||
},
|
||||
})
|
||||
@@ -254,11 +273,14 @@ func newSnapshotVerifyCommand() *cobra.Command {
|
||||
Args: func(cmd *cobra.Command, args []string) error {
|
||||
if len(args) != 1 {
|
||||
_ = cmd.Help()
|
||||
|
||||
if len(args) == 0 {
|
||||
return fmt.Errorf("snapshot ID required")
|
||||
return errors.New("snapshot ID required")
|
||||
}
|
||||
|
||||
return fmt.Errorf("expected 1 argument, got %d", len(args))
|
||||
}
|
||||
|
||||
return nil
|
||||
},
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
@@ -271,6 +293,7 @@ func newSnapshotVerifyCommand() *cobra.Command {
|
||||
}
|
||||
|
||||
rootFlags := GetRootFlags()
|
||||
|
||||
return RunWithApp(cmd.Context(), AppOptions{
|
||||
ConfigPath: configPath,
|
||||
LogOptions: log.LogOptions{
|
||||
@@ -284,23 +307,29 @@ func newSnapshotVerifyCommand() *cobra.Command {
|
||||
lc.Append(fx.Hook{
|
||||
OnStart: func(ctx context.Context) error {
|
||||
go func() {
|
||||
if err := v.VerifySnapshotWithOptions(snapshotID, opts); err != nil {
|
||||
if err != context.Canceled {
|
||||
err := v.VerifySnapshotWithOptions(snapshotID, opts)
|
||||
if err != nil {
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
if !opts.JSON {
|
||||
log.Error("Verification failed", "error", err)
|
||||
ReportError("Verification failed: %v", err)
|
||||
ReportErrorf("Verification failed: %v", err)
|
||||
}
|
||||
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
if err := v.Shutdowner.Shutdown(); err != nil {
|
||||
|
||||
err = v.Shutdowner.Shutdown()
|
||||
if err != nil {
|
||||
log.Error("Failed to shutdown", "error", err)
|
||||
}
|
||||
}()
|
||||
|
||||
return nil
|
||||
},
|
||||
OnStop: func(ctx context.Context) error {
|
||||
v.Cancel()
|
||||
|
||||
return nil
|
||||
},
|
||||
})
|
||||
@@ -345,11 +374,14 @@ nuke --force' — it is the single supported entry point for that.`,
|
||||
Args: func(cmd *cobra.Command, args []string) error {
|
||||
if len(args) != 1 {
|
||||
_ = cmd.Help()
|
||||
|
||||
if len(args) == 0 {
|
||||
return fmt.Errorf("snapshot ID required")
|
||||
return errors.New("snapshot ID required")
|
||||
}
|
||||
|
||||
return fmt.Errorf("expected 1 argument, got %d", len(args))
|
||||
}
|
||||
|
||||
return nil
|
||||
},
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
@@ -360,6 +392,7 @@ nuke --force' — it is the single supported entry point for that.`,
|
||||
}
|
||||
|
||||
rootFlags := GetRootFlags()
|
||||
|
||||
return RunWithApp(cmd.Context(), AppOptions{
|
||||
ConfigPath: configPath,
|
||||
LogOptions: log.LogOptions{
|
||||
@@ -375,22 +408,26 @@ nuke --force' — it is the single supported entry point for that.`,
|
||||
go func() {
|
||||
_, err := v.RemoveSnapshot(args[0], opts)
|
||||
if err != nil {
|
||||
if err != context.Canceled {
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
if !opts.JSON {
|
||||
log.Error("Failed to remove snapshot", "error", err)
|
||||
ReportError("Failed to remove snapshot: %v", err)
|
||||
ReportErrorf("Failed to remove snapshot: %v", err)
|
||||
}
|
||||
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
if err := v.Shutdowner.Shutdown(); err != nil {
|
||||
log.Error("Failed to shutdown", "error", err)
|
||||
}
|
||||
}()
|
||||
|
||||
return nil
|
||||
},
|
||||
OnStop: func(ctx context.Context) error {
|
||||
v.Cancel()
|
||||
|
||||
return nil
|
||||
},
|
||||
})
|
||||
|
||||
@@ -2,6 +2,7 @@ package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
@@ -70,6 +71,7 @@ Examples:
|
||||
// runRestore parses arguments and runs the restore operation through the app framework
|
||||
func runRestore(cmd *cobra.Command, args []string, opts *RestoreOptions) error {
|
||||
snapshotID := args[0]
|
||||
|
||||
opts.TargetDir = args[1]
|
||||
if len(args) > 2 {
|
||||
opts.Paths = args[2:]
|
||||
@@ -83,6 +85,7 @@ func runRestore(cmd *cobra.Command, args []string, opts *RestoreOptions) error {
|
||||
|
||||
// Use the app framework like other commands
|
||||
rootFlags := GetRootFlags()
|
||||
|
||||
return RunWithApp(cmd.Context(), AppOptions{
|
||||
ConfigPath: configPath,
|
||||
LogOptions: log.LogOptions{
|
||||
@@ -129,24 +132,29 @@ func buildRestoreInvokes(snapshotID string, opts *RestoreOptions) []fx.Option {
|
||||
Verify: opts.Verify,
|
||||
SkipErrors: GetRootFlags().SkipErrors,
|
||||
}
|
||||
if err := app.Vaultik.Restore(restoreOpts); err != nil {
|
||||
if err != context.Canceled {
|
||||
|
||||
err := app.Vaultik.Restore(restoreOpts)
|
||||
if err != nil {
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
log.Error("Restore operation failed", "error", err)
|
||||
ReportError("Restore failed: %v", err)
|
||||
ReportErrorf("Restore failed: %v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// Shutdown the app when restore completes
|
||||
if err := app.Shutdowner.Shutdown(); err != nil {
|
||||
err = app.Shutdowner.Shutdown()
|
||||
if err != nil {
|
||||
log.Error("Failed to shutdown", "error", err)
|
||||
}
|
||||
}()
|
||||
|
||||
return nil
|
||||
},
|
||||
OnStop: func(ctx context.Context) error {
|
||||
log.Debug("Stopping restore operation")
|
||||
app.Vaultik.Cancel()
|
||||
|
||||
return nil
|
||||
},
|
||||
})
|
||||
|
||||
@@ -24,6 +24,7 @@ func NewVersionCommand() *cobra.Command {
|
||||
fmt.Printf(" author: %s\n", globals.Author)
|
||||
fmt.Printf(" homepage: %s\n", globals.Homepage)
|
||||
fmt.Printf(" license: %s\n", globals.License)
|
||||
|
||||
if globals.Version == "dev" {
|
||||
fmt.Println()
|
||||
fmt.Println("This is a development build (no version information embedded).")
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -21,12 +22,16 @@ const appName = "vaultik"
|
||||
func expandTilde(path string) string {
|
||||
if path == "~" {
|
||||
home, _ := os.UserHomeDir()
|
||||
|
||||
return home
|
||||
}
|
||||
|
||||
if strings.HasPrefix(path, "~/") {
|
||||
home, _ := os.UserHomeDir()
|
||||
|
||||
return filepath.Join(home, path[2:])
|
||||
}
|
||||
|
||||
return path
|
||||
}
|
||||
|
||||
@@ -34,8 +39,10 @@ func expandTilde(path string) string {
|
||||
func expandTildeInURL(url string) string {
|
||||
if strings.HasPrefix(url, "file://~/") {
|
||||
home, _ := os.UserHomeDir()
|
||||
|
||||
return "file://" + filepath.Join(home, url[9:])
|
||||
}
|
||||
|
||||
return url
|
||||
}
|
||||
|
||||
@@ -63,6 +70,7 @@ func (c *Config) GetExcludes(snapshotName string) []string {
|
||||
combined := make([]string, 0, len(c.Exclude)+len(snap.Exclude))
|
||||
combined = append(combined, c.Exclude...)
|
||||
combined = append(combined, snap.Exclude...)
|
||||
|
||||
return combined
|
||||
}
|
||||
|
||||
@@ -74,6 +82,7 @@ func (c *Config) SnapshotNames() []string {
|
||||
}
|
||||
// Sort for deterministic order
|
||||
sort.Strings(names)
|
||||
|
||||
return names
|
||||
}
|
||||
|
||||
@@ -126,7 +135,7 @@ type ConfigPath string
|
||||
// Returns an error if the path is empty or if loading fails.
|
||||
func New(path ConfigPath) (*Config, error) {
|
||||
if path == "" {
|
||||
return nil, fmt.Errorf("config path not provided")
|
||||
return nil, errors.New("config path not provided")
|
||||
}
|
||||
|
||||
cfg, err := Load(string(path))
|
||||
@@ -159,6 +168,7 @@ func Load(path string) (*Config, error) {
|
||||
|
||||
// Convert smartconfig data to YAML then unmarshal
|
||||
configData := sc.Data()
|
||||
|
||||
yamlBytes, err := yaml.Marshal(configData)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to marshal config data: %w", err)
|
||||
@@ -177,6 +187,7 @@ func Load(path string) (*Config, error) {
|
||||
for i, path := range snap.Paths {
|
||||
snap.Paths[i] = expandTilde(path)
|
||||
}
|
||||
|
||||
cfg.Snapshots[name] = snap
|
||||
}
|
||||
|
||||
@@ -196,6 +207,7 @@ func Load(path string) (*Config, error) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get hostname: %w", err)
|
||||
}
|
||||
|
||||
cfg.Hostname = hostname
|
||||
}
|
||||
|
||||
@@ -203,6 +215,7 @@ func Load(path string) (*Config, error) {
|
||||
if cfg.S3.Region == "" {
|
||||
cfg.S3.Region = "us-east-1"
|
||||
}
|
||||
|
||||
if cfg.S3.PartSize == 0 {
|
||||
cfg.S3.PartSize = Size(5 * 1024 * 1024) // 5MB
|
||||
}
|
||||
@@ -236,11 +249,11 @@ func Load(path string) (*Config, error) {
|
||||
// Returns an error describing the first validation failure encountered.
|
||||
func (c *Config) Validate() error {
|
||||
if len(c.AgeRecipients) == 0 {
|
||||
return fmt.Errorf("at least one age_recipient is required (generate with: age-keygen)")
|
||||
return errors.New("at least one age_recipient is required (generate with: age-keygen)")
|
||||
}
|
||||
|
||||
if len(c.Snapshots) == 0 {
|
||||
return fmt.Errorf("at least one snapshot must be configured (see config.example.yml)")
|
||||
return errors.New("at least one snapshot must be configured (see config.example.yml)")
|
||||
}
|
||||
|
||||
for name, snap := range c.Snapshots {
|
||||
@@ -250,20 +263,21 @@ func (c *Config) Validate() error {
|
||||
}
|
||||
|
||||
// Validate storage configuration
|
||||
if err := c.validateStorage(); err != nil {
|
||||
err := c.validateStorage()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if c.ChunkSize.Int64() < 1024*1024 { // 1MB minimum
|
||||
return fmt.Errorf("chunk_size must be at least 1MB")
|
||||
return errors.New("chunk_size must be at least 1MB")
|
||||
}
|
||||
|
||||
if c.BlobSizeLimit.Int64() < c.ChunkSize.Int64() {
|
||||
return fmt.Errorf("blob_size_limit must be at least chunk_size")
|
||||
return errors.New("blob_size_limit must be at least chunk_size")
|
||||
}
|
||||
|
||||
if c.CompressionLevel < 1 || c.CompressionLevel > 19 {
|
||||
return fmt.Errorf("compression_level must be between 1 and 19")
|
||||
return errors.New("compression_level must be between 1 and 19")
|
||||
}
|
||||
|
||||
return nil
|
||||
@@ -280,38 +294,43 @@ func (c *Config) validateStorage() error {
|
||||
// File storage doesn't need S3 credentials
|
||||
return nil
|
||||
}
|
||||
|
||||
if strings.HasPrefix(c.StorageURL, "s3://") {
|
||||
// S3 storage needs credentials
|
||||
if c.S3.AccessKeyID == "" {
|
||||
return fmt.Errorf("s3.access_key_id is required for s3:// URLs")
|
||||
return errors.New("s3.access_key_id is required for s3:// URLs")
|
||||
}
|
||||
|
||||
if c.S3.SecretAccessKey == "" {
|
||||
return fmt.Errorf("s3.secret_access_key is required for s3:// URLs")
|
||||
return errors.New("s3.secret_access_key is required for s3:// URLs")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
if strings.HasPrefix(c.StorageURL, "rclone://") {
|
||||
// Rclone storage uses rclone's own config
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("storage_url must start with s3://, file://, or rclone://")
|
||||
|
||||
return errors.New("storage_url must start with s3://, file://, or rclone://")
|
||||
}
|
||||
|
||||
// Legacy S3 configuration
|
||||
if c.S3.Endpoint == "" {
|
||||
return fmt.Errorf("storage not configured; set storage_url or provide s3.endpoint + s3.bucket + credentials")
|
||||
return errors.New("storage not configured; set storage_url or provide s3.endpoint + s3.bucket + credentials")
|
||||
}
|
||||
|
||||
if c.S3.Bucket == "" {
|
||||
return fmt.Errorf("s3.bucket is required (or set storage_url)")
|
||||
return errors.New("s3.bucket is required (or set storage_url)")
|
||||
}
|
||||
|
||||
if c.S3.AccessKeyID == "" {
|
||||
return fmt.Errorf("s3.access_key_id is required")
|
||||
return errors.New("s3.access_key_id is required")
|
||||
}
|
||||
|
||||
if c.S3.SecretAccessKey == "" {
|
||||
return fmt.Errorf("s3.secret_access_key is required")
|
||||
return errors.New("s3.secret_access_key is required")
|
||||
}
|
||||
|
||||
return nil
|
||||
@@ -329,6 +348,7 @@ func extractAgeSecretKey(input string) string {
|
||||
if id, ok := identities[0].(*age.X25519Identity); ok {
|
||||
return id.String()
|
||||
}
|
||||
|
||||
return strings.TrimSpace(input)
|
||||
}
|
||||
|
||||
|
||||
@@ -41,6 +41,7 @@ func TestConfigLoad(t *testing.T) {
|
||||
if len(cfg.AgeRecipients) != 2 {
|
||||
t.Errorf("Expected 2 age recipients, got %d", len(cfg.AgeRecipients))
|
||||
}
|
||||
|
||||
if cfg.AgeRecipients[0] != TEST_SNEAK_AGE_PUBLIC_KEY {
|
||||
t.Errorf("Expected first age recipient to be %s, got '%s'", TEST_SNEAK_AGE_PUBLIC_KEY, cfg.AgeRecipients[0])
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/dustin/go-humanize"
|
||||
@@ -14,18 +15,19 @@ type Size int64
|
||||
// UnmarshalYAML implements yaml.Unmarshaler for Size, allowing it to be
|
||||
// parsed from YAML configuration files. It accepts both numeric values
|
||||
// (interpreted as bytes) and string values with units (e.g., "10MB").
|
||||
func (s *Size) UnmarshalYAML(unmarshal func(interface{}) error) error {
|
||||
func (s *Size) UnmarshalYAML(unmarshal func(any) error) error {
|
||||
// Try to unmarshal as int64 first
|
||||
var intVal int64
|
||||
if err := unmarshal(&intVal); err == nil {
|
||||
*s = Size(intVal)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Try to unmarshal as string
|
||||
var strVal string
|
||||
if err := unmarshal(&strVal); err != nil {
|
||||
return fmt.Errorf("size must be a number or string")
|
||||
return errors.New("size must be a number or string")
|
||||
}
|
||||
|
||||
// Parse the string using go-humanize
|
||||
@@ -35,6 +37,7 @@ func (s *Size) UnmarshalYAML(unmarshal func(interface{}) error) error {
|
||||
}
|
||||
|
||||
*s = Size(bytes)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -58,5 +61,6 @@ func ParseSize(s string) (Size, error) {
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("invalid size format: %w", err)
|
||||
}
|
||||
|
||||
return Size(bytes), nil
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package crypto
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"sync"
|
||||
@@ -25,7 +26,7 @@ type Encryptor struct {
|
||||
// public keys are invalid or if no recipients are specified.
|
||||
func NewEncryptor(publicKeys []string) (*Encryptor, error) {
|
||||
if len(publicKeys) == 0 {
|
||||
return nil, fmt.Errorf("at least one recipient is required")
|
||||
return nil, errors.New("at least one recipient is required")
|
||||
}
|
||||
|
||||
recipients := make([]age.Recipient, 0, len(publicKeys))
|
||||
@@ -34,6 +35,7 @@ func NewEncryptor(publicKeys []string) (*Encryptor, error) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parsing age recipient %s: %w", key, err)
|
||||
}
|
||||
|
||||
recipients = append(recipients, recipient)
|
||||
}
|
||||
|
||||
@@ -126,7 +128,7 @@ func (e *Encryptor) EncryptWriter(dst io.Writer) (io.WriteCloser, error) {
|
||||
// of the public keys are invalid or if no recipients are specified.
|
||||
func (e *Encryptor) UpdateRecipients(publicKeys []string) error {
|
||||
if len(publicKeys) == 0 {
|
||||
return fmt.Errorf("at least one recipient is required")
|
||||
return errors.New("at least one recipient is required")
|
||||
}
|
||||
|
||||
recipients := make([]age.Recipient, 0, len(publicKeys))
|
||||
@@ -135,6 +137,7 @@ func (e *Encryptor) UpdateRecipients(publicKeys []string) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("parsing age recipient %s: %w", key, err)
|
||||
}
|
||||
|
||||
recipients = append(recipients, recipient)
|
||||
}
|
||||
|
||||
|
||||
@@ -58,10 +58,12 @@ func TestEncryptorMultipleRecipients(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to generate identity1: %v", err)
|
||||
}
|
||||
|
||||
identity2, err := age.GenerateX25519Identity()
|
||||
if err != nil {
|
||||
t.Fatalf("failed to generate identity2: %v", err)
|
||||
}
|
||||
|
||||
identity3, err := age.GenerateX25519Identity()
|
||||
if err != nil {
|
||||
t.Fatalf("failed to generate identity3: %v", err)
|
||||
@@ -123,6 +125,7 @@ func TestEncryptorUpdateRecipients(t *testing.T) {
|
||||
|
||||
// Encrypt with first key
|
||||
plaintext := []byte("test data")
|
||||
|
||||
ciphertext1, err := enc.Encrypt(plaintext)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to encrypt: %v", err)
|
||||
@@ -143,6 +146,7 @@ func TestEncryptorUpdateRecipients(t *testing.T) {
|
||||
if _, err := age.Decrypt(bytes.NewReader(ciphertext1), identity1); err != nil {
|
||||
t.Error("failed to decrypt with identity1")
|
||||
}
|
||||
|
||||
if _, err := age.Decrypt(bytes.NewReader(ciphertext1), identity2); err == nil {
|
||||
t.Error("should not decrypt with identity2")
|
||||
}
|
||||
@@ -151,6 +155,7 @@ func TestEncryptorUpdateRecipients(t *testing.T) {
|
||||
if _, err := age.Decrypt(bytes.NewReader(ciphertext2), identity2); err != nil {
|
||||
t.Error("failed to decrypt with identity2")
|
||||
}
|
||||
|
||||
if _, err := age.Decrypt(bytes.NewReader(ciphertext2), identity1); err == nil {
|
||||
t.Error("should not decrypt with identity1")
|
||||
}
|
||||
|
||||
@@ -49,12 +49,15 @@ func (r *BlobChunkRepository) GetByBlobID(ctx context.Context, blobID string) ([
|
||||
defer CloseRows(rows)
|
||||
|
||||
var blobChunks []*BlobChunk
|
||||
|
||||
for rows.Next() {
|
||||
var bc BlobChunk
|
||||
|
||||
err := rows.Scan(&bc.BlobID, &bc.ChunkHash, &bc.Offset, &bc.Length)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scanning blob chunk: %w", err)
|
||||
}
|
||||
|
||||
blobChunks = append(blobChunks, &bc)
|
||||
}
|
||||
|
||||
@@ -70,7 +73,9 @@ func (r *BlobChunkRepository) GetByChunkHash(ctx context.Context, chunkHash stri
|
||||
`
|
||||
|
||||
LogSQL("GetByChunkHash", query, chunkHash)
|
||||
|
||||
var bc BlobChunk
|
||||
|
||||
err := r.db.conn.QueryRowContext(ctx, query, chunkHash).Scan(
|
||||
&bc.BlobID,
|
||||
&bc.ChunkHash,
|
||||
@@ -80,14 +85,18 @@ func (r *BlobChunkRepository) GetByChunkHash(ctx context.Context, chunkHash stri
|
||||
|
||||
if err == sql.ErrNoRows {
|
||||
LogSQL("GetByChunkHash", "No rows found", chunkHash)
|
||||
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
LogSQL("GetByChunkHash", "Error", chunkHash, err)
|
||||
|
||||
return nil, fmt.Errorf("querying blob chunk: %w", err)
|
||||
}
|
||||
|
||||
LogSQL("GetByChunkHash", "Found blob", chunkHash, "blob", bc.BlobID)
|
||||
|
||||
return &bc, nil
|
||||
}
|
||||
|
||||
@@ -101,7 +110,9 @@ func (r *BlobChunkRepository) GetByChunkHashTx(ctx context.Context, tx *sql.Tx,
|
||||
`
|
||||
|
||||
LogSQL("GetByChunkHashTx", query, chunkHash)
|
||||
|
||||
var bc BlobChunk
|
||||
|
||||
err := tx.QueryRowContext(ctx, query, chunkHash).Scan(
|
||||
&bc.BlobID,
|
||||
&bc.ChunkHash,
|
||||
@@ -111,14 +122,18 @@ func (r *BlobChunkRepository) GetByChunkHashTx(ctx context.Context, tx *sql.Tx,
|
||||
|
||||
if err == sql.ErrNoRows {
|
||||
LogSQL("GetByChunkHashTx", "No rows found", chunkHash)
|
||||
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
LogSQL("GetByChunkHashTx", "Error", chunkHash, err)
|
||||
|
||||
return nil, fmt.Errorf("querying blob chunk: %w", err)
|
||||
}
|
||||
|
||||
LogSQL("GetByChunkHashTx", "Found blob", chunkHash, "blob", bc.BlobID)
|
||||
|
||||
return &bc, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -22,6 +22,7 @@ func TestBlobChunkRepository(t *testing.T) {
|
||||
Hash: types.BlobHash("blob1-hash"),
|
||||
CreatedTS: time.Now(),
|
||||
}
|
||||
|
||||
err := repos.Blobs.Create(ctx, nil, blob)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create blob: %v", err)
|
||||
@@ -34,6 +35,7 @@ func TestBlobChunkRepository(t *testing.T) {
|
||||
ChunkHash: chunkHash,
|
||||
Size: 1024,
|
||||
}
|
||||
|
||||
err = repos.Chunks.Create(ctx, nil, chunk)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create chunk %s: %v", chunkHash, err)
|
||||
@@ -60,6 +62,7 @@ func TestBlobChunkRepository(t *testing.T) {
|
||||
Offset: 1024,
|
||||
Length: 2048,
|
||||
}
|
||||
|
||||
err = repos.BlobChunks.Create(ctx, nil, bc2)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create second blob chunk: %v", err)
|
||||
@@ -71,6 +74,7 @@ func TestBlobChunkRepository(t *testing.T) {
|
||||
Offset: 3072,
|
||||
Length: 512,
|
||||
}
|
||||
|
||||
err = repos.BlobChunks.Create(ctx, nil, bc3)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create third blob chunk: %v", err)
|
||||
@@ -81,6 +85,7 @@ func TestBlobChunkRepository(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get blob chunks: %v", err)
|
||||
}
|
||||
|
||||
if len(blobChunks) != 3 {
|
||||
t.Errorf("expected 3 chunks, got %d", len(blobChunks))
|
||||
}
|
||||
@@ -98,12 +103,15 @@ func TestBlobChunkRepository(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get blob chunk by chunk hash: %v", err)
|
||||
}
|
||||
|
||||
if bc == nil {
|
||||
t.Fatal("expected blob chunk, got nil")
|
||||
}
|
||||
|
||||
if bc.BlobID != blob.ID {
|
||||
t.Errorf("wrong blob ID: expected %s, got %s", blob.ID, bc.BlobID)
|
||||
}
|
||||
|
||||
if bc.Offset != 1024 {
|
||||
t.Errorf("wrong offset: expected 1024, got %d", bc.Offset)
|
||||
}
|
||||
@@ -113,6 +121,7 @@ func TestBlobChunkRepository(t *testing.T) {
|
||||
if err == nil {
|
||||
t.Fatal("duplicate blob_chunk insert should fail due to primary key constraint")
|
||||
}
|
||||
|
||||
if !strings.Contains(err.Error(), "UNIQUE") && !strings.Contains(err.Error(), "constraint") {
|
||||
t.Fatalf("expected constraint error, got: %v", err)
|
||||
}
|
||||
@@ -122,6 +131,7 @@ func TestBlobChunkRepository(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if bc != nil {
|
||||
t.Error("expected nil for non-existent chunk")
|
||||
}
|
||||
@@ -150,6 +160,7 @@ func TestBlobChunkRepositoryMultipleBlobs(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create blob1: %v", err)
|
||||
}
|
||||
|
||||
err = repos.Blobs.Create(ctx, nil, blob2)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create blob2: %v", err)
|
||||
@@ -162,6 +173,7 @@ func TestBlobChunkRepositoryMultipleBlobs(t *testing.T) {
|
||||
ChunkHash: chunkHash,
|
||||
Size: 1024,
|
||||
}
|
||||
|
||||
err = repos.Chunks.Create(ctx, nil, chunk)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create chunk %s: %v", chunkHash, err)
|
||||
@@ -189,6 +201,7 @@ func TestBlobChunkRepositoryMultipleBlobs(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get blob1 chunks: %v", err)
|
||||
}
|
||||
|
||||
if len(chunks) != 2 {
|
||||
t.Errorf("expected 2 chunks for blob1, got %d", len(chunks))
|
||||
}
|
||||
@@ -198,6 +211,7 @@ func TestBlobChunkRepositoryMultipleBlobs(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get blob2 chunks: %v", err)
|
||||
}
|
||||
|
||||
if len(chunks) != 2 {
|
||||
t.Errorf("expected 2 chunks for blob2, got %d", len(chunks))
|
||||
}
|
||||
@@ -207,6 +221,7 @@ func TestBlobChunkRepositoryMultipleBlobs(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get shared chunk: %v", err)
|
||||
}
|
||||
|
||||
if bc == nil {
|
||||
t.Fatal("expected shared chunk, got nil")
|
||||
}
|
||||
|
||||
@@ -24,10 +24,12 @@ func (r *BlobRepository) Create(ctx context.Context, tx *sql.Tx, blob *Blob) err
|
||||
`
|
||||
|
||||
var finishedTS, uploadedTS *int64
|
||||
|
||||
if blob.FinishedTS != nil {
|
||||
ts := blob.FinishedTS.Unix()
|
||||
finishedTS = &ts
|
||||
}
|
||||
|
||||
if blob.UploadedTS != nil {
|
||||
ts := blob.UploadedTS.Unix()
|
||||
uploadedTS = &ts
|
||||
@@ -56,9 +58,11 @@ func (r *BlobRepository) GetByHash(ctx context.Context, hash string) (*Blob, err
|
||||
WHERE blob_hash = ?
|
||||
`
|
||||
|
||||
var blob Blob
|
||||
var createdTSUnix int64
|
||||
var finishedTSUnix, uploadedTSUnix sql.NullInt64
|
||||
var (
|
||||
blob Blob
|
||||
createdTSUnix int64
|
||||
finishedTSUnix, uploadedTSUnix sql.NullInt64
|
||||
)
|
||||
|
||||
err := r.db.conn.QueryRowContext(ctx, query, hash).Scan(
|
||||
&blob.ID,
|
||||
@@ -73,6 +77,7 @@ func (r *BlobRepository) GetByHash(ctx context.Context, hash string) (*Blob, err
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("querying blob: %w", err)
|
||||
}
|
||||
@@ -82,10 +87,12 @@ func (r *BlobRepository) GetByHash(ctx context.Context, hash string) (*Blob, err
|
||||
ts := time.Unix(finishedTSUnix.Int64, 0).UTC()
|
||||
blob.FinishedTS = &ts
|
||||
}
|
||||
|
||||
if uploadedTSUnix.Valid {
|
||||
ts := time.Unix(uploadedTSUnix.Int64, 0).UTC()
|
||||
blob.UploadedTS = &ts
|
||||
}
|
||||
|
||||
return &blob, nil
|
||||
}
|
||||
|
||||
@@ -97,9 +104,11 @@ func (r *BlobRepository) GetByID(ctx context.Context, id string) (*Blob, error)
|
||||
WHERE id = ?
|
||||
`
|
||||
|
||||
var blob Blob
|
||||
var createdTSUnix int64
|
||||
var finishedTSUnix, uploadedTSUnix sql.NullInt64
|
||||
var (
|
||||
blob Blob
|
||||
createdTSUnix int64
|
||||
finishedTSUnix, uploadedTSUnix sql.NullInt64
|
||||
)
|
||||
|
||||
err := r.db.conn.QueryRowContext(ctx, query, id).Scan(
|
||||
&blob.ID,
|
||||
@@ -114,6 +123,7 @@ func (r *BlobRepository) GetByID(ctx context.Context, id string) (*Blob, error)
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("querying blob: %w", err)
|
||||
}
|
||||
@@ -123,10 +133,12 @@ func (r *BlobRepository) GetByID(ctx context.Context, id string) (*Blob, error)
|
||||
ts := time.Unix(finishedTSUnix.Int64, 0).UTC()
|
||||
blob.FinishedTS = &ts
|
||||
}
|
||||
|
||||
if uploadedTSUnix.Valid {
|
||||
ts := time.Unix(uploadedTSUnix.Int64, 0).UTC()
|
||||
blob.UploadedTS = &ts
|
||||
}
|
||||
|
||||
return &blob, nil
|
||||
}
|
||||
|
||||
@@ -146,11 +158,15 @@ func (r *BlobRepository) GetAll(ctx context.Context) (map[string]*Blob, error) {
|
||||
defer CloseRows(rows)
|
||||
|
||||
out := make(map[string]*Blob)
|
||||
|
||||
for rows.Next() {
|
||||
var blob Blob
|
||||
var createdTSUnix int64
|
||||
var finishedTSUnix, uploadedTSUnix sql.NullInt64
|
||||
if err := rows.Scan(
|
||||
var (
|
||||
blob Blob
|
||||
createdTSUnix int64
|
||||
finishedTSUnix, uploadedTSUnix sql.NullInt64
|
||||
)
|
||||
|
||||
err := rows.Scan(
|
||||
&blob.ID,
|
||||
&blob.Hash,
|
||||
&createdTSUnix,
|
||||
@@ -158,20 +174,25 @@ func (r *BlobRepository) GetAll(ctx context.Context) (map[string]*Blob, error) {
|
||||
&blob.UncompressedSize,
|
||||
&blob.CompressedSize,
|
||||
&uploadedTSUnix,
|
||||
); err != nil {
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scanning blob: %w", err)
|
||||
}
|
||||
|
||||
blob.CreatedTS = time.Unix(createdTSUnix, 0).UTC()
|
||||
if finishedTSUnix.Valid {
|
||||
ts := time.Unix(finishedTSUnix.Int64, 0).UTC()
|
||||
blob.FinishedTS = &ts
|
||||
}
|
||||
|
||||
if uploadedTSUnix.Valid {
|
||||
ts := time.Unix(uploadedTSUnix.Int64, 0).UTC()
|
||||
blob.UploadedTS = &ts
|
||||
}
|
||||
|
||||
out[blob.ID.String()] = &blob
|
||||
}
|
||||
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
@@ -184,6 +205,7 @@ func (r *BlobRepository) UpdateFinished(ctx context.Context, tx *sql.Tx, id stri
|
||||
`
|
||||
|
||||
now := time.Now().UTC().Unix()
|
||||
|
||||
var err error
|
||||
if tx != nil {
|
||||
_, err = tx.ExecContext(ctx, query, hash, now, uncompressedSize, compressedSize, id)
|
||||
@@ -207,6 +229,7 @@ func (r *BlobRepository) UpdateUploaded(ctx context.Context, tx *sql.Tx, id stri
|
||||
`
|
||||
|
||||
now := time.Now().UTC().Unix()
|
||||
|
||||
var err error
|
||||
if tx != nil {
|
||||
_, err = tx.ExecContext(ctx, query, now, id)
|
||||
|
||||
@@ -32,12 +32,15 @@ func TestBlobRepository(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get blob: %v", err)
|
||||
}
|
||||
|
||||
if retrieved == nil {
|
||||
t.Fatal("expected blob, got nil")
|
||||
}
|
||||
|
||||
if retrieved.Hash != blob.Hash {
|
||||
t.Errorf("blob hash mismatch: got %s, want %s", retrieved.Hash, blob.Hash)
|
||||
}
|
||||
|
||||
if !retrieved.CreatedTS.Equal(blob.CreatedTS) {
|
||||
t.Errorf("created timestamp mismatch: got %v, want %v", retrieved.CreatedTS, blob.CreatedTS)
|
||||
}
|
||||
@@ -47,9 +50,11 @@ func TestBlobRepository(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get blob by ID: %v", err)
|
||||
}
|
||||
|
||||
if retrievedByID == nil {
|
||||
t.Fatal("expected blob, got nil")
|
||||
}
|
||||
|
||||
if retrievedByID.ID != blob.ID {
|
||||
t.Errorf("blob ID mismatch: got %s, want %s", retrievedByID.ID, blob.ID)
|
||||
}
|
||||
@@ -60,6 +65,7 @@ func TestBlobRepository(t *testing.T) {
|
||||
Hash: types.BlobHash("blobhash456"),
|
||||
CreatedTS: time.Now().Truncate(time.Second),
|
||||
}
|
||||
|
||||
err = repo.Create(ctx, nil, blob2)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create second blob: %v", err)
|
||||
@@ -67,6 +73,7 @@ func TestBlobRepository(t *testing.T) {
|
||||
|
||||
// Test UpdateFinished
|
||||
now := time.Now()
|
||||
|
||||
err = repo.UpdateFinished(ctx, nil, blob.ID.String(), blob.Hash.String(), 1000, 500)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to update blob as finished: %v", err)
|
||||
@@ -77,12 +84,15 @@ func TestBlobRepository(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get updated blob: %v", err)
|
||||
}
|
||||
|
||||
if updated.FinishedTS == nil {
|
||||
t.Fatal("expected finished timestamp to be set")
|
||||
}
|
||||
|
||||
if updated.UncompressedSize != 1000 {
|
||||
t.Errorf("expected uncompressed size 1000, got %d", updated.UncompressedSize)
|
||||
}
|
||||
|
||||
if updated.CompressedSize != 500 {
|
||||
t.Errorf("expected compressed size 500, got %d", updated.CompressedSize)
|
||||
}
|
||||
@@ -98,6 +108,7 @@ func TestBlobRepository(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get uploaded blob: %v", err)
|
||||
}
|
||||
|
||||
if uploaded.UploadedTS == nil {
|
||||
t.Fatal("expected uploaded timestamp to be set")
|
||||
}
|
||||
|
||||
@@ -19,10 +19,12 @@ func TestCascadeDeleteDebug(t *testing.T) {
|
||||
|
||||
// Check if foreign keys are enabled
|
||||
var fkEnabled int
|
||||
|
||||
err := db.conn.QueryRow("PRAGMA foreign_keys").Scan(&fkEnabled)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
t.Logf("Foreign keys enabled: %d", fkEnabled)
|
||||
|
||||
// Create a file
|
||||
@@ -34,18 +36,21 @@ func TestCascadeDeleteDebug(t *testing.T) {
|
||||
UID: 1000,
|
||||
GID: 1000,
|
||||
}
|
||||
|
||||
err = repos.Files.Create(ctx, nil, file)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create file: %v", err)
|
||||
}
|
||||
|
||||
t.Logf("Created file with ID: %s", file.ID)
|
||||
|
||||
// Create chunks and file-chunk mappings
|
||||
for i := 0; i < 3; i++ {
|
||||
for i := range 3 {
|
||||
chunk := &Chunk{
|
||||
ChunkHash: types.ChunkHash(fmt.Sprintf("cascade-chunk-%d", i)),
|
||||
Size: 1024,
|
||||
}
|
||||
|
||||
err = repos.Chunks.Create(ctx, nil, chunk)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create chunk: %v", err)
|
||||
@@ -56,10 +61,12 @@ func TestCascadeDeleteDebug(t *testing.T) {
|
||||
Idx: i,
|
||||
ChunkHash: chunk.ChunkHash,
|
||||
}
|
||||
|
||||
err = repos.FileChunks.Create(ctx, nil, fc)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create file chunk: %v", err)
|
||||
}
|
||||
|
||||
t.Logf("Created file chunk mapping: file_id=%s, idx=%d, chunk=%s", fc.FileID, fc.Idx, fc.ChunkHash)
|
||||
}
|
||||
|
||||
@@ -68,10 +75,12 @@ func TestCascadeDeleteDebug(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
t.Logf("File chunks before delete: %d", len(fileChunks))
|
||||
|
||||
// Check the foreign key constraint
|
||||
var fkInfo string
|
||||
|
||||
err = db.conn.QueryRow(`
|
||||
SELECT sql FROM sqlite_master
|
||||
WHERE type='table' AND name='file_chunks'
|
||||
@@ -79,10 +88,12 @@ func TestCascadeDeleteDebug(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
t.Logf("file_chunks table definition:\n%s", fkInfo)
|
||||
|
||||
// Delete the file
|
||||
t.Log("Deleting file...")
|
||||
|
||||
err = repos.Files.DeleteByID(ctx, nil, file.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to delete file: %v", err)
|
||||
@@ -93,6 +104,7 @@ func TestCascadeDeleteDebug(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if deletedFile != nil {
|
||||
t.Error("file should have been deleted")
|
||||
} else {
|
||||
@@ -104,14 +116,17 @@ func TestCascadeDeleteDebug(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
t.Logf("File chunks after delete: %d", len(fileChunks))
|
||||
|
||||
// Manually check the database
|
||||
var count int
|
||||
|
||||
err = db.conn.QueryRow("SELECT COUNT(*) FROM file_chunks WHERE file_id = ?", file.ID).Scan(&count)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
t.Logf("Manual count of file_chunks for deleted file: %d", count)
|
||||
|
||||
if len(fileChunks) != 0 {
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"sneak.berlin/go/vaultik/internal/types"
|
||||
)
|
||||
@@ -90,18 +91,25 @@ func (r *ChunkFileRepository) GetByFileID(ctx context.Context, fileID types.File
|
||||
// scanChunkFiles is a helper that scans chunk file rows
|
||||
func (r *ChunkFileRepository) scanChunkFiles(rows *sql.Rows) ([]*ChunkFile, error) {
|
||||
var chunkFiles []*ChunkFile
|
||||
|
||||
for rows.Next() {
|
||||
var cf ChunkFile
|
||||
var chunkHashStr, fileIDStr string
|
||||
var (
|
||||
cf ChunkFile
|
||||
chunkHashStr, fileIDStr string
|
||||
)
|
||||
|
||||
err := rows.Scan(&chunkHashStr, &fileIDStr, &cf.FileOffset, &cf.Length)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scanning chunk file: %w", err)
|
||||
}
|
||||
|
||||
cf.ChunkHash = types.ChunkHash(chunkHashStr)
|
||||
|
||||
cf.FileID, err = types.ParseFileID(fileIDStr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parsing file ID: %w", err)
|
||||
}
|
||||
|
||||
chunkFiles = append(chunkFiles, &cf)
|
||||
}
|
||||
|
||||
@@ -136,14 +144,13 @@ func (r *ChunkFileRepository) DeleteByFileIDs(ctx context.Context, tx *sql.Tx, f
|
||||
const batchSize = 500
|
||||
|
||||
for i := 0; i < len(fileIDs); i += batchSize {
|
||||
end := i + batchSize
|
||||
if end > len(fileIDs) {
|
||||
end = len(fileIDs)
|
||||
}
|
||||
end := min(i+batchSize, len(fileIDs))
|
||||
|
||||
batch := fileIDs[i:end]
|
||||
|
||||
query := "DELETE FROM chunk_files WHERE file_id IN (?" + repeatPlaceholder(len(batch)-1) + ")"
|
||||
args := make([]interface{}, len(batch))
|
||||
|
||||
args := make([]any, len(batch))
|
||||
for j, id := range batch {
|
||||
args[j] = id.String()
|
||||
}
|
||||
@@ -154,6 +161,7 @@ func (r *ChunkFileRepository) DeleteByFileIDs(ctx context.Context, tx *sql.Tx, f
|
||||
} else {
|
||||
_, err = r.db.ExecWithLog(ctx, query, args...)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("batch deleting chunk_files: %w", err)
|
||||
}
|
||||
@@ -172,21 +180,28 @@ func (r *ChunkFileRepository) CreateBatch(ctx context.Context, tx *sql.Tx, cfs [
|
||||
const batchSize = 200
|
||||
|
||||
for i := 0; i < len(cfs); i += batchSize {
|
||||
end := i + batchSize
|
||||
if end > len(cfs) {
|
||||
end = len(cfs)
|
||||
}
|
||||
end := min(i+batchSize, len(cfs))
|
||||
|
||||
batch := cfs[i:end]
|
||||
|
||||
query := "INSERT INTO chunk_files (chunk_hash, file_id, file_offset, length) VALUES "
|
||||
args := make([]interface{}, 0, len(batch)*4)
|
||||
|
||||
args := make([]any, 0, len(batch)*4)
|
||||
|
||||
var querySb183 strings.Builder
|
||||
|
||||
for j, cf := range batch {
|
||||
if j > 0 {
|
||||
query += ", "
|
||||
querySb183.WriteString(", ")
|
||||
}
|
||||
query += "(?, ?, ?, ?)"
|
||||
|
||||
querySb183.WriteString("(?, ?, ?, ?)")
|
||||
|
||||
args = append(args, cf.ChunkHash.String(), cf.FileID.String(), cf.FileOffset, cf.Length)
|
||||
}
|
||||
|
||||
query += querySb183.String()
|
||||
|
||||
query += " ON CONFLICT(chunk_hash, file_id) DO NOTHING"
|
||||
|
||||
var err error
|
||||
@@ -195,6 +210,7 @@ func (r *ChunkFileRepository) CreateBatch(ctx context.Context, tx *sql.Tx, cfs [
|
||||
} else {
|
||||
_, err = r.db.ExecWithLog(ctx, query, args...)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("batch inserting chunk_files: %w", err)
|
||||
}
|
||||
|
||||
@@ -28,6 +28,7 @@ func TestChunkFileRepository(t *testing.T) {
|
||||
GID: 1000,
|
||||
LinkTarget: "",
|
||||
}
|
||||
|
||||
err := fileRepo.Create(ctx, nil, file1)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create file1: %v", err)
|
||||
@@ -42,6 +43,7 @@ func TestChunkFileRepository(t *testing.T) {
|
||||
GID: 1000,
|
||||
LinkTarget: "",
|
||||
}
|
||||
|
||||
err = fileRepo.Create(ctx, nil, file2)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create file2: %v", err)
|
||||
@@ -52,6 +54,7 @@ func TestChunkFileRepository(t *testing.T) {
|
||||
ChunkHash: types.ChunkHash("chunk1"),
|
||||
Size: 1024,
|
||||
}
|
||||
|
||||
err = chunksRepo.Create(ctx, nil, chunk)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create chunk: %v", err)
|
||||
@@ -77,6 +80,7 @@ func TestChunkFileRepository(t *testing.T) {
|
||||
FileOffset: 2048,
|
||||
Length: 1024,
|
||||
}
|
||||
|
||||
err = repo.Create(ctx, nil, cf2)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create second chunk file: %v", err)
|
||||
@@ -87,6 +91,7 @@ func TestChunkFileRepository(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get chunk files: %v", err)
|
||||
}
|
||||
|
||||
if len(chunkFiles) != 2 {
|
||||
t.Errorf("expected 2 files for chunk, got %d", len(chunkFiles))
|
||||
}
|
||||
@@ -94,14 +99,17 @@ func TestChunkFileRepository(t *testing.T) {
|
||||
// Verify both files are returned
|
||||
foundFile1 := false
|
||||
foundFile2 := false
|
||||
|
||||
for _, cf := range chunkFiles {
|
||||
if cf.FileID == file1.ID && cf.FileOffset == 0 {
|
||||
foundFile1 = true
|
||||
}
|
||||
|
||||
if cf.FileID == file2.ID && cf.FileOffset == 2048 {
|
||||
foundFile2 = true
|
||||
}
|
||||
}
|
||||
|
||||
if !foundFile1 || !foundFile2 {
|
||||
t.Error("not all expected files found")
|
||||
}
|
||||
@@ -111,9 +119,11 @@ func TestChunkFileRepository(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get chunks by file ID: %v", err)
|
||||
}
|
||||
|
||||
if len(chunkFiles) != 1 {
|
||||
t.Errorf("expected 1 chunk for file, got %d", len(chunkFiles))
|
||||
}
|
||||
|
||||
if chunkFiles[0].ChunkHash != types.ChunkHash("chunk1") {
|
||||
t.Errorf("wrong chunk hash: expected chunk1, got %s", chunkFiles[0].ChunkHash)
|
||||
}
|
||||
@@ -143,9 +153,11 @@ func TestChunkFileRepositoryComplexDeduplication(t *testing.T) {
|
||||
if err := fileRepo.Create(ctx, nil, file1); err != nil {
|
||||
t.Fatalf("failed to create file1: %v", err)
|
||||
}
|
||||
|
||||
if err := fileRepo.Create(ctx, nil, file2); err != nil {
|
||||
t.Fatalf("failed to create file2: %v", err)
|
||||
}
|
||||
|
||||
if err := fileRepo.Create(ctx, nil, file3); err != nil {
|
||||
t.Fatalf("failed to create file3: %v", err)
|
||||
}
|
||||
@@ -157,6 +169,7 @@ func TestChunkFileRepositoryComplexDeduplication(t *testing.T) {
|
||||
ChunkHash: chunkHash,
|
||||
Size: 1024,
|
||||
}
|
||||
|
||||
err := chunksRepo.Create(ctx, nil, chunk)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create chunk %s: %v", chunkHash, err)
|
||||
@@ -194,6 +207,7 @@ func TestChunkFileRepositoryComplexDeduplication(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get files for chunk1: %v", err)
|
||||
}
|
||||
|
||||
if len(files) != 2 {
|
||||
t.Errorf("expected 2 files for chunk1, got %d", len(files))
|
||||
}
|
||||
@@ -203,6 +217,7 @@ func TestChunkFileRepositoryComplexDeduplication(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get files for chunk2: %v", err)
|
||||
}
|
||||
|
||||
if len(files) != 2 {
|
||||
t.Errorf("expected 2 files for chunk2, got %d", len(files))
|
||||
}
|
||||
@@ -212,6 +227,7 @@ func TestChunkFileRepositoryComplexDeduplication(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get chunks for file2: %v", err)
|
||||
}
|
||||
|
||||
if len(file2Chunks) != 3 {
|
||||
t.Errorf("expected 3 chunks for file2, got %d", len(file2Chunks))
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
)
|
||||
@@ -54,6 +55,7 @@ func (r *ChunkRepository) GetByHash(ctx context.Context, hash string) (*Chunk, e
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("querying chunk: %w", err)
|
||||
}
|
||||
@@ -71,14 +73,22 @@ func (r *ChunkRepository) GetByHashes(ctx context.Context, hashes []string) ([]*
|
||||
FROM chunks
|
||||
WHERE chunk_hash IN (`
|
||||
|
||||
args := make([]interface{}, len(hashes))
|
||||
args := make([]any, len(hashes))
|
||||
|
||||
var querySb75 strings.Builder
|
||||
|
||||
for i, hash := range hashes {
|
||||
if i > 0 {
|
||||
query += ", "
|
||||
querySb75.WriteString(", ")
|
||||
}
|
||||
query += "?"
|
||||
|
||||
querySb75.WriteString("?")
|
||||
|
||||
args[i] = hash
|
||||
}
|
||||
|
||||
query += querySb75.String()
|
||||
|
||||
query += ") ORDER BY chunk_hash"
|
||||
|
||||
rows, err := r.db.conn.QueryContext(ctx, query, args...)
|
||||
@@ -88,6 +98,7 @@ func (r *ChunkRepository) GetByHashes(ctx context.Context, hashes []string) ([]*
|
||||
defer CloseRows(rows)
|
||||
|
||||
var chunks []*Chunk
|
||||
|
||||
for rows.Next() {
|
||||
var chunk Chunk
|
||||
|
||||
@@ -122,6 +133,7 @@ func (r *ChunkRepository) ListUnpacked(ctx context.Context, limit int) ([]*Chunk
|
||||
defer CloseRows(rows)
|
||||
|
||||
var chunks []*Chunk
|
||||
|
||||
for rows.Next() {
|
||||
var chunk Chunk
|
||||
|
||||
|
||||
@@ -19,6 +19,7 @@ func (r *ChunkRepository) List(ctx context.Context) ([]*Chunk, error) {
|
||||
defer CloseRows(rows)
|
||||
|
||||
var chunks []*Chunk
|
||||
|
||||
for rows.Next() {
|
||||
var chunk Chunk
|
||||
|
||||
|
||||
@@ -30,12 +30,15 @@ func TestChunkRepository(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get chunk: %v", err)
|
||||
}
|
||||
|
||||
if retrieved == nil {
|
||||
t.Fatal("expected chunk, got nil")
|
||||
}
|
||||
|
||||
if retrieved.ChunkHash != chunk.ChunkHash {
|
||||
t.Errorf("chunk hash mismatch: got %s, want %s", retrieved.ChunkHash, chunk.ChunkHash)
|
||||
}
|
||||
|
||||
if retrieved.Size != chunk.Size {
|
||||
t.Errorf("size mismatch: got %d, want %d", retrieved.Size, chunk.Size)
|
||||
}
|
||||
@@ -51,6 +54,7 @@ func TestChunkRepository(t *testing.T) {
|
||||
ChunkHash: types.ChunkHash("chunkhash456"),
|
||||
Size: 8192,
|
||||
}
|
||||
|
||||
err = repo.Create(ctx, nil, chunk2)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create second chunk: %v", err)
|
||||
@@ -60,6 +64,7 @@ func TestChunkRepository(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get chunks by hashes: %v", err)
|
||||
}
|
||||
|
||||
if len(chunks) != 2 {
|
||||
t.Errorf("expected 2 chunks, got %d", len(chunks))
|
||||
}
|
||||
@@ -69,6 +74,7 @@ func TestChunkRepository(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to list unpacked chunks: %v", err)
|
||||
}
|
||||
|
||||
if len(unpacked) != 2 {
|
||||
t.Errorf("expected 2 unpacked chunks, got %d", len(unpacked))
|
||||
}
|
||||
@@ -86,6 +92,7 @@ func TestChunkRepositoryNotFound(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if chunk != nil {
|
||||
t.Error("expected nil for non-existent chunk")
|
||||
}
|
||||
@@ -95,6 +102,7 @@ func TestChunkRepositoryNotFound(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if chunks != nil {
|
||||
t.Error("expected nil for empty hash list")
|
||||
}
|
||||
|
||||
@@ -57,8 +57,8 @@ func ParseMigrationVersion(filename string) (int, error) {
|
||||
// Split on underscore to separate version from description.
|
||||
// If there's no underscore, the entire stem is the version.
|
||||
versionStr := name
|
||||
if idx := strings.IndexByte(name, '_'); idx >= 0 {
|
||||
versionStr = name[:idx]
|
||||
if before, _, ok := strings.Cut(name, "_"); ok {
|
||||
versionStr = before
|
||||
}
|
||||
|
||||
if versionStr == "" {
|
||||
@@ -98,6 +98,7 @@ func New(ctx context.Context, path string) (*DB, error) {
|
||||
|
||||
// First attempt with standard WAL mode
|
||||
log.Debug("Attempting to open database with WAL mode", "path", path)
|
||||
|
||||
conn, err := sql.Open(
|
||||
"sqlite",
|
||||
path+"?_journal_mode=WAL&_synchronous=NORMAL&_busy_timeout=10000&_locking_mode=NORMAL&_foreign_keys=ON",
|
||||
@@ -110,7 +111,8 @@ func New(ctx context.Context, path string) (*DB, error) {
|
||||
conn.SetMaxOpenConns(1)
|
||||
conn.SetMaxIdleConns(1)
|
||||
|
||||
if err := conn.PingContext(ctx); err == nil {
|
||||
err := conn.PingContext(ctx)
|
||||
if err == nil {
|
||||
// Success on first try
|
||||
log.Debug("Database opened successfully with WAL mode", "path", path)
|
||||
|
||||
@@ -120,13 +122,19 @@ func New(ctx context.Context, path string) (*DB, error) {
|
||||
}
|
||||
|
||||
db := &DB{conn: conn, path: path}
|
||||
if err := applyMigrations(ctx, conn); err != nil {
|
||||
|
||||
err := applyMigrations(ctx, conn)
|
||||
if err != nil {
|
||||
_ = conn.Close()
|
||||
|
||||
return nil, fmt.Errorf("applying migrations: %w", err)
|
||||
}
|
||||
|
||||
return db, nil
|
||||
}
|
||||
|
||||
log.Debug("Failed to ping database, closing connection", "path", path, "error", err)
|
||||
|
||||
_ = conn.Close()
|
||||
}
|
||||
|
||||
@@ -135,6 +143,7 @@ func New(ctx context.Context, path string) (*DB, error) {
|
||||
"Database appears locked, attempting recovery with TRUNCATE mode",
|
||||
"path", path,
|
||||
)
|
||||
|
||||
conn, err = sql.Open(
|
||||
"sqlite",
|
||||
path+"?_journal_mode=TRUNCATE&_synchronous=NORMAL&_busy_timeout=10000&_foreign_keys=ON",
|
||||
@@ -152,7 +161,9 @@ func New(ctx context.Context, path string) (*DB, error) {
|
||||
|
||||
if err := conn.PingContext(ctx); err != nil {
|
||||
log.Debug("Failed to ping database in recovery mode, closing", "path", path, "error", err)
|
||||
|
||||
_ = conn.Close()
|
||||
|
||||
return nil, fmt.Errorf(
|
||||
"database still locked after recovery attempt: %w",
|
||||
err,
|
||||
@@ -163,6 +174,7 @@ func New(ctx context.Context, path string) (*DB, error) {
|
||||
|
||||
// Switch back to WAL mode
|
||||
log.Debug("Switching database back to WAL mode", "path", path)
|
||||
|
||||
if _, err := conn.ExecContext(ctx, "PRAGMA journal_mode=WAL"); err != nil {
|
||||
log.Warn("Failed to switch back to WAL mode", "path", path, "error", err)
|
||||
}
|
||||
@@ -175,10 +187,12 @@ func New(ctx context.Context, path string) (*DB, error) {
|
||||
db := &DB{conn: conn, path: path}
|
||||
if err := applyMigrations(ctx, conn); err != nil {
|
||||
_ = conn.Close()
|
||||
|
||||
return nil, fmt.Errorf("applying migrations: %w", err)
|
||||
}
|
||||
|
||||
log.Debug("Database connection established successfully", "path", path)
|
||||
|
||||
return db, nil
|
||||
}
|
||||
|
||||
@@ -187,11 +201,16 @@ func New(ctx context.Context, path string) (*DB, error) {
|
||||
// Returns an error if the database connection cannot be closed properly.
|
||||
func (db *DB) Close() error {
|
||||
log.Debug("Closing database connection", "path", db.path)
|
||||
if err := db.conn.Close(); err != nil {
|
||||
|
||||
err := db.conn.Close()
|
||||
if err != nil {
|
||||
log.Error("Failed to close database", "path", db.path, "error", err)
|
||||
|
||||
return fmt.Errorf("failed to close database: %w", err)
|
||||
}
|
||||
|
||||
log.Debug("Database connection closed successfully", "path", db.path)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -227,9 +246,10 @@ func (db *DB) BeginTx(
|
||||
func (db *DB) ExecWithLog(
|
||||
ctx context.Context,
|
||||
query string,
|
||||
args ...interface{},
|
||||
args ...any,
|
||||
) (sql.Result, error) {
|
||||
LogSQL("Execute", query, args...)
|
||||
|
||||
return db.conn.ExecContext(ctx, query, args...)
|
||||
}
|
||||
|
||||
@@ -240,9 +260,10 @@ func (db *DB) ExecWithLog(
|
||||
func (db *DB) QueryRowWithLog(
|
||||
ctx context.Context,
|
||||
query string,
|
||||
args ...interface{},
|
||||
args ...any,
|
||||
) *sql.Row {
|
||||
LogSQL("QueryRow", query, args...)
|
||||
|
||||
return db.conn.QueryRowContext(ctx, query, args...)
|
||||
}
|
||||
|
||||
@@ -375,6 +396,7 @@ func repeatPlaceholder(n int) string {
|
||||
if n <= 0 {
|
||||
return ""
|
||||
}
|
||||
|
||||
return strings.Repeat(", ?", n)
|
||||
}
|
||||
|
||||
@@ -385,7 +407,7 @@ func repeatPlaceholder(n int) string {
|
||||
// The operation parameter describes the type of SQL operation (e.g., "Execute", "Query").
|
||||
// The query parameter is the SQL statement being executed.
|
||||
// The args parameter contains the query arguments that will be interpolated.
|
||||
func LogSQL(operation, query string, args ...interface{}) {
|
||||
func LogSQL(operation, query string, args ...any) {
|
||||
if strings.Contains(os.Getenv("GODEBUG"), "vaultik") {
|
||||
log.Debug(
|
||||
"SQL "+operation,
|
||||
|
||||
@@ -17,7 +17,8 @@ func TestDatabase(t *testing.T) {
|
||||
t.Fatalf("failed to create database: %v", err)
|
||||
}
|
||||
defer func() {
|
||||
if err := db.Close(); err != nil {
|
||||
err := db.Close()
|
||||
if err != nil {
|
||||
t.Errorf("failed to close database: %v", err)
|
||||
}
|
||||
}()
|
||||
@@ -37,6 +38,7 @@ func TestDatabase(t *testing.T) {
|
||||
|
||||
for _, table := range tables {
|
||||
var name string
|
||||
|
||||
err := db.conn.QueryRow("SELECT name FROM sqlite_master WHERE type='table' AND name=?", table).Scan(&name)
|
||||
if err != nil {
|
||||
t.Errorf("table %s does not exist: %v", table, err)
|
||||
@@ -63,7 +65,8 @@ func TestDatabaseConcurrentAccess(t *testing.T) {
|
||||
t.Fatalf("failed to create database: %v", err)
|
||||
}
|
||||
defer func() {
|
||||
if err := db.Close(); err != nil {
|
||||
err := db.Close()
|
||||
if err != nil {
|
||||
t.Errorf("failed to close database: %v", err)
|
||||
}
|
||||
}()
|
||||
@@ -73,9 +76,10 @@ func TestDatabaseConcurrentAccess(t *testing.T) {
|
||||
index int
|
||||
err error
|
||||
}
|
||||
|
||||
results := make(chan result, 10)
|
||||
|
||||
for i := 0; i < 10; i++ {
|
||||
for i := range 10 {
|
||||
go func(i int) {
|
||||
_, err := db.ExecWithLog(ctx, "INSERT INTO chunks (chunk_hash, size) VALUES (?, ?)",
|
||||
fmt.Sprintf("hash%d", i), i*1024)
|
||||
@@ -84,7 +88,7 @@ func TestDatabaseConcurrentAccess(t *testing.T) {
|
||||
}
|
||||
|
||||
// Wait for all goroutines and check results
|
||||
for i := 0; i < 10; i++ {
|
||||
for range 10 {
|
||||
r := <-results
|
||||
if r.err != nil {
|
||||
t.Fatalf("concurrent insert %d failed: %v", r.index, r.err)
|
||||
@@ -93,10 +97,12 @@ func TestDatabaseConcurrentAccess(t *testing.T) {
|
||||
|
||||
// Verify all inserts succeeded
|
||||
var count int
|
||||
|
||||
err = db.conn.QueryRowContext(ctx, "SELECT COUNT(*) FROM chunks").Scan(&count)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to count chunks: %v", err)
|
||||
}
|
||||
|
||||
if count != 10 {
|
||||
t.Errorf("expected 10 chunks, got %d", count)
|
||||
}
|
||||
@@ -127,12 +133,16 @@ func TestParseMigrationVersion(t *testing.T) {
|
||||
if err == nil {
|
||||
t.Errorf("ParseMigrationVersion(%q) = %d, nil; want error", tc.filename, got)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
t.Errorf("ParseMigrationVersion(%q) unexpected error: %v", tc.filename, err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if got != tc.wantVer {
|
||||
t.Errorf("ParseMigrationVersion(%q) = %d; want %d", tc.filename, got, tc.wantVer)
|
||||
}
|
||||
@@ -148,7 +158,8 @@ func TestApplyMigrations_Idempotent(t *testing.T) {
|
||||
t.Fatalf("failed to open database: %v", err)
|
||||
}
|
||||
defer func() {
|
||||
if err := conn.Close(); err != nil {
|
||||
err := conn.Close()
|
||||
if err != nil {
|
||||
t.Errorf("failed to close database: %v", err)
|
||||
}
|
||||
}()
|
||||
@@ -191,7 +202,8 @@ func TestBootstrapMigrationsTable_FreshDatabase(t *testing.T) {
|
||||
t.Fatalf("failed to open database: %v", err)
|
||||
}
|
||||
defer func() {
|
||||
if err := conn.Close(); err != nil {
|
||||
err := conn.Close()
|
||||
if err != nil {
|
||||
t.Errorf("failed to close database: %v", err)
|
||||
}
|
||||
}()
|
||||
@@ -206,6 +218,7 @@ func TestBootstrapMigrationsTable_FreshDatabase(t *testing.T) {
|
||||
).Scan(&tableBefore); err != nil {
|
||||
t.Fatalf("failed to check for table before bootstrap: %v", err)
|
||||
}
|
||||
|
||||
if tableBefore != 0 {
|
||||
t.Fatal("schema_migrations table should not exist before bootstrap")
|
||||
}
|
||||
@@ -222,6 +235,7 @@ func TestBootstrapMigrationsTable_FreshDatabase(t *testing.T) {
|
||||
).Scan(&tableAfter); err != nil {
|
||||
t.Fatalf("failed to check for table after bootstrap: %v", err)
|
||||
}
|
||||
|
||||
if tableAfter != 1 {
|
||||
t.Fatalf("schema_migrations table should exist after bootstrap, got count=%d", tableAfter)
|
||||
}
|
||||
@@ -233,6 +247,7 @@ func TestBootstrapMigrationsTable_FreshDatabase(t *testing.T) {
|
||||
).Scan(&version); err != nil {
|
||||
t.Fatalf("version 0 row not found in schema_migrations: %v", err)
|
||||
}
|
||||
|
||||
if version != 0 {
|
||||
t.Errorf("expected version 0, got %d", version)
|
||||
}
|
||||
|
||||
@@ -6,15 +6,16 @@ import (
|
||||
"os"
|
||||
)
|
||||
|
||||
// Fatal prints an error message to stderr and exits with status 1
|
||||
func Fatal(format string, args ...interface{}) {
|
||||
// Fatalf prints an error message to stderr and exits with status 1
|
||||
func Fatalf(format string, args ...any) {
|
||||
fmt.Fprintf(os.Stderr, "FATAL: "+format+"\n", args...)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// CloseRows closes rows and exits on error
|
||||
func CloseRows(rows *sql.Rows) {
|
||||
if err := rows.Close(); err != nil {
|
||||
Fatal("failed to close rows: %v", err)
|
||||
err := rows.Close()
|
||||
if err != nil {
|
||||
Fatalf("failed to close rows: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"sneak.berlin/go/vaultik/internal/types"
|
||||
)
|
||||
@@ -84,6 +85,7 @@ func (r *FileChunkRepository) GetByPathTx(ctx context.Context, tx *sql.Tx, path
|
||||
`
|
||||
|
||||
LogSQL("GetByPathTx", query, path)
|
||||
|
||||
rows, err := tx.QueryContext(ctx, query, path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("querying file chunks: %w", err)
|
||||
@@ -92,23 +94,30 @@ func (r *FileChunkRepository) GetByPathTx(ctx context.Context, tx *sql.Tx, path
|
||||
|
||||
fileChunks, err := r.scanFileChunks(rows)
|
||||
LogSQL("GetByPathTx", "Complete", path, "count", len(fileChunks))
|
||||
|
||||
return fileChunks, err
|
||||
}
|
||||
|
||||
// scanFileChunks is a helper that scans file chunk rows
|
||||
func (r *FileChunkRepository) scanFileChunks(rows *sql.Rows) ([]*FileChunk, error) {
|
||||
var fileChunks []*FileChunk
|
||||
|
||||
for rows.Next() {
|
||||
var fc FileChunk
|
||||
var fileIDStr, chunkHashStr string
|
||||
var (
|
||||
fc FileChunk
|
||||
fileIDStr, chunkHashStr string
|
||||
)
|
||||
|
||||
err := rows.Scan(&fileIDStr, &fc.Idx, &chunkHashStr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scanning file chunk: %w", err)
|
||||
}
|
||||
|
||||
fc.FileID, err = types.ParseFileID(fileIDStr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parsing file ID: %w", err)
|
||||
}
|
||||
|
||||
fc.ChunkHash = types.ChunkHash(chunkHashStr)
|
||||
fileChunks = append(fileChunks, &fc)
|
||||
}
|
||||
@@ -161,14 +170,13 @@ func (r *FileChunkRepository) DeleteByFileIDs(ctx context.Context, tx *sql.Tx, f
|
||||
const batchSize = 500
|
||||
|
||||
for i := 0; i < len(fileIDs); i += batchSize {
|
||||
end := i + batchSize
|
||||
if end > len(fileIDs) {
|
||||
end = len(fileIDs)
|
||||
}
|
||||
end := min(i+batchSize, len(fileIDs))
|
||||
|
||||
batch := fileIDs[i:end]
|
||||
|
||||
query := "DELETE FROM file_chunks WHERE file_id IN (?" + repeatPlaceholder(len(batch)-1) + ")"
|
||||
args := make([]interface{}, len(batch))
|
||||
|
||||
args := make([]any, len(batch))
|
||||
for j, id := range batch {
|
||||
args[j] = id.String()
|
||||
}
|
||||
@@ -179,6 +187,7 @@ func (r *FileChunkRepository) DeleteByFileIDs(ctx context.Context, tx *sql.Tx, f
|
||||
} else {
|
||||
_, err = r.db.ExecWithLog(ctx, query, args...)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("batch deleting file_chunks: %w", err)
|
||||
}
|
||||
@@ -199,22 +208,29 @@ func (r *FileChunkRepository) CreateBatch(ctx context.Context, tx *sql.Tx, fcs [
|
||||
const batchSize = 300
|
||||
|
||||
for i := 0; i < len(fcs); i += batchSize {
|
||||
end := i + batchSize
|
||||
if end > len(fcs) {
|
||||
end = len(fcs)
|
||||
}
|
||||
end := min(i+batchSize, len(fcs))
|
||||
|
||||
batch := fcs[i:end]
|
||||
|
||||
// Build the query with multiple value sets
|
||||
query := "INSERT INTO file_chunks (file_id, idx, chunk_hash) VALUES "
|
||||
args := make([]interface{}, 0, len(batch)*3)
|
||||
|
||||
args := make([]any, 0, len(batch)*3)
|
||||
|
||||
var querySb211 strings.Builder
|
||||
|
||||
for j, fc := range batch {
|
||||
if j > 0 {
|
||||
query += ", "
|
||||
querySb211.WriteString(", ")
|
||||
}
|
||||
query += "(?, ?, ?)"
|
||||
|
||||
querySb211.WriteString("(?, ?, ?)")
|
||||
|
||||
args = append(args, fc.FileID.String(), fc.Idx, fc.ChunkHash.String())
|
||||
}
|
||||
|
||||
query += querySb211.String()
|
||||
|
||||
query += " ON CONFLICT(file_id, idx) DO NOTHING"
|
||||
|
||||
var err error
|
||||
@@ -223,6 +239,7 @@ func (r *FileChunkRepository) CreateBatch(ctx context.Context, tx *sql.Tx, fcs [
|
||||
} else {
|
||||
_, err = r.db.ExecWithLog(ctx, query, args...)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("batch inserting file_chunks: %w", err)
|
||||
}
|
||||
@@ -236,6 +253,7 @@ func (r *FileChunkRepository) GetByFile(ctx context.Context, path string) ([]*Fi
|
||||
LogSQL("GetByFile", "Starting", path)
|
||||
result, err := r.GetByPath(ctx, path)
|
||||
LogSQL("GetByFile", "Complete", path, "count", len(result))
|
||||
|
||||
return result, err
|
||||
}
|
||||
|
||||
@@ -244,5 +262,6 @@ func (r *FileChunkRepository) GetByFileTx(ctx context.Context, tx *sql.Tx, path
|
||||
LogSQL("GetByFileTx", "Starting", path)
|
||||
result, err := r.GetByPathTx(ctx, tx, path)
|
||||
LogSQL("GetByFileTx", "Complete", path, "count", len(result))
|
||||
|
||||
return result, err
|
||||
}
|
||||
|
||||
@@ -28,6 +28,7 @@ func TestFileChunkRepository(t *testing.T) {
|
||||
GID: 1000,
|
||||
LinkTarget: "",
|
||||
}
|
||||
|
||||
err := fileRepo.Create(ctx, nil, file)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create file: %v", err)
|
||||
@@ -36,11 +37,13 @@ func TestFileChunkRepository(t *testing.T) {
|
||||
// Create chunks first
|
||||
chunks := []types.ChunkHash{"chunk1", "chunk2", "chunk3"}
|
||||
chunkRepo := NewChunkRepository(db)
|
||||
|
||||
for _, chunkHash := range chunks {
|
||||
chunk := &Chunk{
|
||||
ChunkHash: chunkHash,
|
||||
Size: 1024,
|
||||
}
|
||||
|
||||
err = chunkRepo.Create(ctx, nil, chunk)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create chunk %s: %v", chunkHash, err)
|
||||
@@ -65,6 +68,7 @@ func TestFileChunkRepository(t *testing.T) {
|
||||
Idx: 1,
|
||||
ChunkHash: types.ChunkHash("chunk2"),
|
||||
}
|
||||
|
||||
err = repo.Create(ctx, nil, fc2)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create second file chunk: %v", err)
|
||||
@@ -75,6 +79,7 @@ func TestFileChunkRepository(t *testing.T) {
|
||||
Idx: 2,
|
||||
ChunkHash: types.ChunkHash("chunk3"),
|
||||
}
|
||||
|
||||
err = repo.Create(ctx, nil, fc3)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create third file chunk: %v", err)
|
||||
@@ -85,6 +90,7 @@ func TestFileChunkRepository(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get file chunks: %v", err)
|
||||
}
|
||||
|
||||
if len(fileChunks) != 3 {
|
||||
t.Errorf("expected 3 chunks, got %d", len(fileChunks))
|
||||
}
|
||||
@@ -112,6 +118,7 @@ func TestFileChunkRepository(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get deleted file chunks: %v", err)
|
||||
}
|
||||
|
||||
if len(fileChunks) != 0 {
|
||||
t.Errorf("expected 0 chunks after delete, got %d", len(fileChunks))
|
||||
}
|
||||
@@ -140,22 +147,26 @@ func TestFileChunkRepositoryMultipleFiles(t *testing.T) {
|
||||
GID: 1000,
|
||||
LinkTarget: "",
|
||||
}
|
||||
|
||||
err := fileRepo.Create(ctx, nil, file)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create file %s: %v", path, err)
|
||||
}
|
||||
|
||||
files[i] = file
|
||||
}
|
||||
|
||||
// Create all chunks first
|
||||
chunkRepo := NewChunkRepository(db)
|
||||
|
||||
for i := range files {
|
||||
for j := 0; j < 2; j++ {
|
||||
for j := range 2 {
|
||||
chunkHash := types.ChunkHash(fmt.Sprintf("file%d_chunk%d", i, j))
|
||||
chunk := &Chunk{
|
||||
ChunkHash: chunkHash,
|
||||
Size: 1024,
|
||||
}
|
||||
|
||||
err := chunkRepo.Create(ctx, nil, chunk)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create chunk %s: %v", chunkHash, err)
|
||||
@@ -165,12 +176,13 @@ func TestFileChunkRepositoryMultipleFiles(t *testing.T) {
|
||||
|
||||
// Create chunks for multiple files
|
||||
for i, file := range files {
|
||||
for j := 0; j < 2; j++ {
|
||||
for j := range 2 {
|
||||
fc := &FileChunk{
|
||||
FileID: file.ID,
|
||||
Idx: j,
|
||||
ChunkHash: types.ChunkHash(fmt.Sprintf("file%d_chunk%d", i, j)),
|
||||
}
|
||||
|
||||
err := repo.Create(ctx, nil, fc)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create file chunk: %v", err)
|
||||
@@ -184,6 +196,7 @@ func TestFileChunkRepositoryMultipleFiles(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get chunks for file %d: %v", i, err)
|
||||
}
|
||||
|
||||
if len(chunks) != 2 {
|
||||
t.Errorf("expected 2 chunks for file %d, got %d", i, len(chunks))
|
||||
}
|
||||
|
||||
@@ -3,7 +3,9 @@ package database
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
@@ -38,8 +40,11 @@ func (r *FileRepository) Create(ctx context.Context, tx *sql.Tx, file *File) err
|
||||
RETURNING id
|
||||
`
|
||||
|
||||
var idStr string
|
||||
var err error
|
||||
var (
|
||||
idStr string
|
||||
err error
|
||||
)
|
||||
|
||||
if tx != nil {
|
||||
LogSQL("Execute", query, file.ID.String(), file.Path.String(), file.SourcePath.String(), file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID, file.LinkTarget.String())
|
||||
err = tx.QueryRowContext(ctx, query, file.ID.String(), file.Path.String(), file.SourcePath.String(), file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID, file.LinkTarget.String()).Scan(&idStr)
|
||||
@@ -68,9 +73,10 @@ func (r *FileRepository) GetByPath(ctx context.Context, path string) (*File, err
|
||||
`
|
||||
|
||||
file, err := r.scanFile(r.db.conn.QueryRowContext(ctx, query, path))
|
||||
if err == sql.ErrNoRows {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("querying file: %w", err)
|
||||
}
|
||||
@@ -87,9 +93,10 @@ func (r *FileRepository) GetByID(ctx context.Context, id types.FileID) (*File, e
|
||||
`
|
||||
|
||||
file, err := r.scanFile(r.db.conn.QueryRowContext(ctx, query, id.String()))
|
||||
if err == sql.ErrNoRows {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("querying file: %w", err)
|
||||
}
|
||||
@@ -108,9 +115,10 @@ func (r *FileRepository) GetByPathTx(ctx context.Context, tx *sql.Tx, path strin
|
||||
file, err := r.scanFile(tx.QueryRowContext(ctx, query, path))
|
||||
LogSQL("GetByPathTx Scan complete", query, path)
|
||||
|
||||
if err == sql.ErrNoRows {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("querying file: %w", err)
|
||||
}
|
||||
@@ -120,10 +128,12 @@ func (r *FileRepository) GetByPathTx(ctx context.Context, tx *sql.Tx, path strin
|
||||
|
||||
// scanFile is a helper that scans a single file row
|
||||
func (r *FileRepository) scanFile(row *sql.Row) (*File, error) {
|
||||
var file File
|
||||
var idStr, pathStr, sourcePathStr string
|
||||
var mtimeUnix int64
|
||||
var linkTarget sql.NullString
|
||||
var (
|
||||
file File
|
||||
idStr, pathStr, sourcePathStr string
|
||||
mtimeUnix int64
|
||||
linkTarget sql.NullString
|
||||
)
|
||||
|
||||
err := row.Scan(
|
||||
&idStr,
|
||||
@@ -144,8 +154,10 @@ func (r *FileRepository) scanFile(row *sql.Row) (*File, error) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parsing file ID: %w", err)
|
||||
}
|
||||
|
||||
file.Path = types.FilePath(pathStr)
|
||||
file.SourcePath = types.SourcePath(sourcePathStr)
|
||||
|
||||
file.MTime = time.Unix(mtimeUnix, 0).UTC()
|
||||
if linkTarget.Valid {
|
||||
file.LinkTarget = types.FilePath(linkTarget.String)
|
||||
@@ -156,10 +168,12 @@ func (r *FileRepository) scanFile(row *sql.Row) (*File, error) {
|
||||
|
||||
// scanFileRows is a helper that scans a file row from rows iterator
|
||||
func (r *FileRepository) scanFileRows(rows *sql.Rows) (*File, error) {
|
||||
var file File
|
||||
var idStr, pathStr, sourcePathStr string
|
||||
var mtimeUnix int64
|
||||
var linkTarget sql.NullString
|
||||
var (
|
||||
file File
|
||||
idStr, pathStr, sourcePathStr string
|
||||
mtimeUnix int64
|
||||
linkTarget sql.NullString
|
||||
)
|
||||
|
||||
err := rows.Scan(
|
||||
&idStr,
|
||||
@@ -180,8 +194,10 @@ func (r *FileRepository) scanFileRows(rows *sql.Rows) (*File, error) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parsing file ID: %w", err)
|
||||
}
|
||||
|
||||
file.Path = types.FilePath(pathStr)
|
||||
file.SourcePath = types.SourcePath(sourcePathStr)
|
||||
|
||||
file.MTime = time.Unix(mtimeUnix, 0).UTC()
|
||||
if linkTarget.Valid {
|
||||
file.LinkTarget = types.FilePath(linkTarget.String)
|
||||
@@ -205,11 +221,13 @@ func (r *FileRepository) ListModifiedSince(ctx context.Context, since time.Time)
|
||||
defer CloseRows(rows)
|
||||
|
||||
var files []*File
|
||||
|
||||
for rows.Next() {
|
||||
file, err := r.scanFileRows(rows)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scanning file: %w", err)
|
||||
}
|
||||
|
||||
files = append(files, file)
|
||||
}
|
||||
|
||||
@@ -266,11 +284,13 @@ func (r *FileRepository) ListByPrefix(ctx context.Context, prefix string) ([]*Fi
|
||||
defer CloseRows(rows)
|
||||
|
||||
var files []*File
|
||||
|
||||
for rows.Next() {
|
||||
file, err := r.scanFileRows(rows)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scanning file: %w", err)
|
||||
}
|
||||
|
||||
files = append(files, file)
|
||||
}
|
||||
|
||||
@@ -292,11 +312,13 @@ func (r *FileRepository) ListAll(ctx context.Context) ([]*File, error) {
|
||||
defer CloseRows(rows)
|
||||
|
||||
var files []*File
|
||||
|
||||
for rows.Next() {
|
||||
file, err := r.scanFileRows(rows)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scanning file: %w", err)
|
||||
}
|
||||
|
||||
files = append(files, file)
|
||||
}
|
||||
|
||||
@@ -314,21 +336,28 @@ func (r *FileRepository) CreateBatch(ctx context.Context, tx *sql.Tx, files []*F
|
||||
const batchSize = 100
|
||||
|
||||
for i := 0; i < len(files); i += batchSize {
|
||||
end := i + batchSize
|
||||
if end > len(files) {
|
||||
end = len(files)
|
||||
}
|
||||
end := min(i+batchSize, len(files))
|
||||
|
||||
batch := files[i:end]
|
||||
|
||||
query := `INSERT INTO files (id, path, source_path, mtime, size, mode, uid, gid, link_target) VALUES `
|
||||
args := make([]interface{}, 0, len(batch)*9)
|
||||
|
||||
args := make([]any, 0, len(batch)*9)
|
||||
|
||||
var querySb325 strings.Builder
|
||||
|
||||
for j, f := range batch {
|
||||
if j > 0 {
|
||||
query += ", "
|
||||
querySb325.WriteString(", ")
|
||||
}
|
||||
query += "(?, ?, ?, ?, ?, ?, ?, ?, ?)"
|
||||
|
||||
querySb325.WriteString("(?, ?, ?, ?, ?, ?, ?, ?, ?)")
|
||||
|
||||
args = append(args, f.ID.String(), f.Path.String(), f.SourcePath.String(), f.MTime.Unix(), f.Size, f.Mode, f.UID, f.GID, f.LinkTarget.String())
|
||||
}
|
||||
|
||||
query += querySb325.String()
|
||||
|
||||
query += ` ON CONFLICT(path) DO UPDATE SET
|
||||
source_path = excluded.source_path,
|
||||
mtime = excluded.mtime,
|
||||
@@ -344,6 +373,7 @@ func (r *FileRepository) CreateBatch(ctx context.Context, tx *sql.Tx, files []*F
|
||||
} else {
|
||||
_, err = r.db.ExecWithLog(ctx, query, args...)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("batch inserting files: %w", err)
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@ package database
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
@@ -20,7 +20,8 @@ func setupTestDB(t *testing.T) (*DB, func()) {
|
||||
}
|
||||
|
||||
cleanup := func() {
|
||||
if err := db.Close(); err != nil {
|
||||
err := db.Close()
|
||||
if err != nil {
|
||||
t.Errorf("failed to close database: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -56,18 +57,23 @@ func TestFileRepository(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get file: %v", err)
|
||||
}
|
||||
|
||||
if retrieved == nil {
|
||||
t.Fatal("expected file, got nil")
|
||||
}
|
||||
|
||||
if retrieved.Path != file.Path {
|
||||
t.Errorf("path mismatch: got %s, want %s", retrieved.Path, file.Path)
|
||||
}
|
||||
|
||||
if !retrieved.MTime.Equal(file.MTime) {
|
||||
t.Errorf("mtime mismatch: got %v, want %v", retrieved.MTime, file.MTime)
|
||||
}
|
||||
|
||||
if retrieved.Size != file.Size {
|
||||
t.Errorf("size mismatch: got %d, want %d", retrieved.Size, file.Size)
|
||||
}
|
||||
|
||||
if retrieved.Mode != file.Mode {
|
||||
t.Errorf("mode mismatch: got %o, want %o", retrieved.Mode, file.Mode)
|
||||
}
|
||||
@@ -75,6 +81,7 @@ func TestFileRepository(t *testing.T) {
|
||||
// Test Update (upsert)
|
||||
file.Size = 2048
|
||||
file.MTime = time.Now().Truncate(time.Second)
|
||||
|
||||
err = repo.Create(ctx, nil, file)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to update file: %v", err)
|
||||
@@ -84,6 +91,7 @@ func TestFileRepository(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get updated file: %v", err)
|
||||
}
|
||||
|
||||
if retrieved.Size != 2048 {
|
||||
t.Errorf("size not updated: got %d, want %d", retrieved.Size, 2048)
|
||||
}
|
||||
@@ -93,6 +101,7 @@ func TestFileRepository(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to list files: %v", err)
|
||||
}
|
||||
|
||||
if len(files) != 1 {
|
||||
t.Errorf("expected 1 file, got %d", len(files))
|
||||
}
|
||||
@@ -107,6 +116,7 @@ func TestFileRepository(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("error getting deleted file: %v", err)
|
||||
}
|
||||
|
||||
if retrieved != nil {
|
||||
t.Error("expected nil for deleted file")
|
||||
}
|
||||
@@ -139,9 +149,11 @@ func TestFileRepositorySymlink(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get symlink: %v", err)
|
||||
}
|
||||
|
||||
if !retrieved.IsSymlink() {
|
||||
t.Error("expected IsSymlink() to be true")
|
||||
}
|
||||
|
||||
if retrieved.LinkTarget != symlink.LinkTarget {
|
||||
t.Errorf("link target mismatch: got %s, want %s", retrieved.LinkTarget, symlink.LinkTarget)
|
||||
}
|
||||
@@ -165,12 +177,13 @@ func TestFileRepositoryTransaction(t *testing.T) {
|
||||
GID: 1000,
|
||||
}
|
||||
|
||||
if err := repos.Files.Create(ctx, tx, file); err != nil {
|
||||
err := repos.Files.Create(ctx, tx, file)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Return error to trigger rollback
|
||||
return fmt.Errorf("test rollback")
|
||||
return errors.New("test rollback")
|
||||
})
|
||||
|
||||
if err == nil || err.Error() != "test rollback" {
|
||||
@@ -182,6 +195,7 @@ func TestFileRepositoryTransaction(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("error checking for file: %v", err)
|
||||
}
|
||||
|
||||
if retrieved != nil {
|
||||
t.Error("file should not exist after rollback")
|
||||
}
|
||||
|
||||
@@ -27,15 +27,18 @@ func NewLocalMetaRepository(db *DB) *LocalMetaRepository {
|
||||
// "unset" (bind on first use) from "set to something" (compare).
|
||||
func (r *LocalMetaRepository) Get(ctx context.Context, key string) (string, error) {
|
||||
var value string
|
||||
|
||||
err := r.db.conn.QueryRowContext(ctx,
|
||||
"SELECT value FROM local_meta WHERE key = ?", key,
|
||||
).Scan(&value)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("reading local_meta %q: %w", key, err)
|
||||
}
|
||||
|
||||
return value, nil
|
||||
}
|
||||
|
||||
@@ -49,5 +52,6 @@ func (r *LocalMetaRepository) Set(ctx context.Context, key, value string) error
|
||||
if err != nil {
|
||||
return fmt.Errorf("writing local_meta %q: %w", key, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -11,18 +11,20 @@ import (
|
||||
func TestLocalMetaEmptyOnFresh(t *testing.T) {
|
||||
db, err := database.NewTestDB()
|
||||
require.NoError(t, err)
|
||||
|
||||
defer func() { _ = db.Close() }()
|
||||
|
||||
repos := database.NewRepositories(db)
|
||||
|
||||
got, err := repos.LocalMeta.Get(context.Background(), database.LocalMetaKeyStorageURL)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "", got, "fresh DB must return empty for unset keys, not error")
|
||||
require.Empty(t, got, "fresh DB must return empty for unset keys, not error")
|
||||
}
|
||||
|
||||
func TestLocalMetaSetGetRoundTrip(t *testing.T) {
|
||||
db, err := database.NewTestDB()
|
||||
require.NoError(t, err)
|
||||
|
||||
defer func() { _ = db.Close() }()
|
||||
|
||||
repos := database.NewRepositories(db)
|
||||
@@ -38,6 +40,7 @@ func TestLocalMetaSetGetRoundTrip(t *testing.T) {
|
||||
func TestLocalMetaSetOverwrites(t *testing.T) {
|
||||
db, err := database.NewTestDB()
|
||||
require.NoError(t, err)
|
||||
|
||||
defer func() { _ = db.Close() }()
|
||||
|
||||
repos := database.NewRepositories(db)
|
||||
|
||||
@@ -34,11 +34,16 @@ func provideDatabase(lc fx.Lifecycle, cfg *config.Config) (*DB, error) {
|
||||
lc.Append(fx.Hook{
|
||||
OnStop: func(ctx context.Context) error {
|
||||
log.Debug("Database module OnStop hook called")
|
||||
if err := db.Close(); err != nil {
|
||||
|
||||
err := db.Close()
|
||||
if err != nil {
|
||||
log.Error("Failed to close database in OnStop hook", "error", err)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
log.Debug("Database closed successfully in OnStop hook")
|
||||
|
||||
return nil
|
||||
},
|
||||
})
|
||||
|
||||
@@ -50,21 +50,26 @@ type TxFunc func(ctx context.Context, tx *sql.Tx) error
|
||||
// This method should be used for all write operations to ensure atomicity.
|
||||
func (r *Repositories) WithTx(ctx context.Context, fn TxFunc) error {
|
||||
LogSQL("WithTx", "Beginning transaction", "")
|
||||
|
||||
tx, err := r.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("beginning transaction: %w", err)
|
||||
}
|
||||
|
||||
LogSQL("WithTx", "Transaction started", "")
|
||||
|
||||
defer func() {
|
||||
if p := recover(); p != nil {
|
||||
if rollbackErr := tx.Rollback(); rollbackErr != nil {
|
||||
Fatal("failed to rollback transaction: %v", rollbackErr)
|
||||
rollbackErr := tx.Rollback()
|
||||
if rollbackErr != nil {
|
||||
Fatalf("failed to rollback transaction: %v", rollbackErr)
|
||||
}
|
||||
|
||||
panic(p)
|
||||
} else if err != nil {
|
||||
if rollbackErr := tx.Rollback(); rollbackErr != nil {
|
||||
Fatal("failed to rollback transaction: %v", rollbackErr)
|
||||
rollbackErr := tx.Rollback()
|
||||
if rollbackErr != nil {
|
||||
Fatalf("failed to rollback transaction: %v", rollbackErr)
|
||||
}
|
||||
}
|
||||
}()
|
||||
@@ -90,6 +95,7 @@ func (r *Repositories) WithReadTx(ctx context.Context, fn TxFunc) error {
|
||||
opts := &sql.TxOptions{
|
||||
ReadOnly: true,
|
||||
}
|
||||
|
||||
tx, err := r.db.BeginTx(ctx, opts)
|
||||
if err != nil {
|
||||
return fmt.Errorf("beginning read transaction: %w", err)
|
||||
@@ -97,13 +103,16 @@ func (r *Repositories) WithReadTx(ctx context.Context, fn TxFunc) error {
|
||||
|
||||
defer func() {
|
||||
if p := recover(); p != nil {
|
||||
if rollbackErr := tx.Rollback(); rollbackErr != nil {
|
||||
Fatal("failed to rollback transaction: %v", rollbackErr)
|
||||
rollbackErr := tx.Rollback()
|
||||
if rollbackErr != nil {
|
||||
Fatalf("failed to rollback transaction: %v", rollbackErr)
|
||||
}
|
||||
|
||||
panic(p)
|
||||
} else if err != nil {
|
||||
if rollbackErr := tx.Rollback(); rollbackErr != nil {
|
||||
Fatal("failed to rollback transaction: %v", rollbackErr)
|
||||
rollbackErr := tx.Rollback()
|
||||
if rollbackErr != nil {
|
||||
Fatalf("failed to rollback transaction: %v", rollbackErr)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
@@ -3,7 +3,7 @@ package database
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -28,7 +28,9 @@ func TestRepositoriesTransaction(t *testing.T) {
|
||||
UID: 1000,
|
||||
GID: 1000,
|
||||
}
|
||||
if err := repos.Files.Create(ctx, tx, file); err != nil {
|
||||
|
||||
err := repos.Files.Create(ctx, tx, file)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -37,7 +39,9 @@ func TestRepositoriesTransaction(t *testing.T) {
|
||||
ChunkHash: types.ChunkHash("tx_chunk1"),
|
||||
Size: 512,
|
||||
}
|
||||
if err := repos.Chunks.Create(ctx, tx, chunk1); err != nil {
|
||||
|
||||
err = repos.Chunks.Create(ctx, tx, chunk1)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -45,7 +49,9 @@ func TestRepositoriesTransaction(t *testing.T) {
|
||||
ChunkHash: types.ChunkHash("tx_chunk2"),
|
||||
Size: 512,
|
||||
}
|
||||
if err := repos.Chunks.Create(ctx, tx, chunk2); err != nil {
|
||||
|
||||
err = repos.Chunks.Create(ctx, tx, chunk2)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -55,7 +61,9 @@ func TestRepositoriesTransaction(t *testing.T) {
|
||||
Idx: 0,
|
||||
ChunkHash: chunk1.ChunkHash,
|
||||
}
|
||||
if err := repos.FileChunks.Create(ctx, tx, fc1); err != nil {
|
||||
|
||||
err = repos.FileChunks.Create(ctx, tx, fc1)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -64,7 +72,9 @@ func TestRepositoriesTransaction(t *testing.T) {
|
||||
Idx: 1,
|
||||
ChunkHash: chunk2.ChunkHash,
|
||||
}
|
||||
if err := repos.FileChunks.Create(ctx, tx, fc2); err != nil {
|
||||
|
||||
err = repos.FileChunks.Create(ctx, tx, fc2)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -74,7 +84,9 @@ func TestRepositoriesTransaction(t *testing.T) {
|
||||
Hash: types.BlobHash("tx_blob1"),
|
||||
CreatedTS: time.Now().Truncate(time.Second),
|
||||
}
|
||||
if err := repos.Blobs.Create(ctx, tx, blob); err != nil {
|
||||
|
||||
err = repos.Blobs.Create(ctx, tx, blob)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -85,7 +97,9 @@ func TestRepositoriesTransaction(t *testing.T) {
|
||||
Offset: 0,
|
||||
Length: 512,
|
||||
}
|
||||
if err := repos.BlobChunks.Create(ctx, tx, bc1); err != nil {
|
||||
|
||||
err = repos.BlobChunks.Create(ctx, tx, bc1)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -95,13 +109,14 @@ func TestRepositoriesTransaction(t *testing.T) {
|
||||
Offset: 512,
|
||||
Length: 512,
|
||||
}
|
||||
if err := repos.BlobChunks.Create(ctx, tx, bc2); err != nil {
|
||||
|
||||
err = repos.BlobChunks.Create(ctx, tx, bc2)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("transaction failed: %v", err)
|
||||
}
|
||||
@@ -111,6 +126,7 @@ func TestRepositoriesTransaction(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get file: %v", err)
|
||||
}
|
||||
|
||||
if file == nil {
|
||||
t.Error("expected file after transaction")
|
||||
}
|
||||
@@ -119,6 +135,7 @@ func TestRepositoriesTransaction(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get file chunks: %v", err)
|
||||
}
|
||||
|
||||
if len(chunks) != 2 {
|
||||
t.Errorf("expected 2 file chunks, got %d", len(chunks))
|
||||
}
|
||||
@@ -127,6 +144,7 @@ func TestRepositoriesTransaction(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get blob: %v", err)
|
||||
}
|
||||
|
||||
if blob == nil {
|
||||
t.Error("expected blob after transaction")
|
||||
}
|
||||
@@ -150,7 +168,9 @@ func TestRepositoriesTransactionRollback(t *testing.T) {
|
||||
UID: 1000,
|
||||
GID: 1000,
|
||||
}
|
||||
if err := repos.Files.Create(ctx, tx, file); err != nil {
|
||||
|
||||
err := repos.Files.Create(ctx, tx, file)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -159,12 +179,14 @@ func TestRepositoriesTransactionRollback(t *testing.T) {
|
||||
ChunkHash: types.ChunkHash("rollback_chunk"),
|
||||
Size: 1024,
|
||||
}
|
||||
if err := repos.Chunks.Create(ctx, tx, chunk); err != nil {
|
||||
|
||||
err = repos.Chunks.Create(ctx, tx, chunk)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Return error to trigger rollback
|
||||
return fmt.Errorf("intentional rollback")
|
||||
return errors.New("intentional rollback")
|
||||
})
|
||||
|
||||
if err == nil || err.Error() != "intentional rollback" {
|
||||
@@ -176,6 +198,7 @@ func TestRepositoriesTransactionRollback(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("error checking for file: %v", err)
|
||||
}
|
||||
|
||||
if file != nil {
|
||||
t.Error("file should not exist after rollback")
|
||||
}
|
||||
@@ -184,6 +207,7 @@ func TestRepositoriesTransactionRollback(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("error checking for chunk: %v", err)
|
||||
}
|
||||
|
||||
if chunk != nil {
|
||||
t.Error("chunk should not exist after rollback")
|
||||
}
|
||||
@@ -205,6 +229,7 @@ func TestRepositoriesReadTransaction(t *testing.T) {
|
||||
UID: 1000,
|
||||
GID: 1000,
|
||||
}
|
||||
|
||||
err := repos.Files.Create(ctx, nil, file)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create file: %v", err)
|
||||
@@ -212,8 +237,10 @@ func TestRepositoriesReadTransaction(t *testing.T) {
|
||||
|
||||
// Test read-only transaction
|
||||
var retrievedFile *File
|
||||
|
||||
err = repos.WithReadTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
|
||||
var err error
|
||||
|
||||
retrievedFile, err = repos.Files.GetByPathTx(ctx, tx, "/test/read_file.txt")
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -232,7 +259,6 @@ func TestRepositoriesReadTransaction(t *testing.T) {
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("read transaction failed: %v", err)
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package database
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -39,6 +40,7 @@ func TestFileRepositoryUUIDGeneration(t *testing.T) {
|
||||
}
|
||||
|
||||
uuids := make(map[string]bool)
|
||||
|
||||
for _, file := range files {
|
||||
err := repo.Create(ctx, nil, file)
|
||||
if err != nil {
|
||||
@@ -54,6 +56,7 @@ func TestFileRepositoryUUIDGeneration(t *testing.T) {
|
||||
if uuids[file.ID.String()] {
|
||||
t.Errorf("duplicate UUID generated: %s", file.ID)
|
||||
}
|
||||
|
||||
uuids[file.ID.String()] = true
|
||||
}
|
||||
}
|
||||
@@ -90,16 +93,19 @@ func TestFileRepositoryGetByID(t *testing.T) {
|
||||
if retrieved.ID != file.ID {
|
||||
t.Errorf("ID mismatch: expected %s, got %s", file.ID, retrieved.ID)
|
||||
}
|
||||
|
||||
if retrieved.Path != file.Path {
|
||||
t.Errorf("Path mismatch: expected %s, got %s", file.Path, retrieved.Path)
|
||||
}
|
||||
|
||||
// Test non-existent ID
|
||||
nonExistentID := types.NewFileID() // Generate a new UUID that won't exist in the database
|
||||
|
||||
nonExistent, err := repo.GetByID(ctx, nonExistentID)
|
||||
if err != nil {
|
||||
t.Fatalf("GetByID should not return error for non-existent ID: %v", err)
|
||||
}
|
||||
|
||||
if nonExistent != nil {
|
||||
t.Error("expected nil for non-existent ID")
|
||||
}
|
||||
@@ -135,6 +141,7 @@ func TestOrphanedFileCleanup(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create file1: %v", err)
|
||||
}
|
||||
|
||||
err = repos.Files.Create(ctx, nil, file2)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create file2: %v", err)
|
||||
@@ -146,6 +153,7 @@ func TestOrphanedFileCleanup(t *testing.T) {
|
||||
Hostname: "test-host",
|
||||
StartedAt: time.Now(),
|
||||
}
|
||||
|
||||
err = repos.Snapshots.Create(ctx, nil, snapshot)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create snapshot: %v", err)
|
||||
@@ -168,6 +176,7 @@ func TestOrphanedFileCleanup(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("error getting file: %v", err)
|
||||
}
|
||||
|
||||
if orphanedFile != nil {
|
||||
t.Error("orphaned file should have been deleted")
|
||||
}
|
||||
@@ -177,6 +186,7 @@ func TestOrphanedFileCleanup(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("error getting file: %v", err)
|
||||
}
|
||||
|
||||
if referencedFile == nil {
|
||||
t.Error("referenced file should not have been deleted")
|
||||
}
|
||||
@@ -204,6 +214,7 @@ func TestOrphanedChunkCleanup(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create chunk1: %v", err)
|
||||
}
|
||||
|
||||
err = repos.Chunks.Create(ctx, nil, chunk2)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create chunk2: %v", err)
|
||||
@@ -218,6 +229,7 @@ func TestOrphanedChunkCleanup(t *testing.T) {
|
||||
UID: 1000,
|
||||
GID: 1000,
|
||||
}
|
||||
|
||||
err = repos.Files.Create(ctx, nil, file)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create file: %v", err)
|
||||
@@ -229,6 +241,7 @@ func TestOrphanedChunkCleanup(t *testing.T) {
|
||||
Idx: 0,
|
||||
ChunkHash: chunk2.ChunkHash,
|
||||
}
|
||||
|
||||
err = repos.FileChunks.Create(ctx, nil, fc)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create file chunk: %v", err)
|
||||
@@ -245,6 +258,7 @@ func TestOrphanedChunkCleanup(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("error getting chunk: %v", err)
|
||||
}
|
||||
|
||||
if orphanedChunk != nil {
|
||||
t.Error("orphaned chunk should have been deleted")
|
||||
}
|
||||
@@ -254,6 +268,7 @@ func TestOrphanedChunkCleanup(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("error getting chunk: %v", err)
|
||||
}
|
||||
|
||||
if referencedChunk == nil {
|
||||
t.Error("referenced chunk should not have been deleted")
|
||||
}
|
||||
@@ -283,6 +298,7 @@ func TestOrphanedBlobCleanup(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create blob1: %v", err)
|
||||
}
|
||||
|
||||
err = repos.Blobs.Create(ctx, nil, blob2)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create blob2: %v", err)
|
||||
@@ -294,6 +310,7 @@ func TestOrphanedBlobCleanup(t *testing.T) {
|
||||
Hostname: "test-host",
|
||||
StartedAt: time.Now(),
|
||||
}
|
||||
|
||||
err = repos.Snapshots.Create(ctx, nil, snapshot)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create snapshot: %v", err)
|
||||
@@ -316,6 +333,7 @@ func TestOrphanedBlobCleanup(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("error getting blob: %v", err)
|
||||
}
|
||||
|
||||
if orphanedBlob != nil {
|
||||
t.Error("orphaned blob should have been deleted")
|
||||
}
|
||||
@@ -325,6 +343,7 @@ func TestOrphanedBlobCleanup(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("error getting blob: %v", err)
|
||||
}
|
||||
|
||||
if referencedBlob == nil {
|
||||
t.Error("referenced blob should not have been deleted")
|
||||
}
|
||||
@@ -347,6 +366,7 @@ func TestFileChunkRepositoryWithUUIDs(t *testing.T) {
|
||||
UID: 1000,
|
||||
GID: 1000,
|
||||
}
|
||||
|
||||
err := repos.Files.Create(ctx, nil, file)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create file: %v", err)
|
||||
@@ -359,6 +379,7 @@ func TestFileChunkRepositoryWithUUIDs(t *testing.T) {
|
||||
ChunkHash: chunkHash,
|
||||
Size: 1024,
|
||||
}
|
||||
|
||||
err = repos.Chunks.Create(ctx, nil, chunk)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create chunk: %v", err)
|
||||
@@ -370,6 +391,7 @@ func TestFileChunkRepositoryWithUUIDs(t *testing.T) {
|
||||
Idx: i,
|
||||
ChunkHash: chunkHash,
|
||||
}
|
||||
|
||||
err = repos.FileChunks.Create(ctx, nil, fc)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create file chunk: %v", err)
|
||||
@@ -381,6 +403,7 @@ func TestFileChunkRepositoryWithUUIDs(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get file chunks: %v", err)
|
||||
}
|
||||
|
||||
if len(fileChunks) != 3 {
|
||||
t.Errorf("expected 3 chunks, got %d", len(fileChunks))
|
||||
}
|
||||
@@ -395,6 +418,7 @@ func TestFileChunkRepositoryWithUUIDs(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get file chunks after delete: %v", err)
|
||||
}
|
||||
|
||||
if len(fileChunks) != 0 {
|
||||
t.Errorf("expected 0 chunks after delete, got %d", len(fileChunks))
|
||||
}
|
||||
@@ -430,6 +454,7 @@ func TestChunkFileRepositoryWithUUIDs(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create file1: %v", err)
|
||||
}
|
||||
|
||||
err = repos.Files.Create(ctx, nil, file2)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create file2: %v", err)
|
||||
@@ -440,6 +465,7 @@ func TestChunkFileRepositoryWithUUIDs(t *testing.T) {
|
||||
ChunkHash: types.ChunkHash("shared-chunk"),
|
||||
Size: 1024,
|
||||
}
|
||||
|
||||
err = repos.Chunks.Create(ctx, nil, chunk)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create chunk: %v", err)
|
||||
@@ -463,6 +489,7 @@ func TestChunkFileRepositoryWithUUIDs(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create chunk file 1: %v", err)
|
||||
}
|
||||
|
||||
err = repos.ChunkFiles.Create(ctx, nil, cf2)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create chunk file 2: %v", err)
|
||||
@@ -473,6 +500,7 @@ func TestChunkFileRepositoryWithUUIDs(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get chunk files: %v", err)
|
||||
}
|
||||
|
||||
if len(chunkFiles) != 2 {
|
||||
t.Errorf("expected 2 files for chunk, got %d", len(chunkFiles))
|
||||
}
|
||||
@@ -482,6 +510,7 @@ func TestChunkFileRepositoryWithUUIDs(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get chunks by file ID: %v", err)
|
||||
}
|
||||
|
||||
if len(chunkFiles) != 1 {
|
||||
t.Errorf("expected 1 chunk for file, got %d", len(chunkFiles))
|
||||
}
|
||||
@@ -528,15 +557,19 @@ func TestSnapshotRepositoryExtendedFields(t *testing.T) {
|
||||
if retrieved.VaultikVersion != snapshot.VaultikVersion {
|
||||
t.Errorf("version mismatch: expected %s, got %s", snapshot.VaultikVersion, retrieved.VaultikVersion)
|
||||
}
|
||||
|
||||
if retrieved.VaultikGitRevision != snapshot.VaultikGitRevision {
|
||||
t.Errorf("git revision mismatch: expected %s, got %s", snapshot.VaultikGitRevision, retrieved.VaultikGitRevision)
|
||||
}
|
||||
|
||||
if retrieved.CompressionLevel != snapshot.CompressionLevel {
|
||||
t.Errorf("compression level mismatch: expected %d, got %d", snapshot.CompressionLevel, retrieved.CompressionLevel)
|
||||
}
|
||||
|
||||
if retrieved.BlobUncompressedSize != snapshot.BlobUncompressedSize {
|
||||
t.Errorf("uncompressed size mismatch: expected %d, got %d", snapshot.BlobUncompressedSize, retrieved.BlobUncompressedSize)
|
||||
}
|
||||
|
||||
if retrieved.UploadDurationMs != snapshot.UploadDurationMs {
|
||||
t.Errorf("upload duration mismatch: expected %d, got %d", snapshot.UploadDurationMs, retrieved.UploadDurationMs)
|
||||
}
|
||||
@@ -566,6 +599,7 @@ func TestComplexOrphanedDataScenario(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create snapshot1: %v", err)
|
||||
}
|
||||
|
||||
err = repos.Snapshots.Create(ctx, nil, snapshot2)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create snapshot2: %v", err)
|
||||
@@ -582,6 +616,7 @@ func TestComplexOrphanedDataScenario(t *testing.T) {
|
||||
UID: 1000,
|
||||
GID: 1000,
|
||||
}
|
||||
|
||||
err = repos.Files.Create(ctx, nil, files[i])
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create file%d: %v", i, err)
|
||||
@@ -598,14 +633,17 @@ func TestComplexOrphanedDataScenario(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
err = repos.Snapshots.AddFileByID(ctx, nil, snapshot1.ID.String(), files[1].ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
err = repos.Snapshots.AddFileByID(ctx, nil, snapshot2.ID.String(), files[1].ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
err = repos.Snapshots.AddFileByID(ctx, nil, snapshot2.ID.String(), files[2].ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -616,6 +654,7 @@ func TestComplexOrphanedDataScenario(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
err = repos.Snapshots.Delete(ctx, snapshot1.ID.String())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -633,6 +672,7 @@ func TestComplexOrphanedDataScenario(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("error getting file0: %v", err)
|
||||
}
|
||||
|
||||
if file0 != nil {
|
||||
t.Error("file0 should have been deleted")
|
||||
}
|
||||
@@ -642,6 +682,7 @@ func TestComplexOrphanedDataScenario(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("error getting file1: %v", err)
|
||||
}
|
||||
|
||||
if file1 == nil {
|
||||
t.Error("file1 should still exist")
|
||||
}
|
||||
@@ -651,6 +692,7 @@ func TestComplexOrphanedDataScenario(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("error getting file2: %v", err)
|
||||
}
|
||||
|
||||
if file2 == nil {
|
||||
t.Error("file2 should still exist")
|
||||
}
|
||||
@@ -673,17 +715,19 @@ func TestCascadeDelete(t *testing.T) {
|
||||
UID: 1000,
|
||||
GID: 1000,
|
||||
}
|
||||
|
||||
err := repos.Files.Create(ctx, nil, file)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create file: %v", err)
|
||||
}
|
||||
|
||||
// Create chunks and file-chunk mappings
|
||||
for i := 0; i < 3; i++ {
|
||||
for i := range 3 {
|
||||
chunk := &Chunk{
|
||||
ChunkHash: types.ChunkHash(fmt.Sprintf("cascade-chunk-%d", i)),
|
||||
Size: 1024,
|
||||
}
|
||||
|
||||
err = repos.Chunks.Create(ctx, nil, chunk)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create chunk: %v", err)
|
||||
@@ -694,6 +738,7 @@ func TestCascadeDelete(t *testing.T) {
|
||||
Idx: i,
|
||||
ChunkHash: chunk.ChunkHash,
|
||||
}
|
||||
|
||||
err = repos.FileChunks.Create(ctx, nil, fc)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create file chunk: %v", err)
|
||||
@@ -705,6 +750,7 @@ func TestCascadeDelete(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if len(fileChunks) != 3 {
|
||||
t.Errorf("expected 3 file chunks, got %d", len(fileChunks))
|
||||
}
|
||||
@@ -720,6 +766,7 @@ func TestCascadeDelete(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if len(fileChunks) != 0 {
|
||||
t.Errorf("expected 0 file chunks after cascade delete, got %d", len(fileChunks))
|
||||
}
|
||||
@@ -744,6 +791,7 @@ func TestTransactionIsolation(t *testing.T) {
|
||||
UID: 1000,
|
||||
GID: 1000,
|
||||
}
|
||||
|
||||
err := repos.Files.Create(ctx, tx, file)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -754,9 +802,8 @@ func TestTransactionIsolation(t *testing.T) {
|
||||
// For now, we'll just test that rollback works
|
||||
|
||||
// Return an error to trigger rollback
|
||||
return fmt.Errorf("intentional rollback")
|
||||
return errors.New("intentional rollback")
|
||||
})
|
||||
|
||||
if err == nil {
|
||||
t.Fatal("expected error from transaction")
|
||||
}
|
||||
@@ -766,6 +813,7 @@ func TestTransactionIsolation(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if len(files) != 0 {
|
||||
t.Error("file should not exist after rollback")
|
||||
}
|
||||
@@ -790,13 +838,14 @@ func TestConcurrentOrphanedCleanup(t *testing.T) {
|
||||
Hostname: "test-host",
|
||||
StartedAt: time.Now(),
|
||||
}
|
||||
|
||||
err := repos.Snapshots.Create(ctx, nil, snapshot)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Create many files, some orphaned
|
||||
for i := 0; i < 20; i++ {
|
||||
for i := range 20 {
|
||||
file := &File{
|
||||
Path: types.FilePath(fmt.Sprintf("/concurrent-%d.txt", i)),
|
||||
MTime: time.Now().Truncate(time.Second),
|
||||
@@ -805,6 +854,7 @@ func TestConcurrentOrphanedCleanup(t *testing.T) {
|
||||
UID: 1000,
|
||||
GID: 1000,
|
||||
}
|
||||
|
||||
err = repos.Files.Create(ctx, nil, file)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -822,14 +872,15 @@ func TestConcurrentOrphanedCleanup(t *testing.T) {
|
||||
// Run multiple cleanup operations concurrently
|
||||
// Note: SQLite has limited support for concurrent writes, so we expect some to fail
|
||||
done := make(chan error, 3)
|
||||
for i := 0; i < 3; i++ {
|
||||
|
||||
for range 3 {
|
||||
go func() {
|
||||
done <- repos.Files.DeleteOrphaned(ctx)
|
||||
}()
|
||||
}
|
||||
|
||||
// Wait for all to complete
|
||||
for i := 0; i < 3; i++ {
|
||||
for i := range 3 {
|
||||
err := <-done
|
||||
if err != nil {
|
||||
t.Errorf("cleanup %d failed: %v", i, err)
|
||||
@@ -850,10 +901,12 @@ func TestConcurrentOrphanedCleanup(t *testing.T) {
|
||||
// Verify all remaining files are even-numbered
|
||||
for _, file := range files {
|
||||
var num int
|
||||
|
||||
_, err := fmt.Sscanf(file.Path.String(), "/concurrent-%d.txt", &num)
|
||||
if err != nil {
|
||||
t.Logf("failed to parse file number from %s: %v", file.Path, err)
|
||||
}
|
||||
|
||||
if num%2 != 0 {
|
||||
t.Errorf("odd-numbered file %s should have been deleted", file.Path)
|
||||
}
|
||||
|
||||
@@ -36,12 +36,14 @@ func TestOrphanedFileCleanupDebug(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create file1: %v", err)
|
||||
}
|
||||
|
||||
t.Logf("Created file1 with ID: %s", file1.ID)
|
||||
|
||||
err = repos.Files.Create(ctx, nil, file2)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create file2: %v", err)
|
||||
}
|
||||
|
||||
t.Logf("Created file2 with ID: %s", file2.ID)
|
||||
|
||||
// Create a snapshot and reference only file2
|
||||
@@ -50,18 +52,22 @@ func TestOrphanedFileCleanupDebug(t *testing.T) {
|
||||
Hostname: "test-host",
|
||||
StartedAt: time.Now(),
|
||||
}
|
||||
|
||||
err = repos.Snapshots.Create(ctx, nil, snapshot)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create snapshot: %v", err)
|
||||
}
|
||||
|
||||
t.Logf("Created snapshot: %s", snapshot.ID)
|
||||
|
||||
// Check snapshot_files before adding
|
||||
var count int
|
||||
|
||||
err = db.conn.QueryRow("SELECT COUNT(*) FROM snapshot_files").Scan(&count)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
t.Logf("snapshot_files count before add: %d", count)
|
||||
|
||||
// Add file2 to snapshot
|
||||
@@ -69,6 +75,7 @@ func TestOrphanedFileCleanupDebug(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to add file to snapshot: %v", err)
|
||||
}
|
||||
|
||||
t.Logf("Added file2 to snapshot")
|
||||
|
||||
// Check snapshot_files after adding
|
||||
@@ -76,6 +83,7 @@ func TestOrphanedFileCleanupDebug(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
t.Logf("snapshot_files count after add: %d", count)
|
||||
|
||||
// Check which files are referenced
|
||||
@@ -84,16 +92,22 @@ func TestOrphanedFileCleanupDebug(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
if err := rows.Close(); err != nil {
|
||||
err := rows.Close()
|
||||
if err != nil {
|
||||
t.Logf("failed to close rows: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
t.Log("Files in snapshot_files:")
|
||||
|
||||
for rows.Next() {
|
||||
var fileID string
|
||||
if err := rows.Scan(&fileID); err != nil {
|
||||
|
||||
err := rows.Scan(&fileID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
t.Logf(" - %s", fileID)
|
||||
}
|
||||
|
||||
@@ -102,6 +116,7 @@ func TestOrphanedFileCleanupDebug(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
t.Logf("Files count before cleanup: %d", count)
|
||||
|
||||
// Run orphaned cleanup
|
||||
@@ -109,6 +124,7 @@ func TestOrphanedFileCleanupDebug(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to delete orphaned files: %v", err)
|
||||
}
|
||||
|
||||
t.Log("Ran orphaned cleanup")
|
||||
|
||||
// Check files after cleanup
|
||||
@@ -116,6 +132,7 @@ func TestOrphanedFileCleanupDebug(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
t.Logf("Files count after cleanup: %d", count)
|
||||
|
||||
// List remaining files
|
||||
@@ -123,7 +140,9 @@ func TestOrphanedFileCleanupDebug(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
t.Log("Remaining files:")
|
||||
|
||||
for _, f := range files {
|
||||
t.Logf(" - ID: %s, Path: %s", f.ID, f.Path)
|
||||
}
|
||||
@@ -133,10 +152,12 @@ func TestOrphanedFileCleanupDebug(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("error getting file: %v", err)
|
||||
}
|
||||
|
||||
if orphanedFile != nil {
|
||||
t.Error("orphaned file should have been deleted")
|
||||
// Let's check why it wasn't deleted
|
||||
var exists bool
|
||||
|
||||
err = db.conn.QueryRow(`
|
||||
SELECT EXISTS(
|
||||
SELECT 1 FROM snapshot_files
|
||||
@@ -145,6 +166,7 @@ func TestOrphanedFileCleanupDebug(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
t.Logf("File1 exists in snapshot_files: %v", exists)
|
||||
} else {
|
||||
t.Log("Orphaned file was correctly deleted")
|
||||
@@ -155,6 +177,7 @@ func TestOrphanedFileCleanupDebug(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("error getting file: %v", err)
|
||||
}
|
||||
|
||||
if referencedFile == nil {
|
||||
t.Error("referenced file should not have been deleted")
|
||||
} else {
|
||||
|
||||
@@ -98,6 +98,7 @@ func TestFileRepositoryEdgeCases(t *testing.T) {
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Create() error = %v, wantErr %v", err, tt.wantErr)
|
||||
}
|
||||
|
||||
if err != nil && tt.errMsg != "" && !strings.Contains(err.Error(), tt.errMsg) {
|
||||
t.Errorf("Create() error = %v, want error containing %q", err, tt.errMsg)
|
||||
}
|
||||
@@ -136,6 +137,7 @@ func TestDuplicateHandling(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create file1: %v", err)
|
||||
}
|
||||
|
||||
originalID := file1.ID
|
||||
|
||||
// Create with same path should update the existing record (UPSERT behavior)
|
||||
@@ -190,6 +192,7 @@ func TestDuplicateHandling(t *testing.T) {
|
||||
UID: 1000,
|
||||
GID: 1000,
|
||||
}
|
||||
|
||||
err := repos.Files.Create(ctx, nil, file)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -199,6 +202,7 @@ func TestDuplicateHandling(t *testing.T) {
|
||||
ChunkHash: types.ChunkHash("test-chunk-dup"),
|
||||
Size: 1024,
|
||||
}
|
||||
|
||||
err = repos.Chunks.Create(ctx, nil, chunk)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -325,6 +329,7 @@ func TestLargeDatasets(t *testing.T) {
|
||||
Hostname: "test-host",
|
||||
StartedAt: time.Now(),
|
||||
}
|
||||
|
||||
err := repos.Snapshots.Create(ctx, nil, snapshot)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -332,11 +337,12 @@ func TestLargeDatasets(t *testing.T) {
|
||||
|
||||
// Create many files
|
||||
const fileCount = 1000
|
||||
|
||||
fileIDs := make([]types.FileID, fileCount)
|
||||
|
||||
t.Run("create many files", func(t *testing.T) {
|
||||
start := time.Now()
|
||||
for i := 0; i < fileCount; i++ {
|
||||
for i := range fileCount {
|
||||
file := &File{
|
||||
Path: types.FilePath(fmt.Sprintf("/large/file%05d.txt", i)),
|
||||
MTime: time.Now(),
|
||||
@@ -345,10 +351,12 @@ func TestLargeDatasets(t *testing.T) {
|
||||
UID: uint32(1000 + (i % 10)),
|
||||
GID: uint32(1000 + (i % 10)),
|
||||
}
|
||||
|
||||
err := repos.Files.Create(ctx, nil, file)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create file %d: %v", i, err)
|
||||
}
|
||||
|
||||
fileIDs[i] = file.ID
|
||||
|
||||
// Add half to snapshot
|
||||
@@ -359,29 +367,35 @@ func TestLargeDatasets(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
t.Logf("Created %d files in %v", fileCount, time.Since(start))
|
||||
})
|
||||
|
||||
// Test ListByPrefix performance
|
||||
t.Run("list by prefix performance", func(t *testing.T) {
|
||||
start := time.Now()
|
||||
|
||||
files, err := repos.Files.ListByPrefix(ctx, "/large/")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if len(files) != fileCount {
|
||||
t.Errorf("expected %d files, got %d", fileCount, len(files))
|
||||
}
|
||||
|
||||
t.Logf("Listed %d files in %v", len(files), time.Since(start))
|
||||
})
|
||||
|
||||
// Test orphaned cleanup performance
|
||||
t.Run("orphaned cleanup performance", func(t *testing.T) {
|
||||
start := time.Now()
|
||||
|
||||
err := repos.Files.DeleteOrphaned(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
t.Logf("Cleaned up orphaned files in %v", time.Since(start))
|
||||
|
||||
// Verify correct number remain
|
||||
@@ -389,6 +403,7 @@ func TestLargeDatasets(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if len(files) != fileCount/2 {
|
||||
t.Errorf("expected %d files after cleanup, got %d", fileCount/2, len(files))
|
||||
}
|
||||
@@ -409,6 +424,7 @@ func TestErrorPropagation(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Errorf("GetByID should not return error for non-existent ID, got: %v", err)
|
||||
}
|
||||
|
||||
if file != nil {
|
||||
t.Error("expected nil file for non-existent ID")
|
||||
}
|
||||
@@ -420,6 +436,7 @@ func TestErrorPropagation(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Errorf("GetByPath should not return error for non-existent path, got: %v", err)
|
||||
}
|
||||
|
||||
if file != nil {
|
||||
t.Error("expected nil file for non-existent path")
|
||||
}
|
||||
@@ -432,10 +449,12 @@ func TestErrorPropagation(t *testing.T) {
|
||||
Idx: 0,
|
||||
ChunkHash: types.ChunkHash("some-chunk"),
|
||||
}
|
||||
|
||||
err := repos.FileChunks.Create(ctx, nil, fc)
|
||||
if err == nil {
|
||||
t.Error("expected error for invalid foreign key")
|
||||
}
|
||||
|
||||
if !strings.Contains(err.Error(), "FOREIGN KEY") {
|
||||
t.Errorf("expected foreign key error, got: %v", err)
|
||||
}
|
||||
@@ -475,6 +494,7 @@ func TestQueryInjection(t *testing.T) {
|
||||
|
||||
// Verify tables still exist
|
||||
var count int
|
||||
|
||||
err := db.conn.QueryRow("SELECT COUNT(*) FROM files").Scan(&count)
|
||||
if err != nil {
|
||||
t.Fatal("files table was damaged by injection")
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/vaultik/internal/types"
|
||||
@@ -26,6 +27,7 @@ func (r *SnapshotRepository) Create(ctx context.Context, tx *sql.Tx, snapshot *S
|
||||
`
|
||||
|
||||
var completedAt *int64
|
||||
|
||||
if snapshot.CompletedAt != nil {
|
||||
ts := snapshot.CompletedAt.Unix()
|
||||
completedAt = &ts
|
||||
@@ -84,9 +86,11 @@ func (r *SnapshotRepository) UpdateCounts(ctx context.Context, tx *sql.Tx, snaps
|
||||
func (r *SnapshotRepository) UpdateExtendedStats(ctx context.Context, tx *sql.Tx, snapshotID string, blobUncompressedSize int64, compressionLevel int, uploadDurationMs int64) error {
|
||||
// Calculate compression ratio based on uncompressed vs compressed sizes
|
||||
var compressionRatio float64
|
||||
|
||||
if blobUncompressedSize > 0 {
|
||||
// Get current blob_size from DB to calculate ratio
|
||||
var blobSize int64
|
||||
|
||||
queryGet := `SELECT blob_size FROM snapshots WHERE id = ?`
|
||||
if tx != nil {
|
||||
err := tx.QueryRowContext(ctx, queryGet, snapshotID).Scan(&blobSize)
|
||||
@@ -99,6 +103,7 @@ func (r *SnapshotRepository) UpdateExtendedStats(ctx context.Context, tx *sql.Tx
|
||||
return fmt.Errorf("getting blob size: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
compressionRatio = float64(blobSize) / float64(blobUncompressedSize)
|
||||
} else {
|
||||
compressionRatio = 1.0
|
||||
@@ -124,6 +129,7 @@ func (r *SnapshotRepository) UpdateExtendedStats(ctx context.Context, tx *sql.Tx
|
||||
if err != nil {
|
||||
return fmt.Errorf("updating extended stats: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -136,9 +142,11 @@ func (r *SnapshotRepository) GetByID(ctx context.Context, snapshotID string) (*S
|
||||
WHERE id = ?
|
||||
`
|
||||
|
||||
var snapshot Snapshot
|
||||
var startedAtUnix int64
|
||||
var completedAtUnix *int64
|
||||
var (
|
||||
snapshot Snapshot
|
||||
startedAtUnix int64
|
||||
completedAtUnix *int64
|
||||
)
|
||||
|
||||
err := r.db.conn.QueryRowContext(ctx, query, snapshotID).Scan(
|
||||
&snapshot.ID,
|
||||
@@ -162,6 +170,7 @@ func (r *SnapshotRepository) GetByID(ctx context.Context, snapshotID string) (*S
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("querying snapshot: %w", err)
|
||||
}
|
||||
@@ -190,10 +199,13 @@ func (r *SnapshotRepository) ListRecent(ctx context.Context, limit int) ([]*Snap
|
||||
defer CloseRows(rows)
|
||||
|
||||
var snapshots []*Snapshot
|
||||
|
||||
for rows.Next() {
|
||||
var snapshot Snapshot
|
||||
var startedAtUnix int64
|
||||
var completedAtUnix *int64
|
||||
var (
|
||||
snapshot Snapshot
|
||||
startedAtUnix int64
|
||||
completedAtUnix *int64
|
||||
)
|
||||
|
||||
err := rows.Scan(
|
||||
&snapshot.ID,
|
||||
@@ -301,28 +313,35 @@ func (r *SnapshotRepository) AddFilesByIDBatch(ctx context.Context, tx *sql.Tx,
|
||||
const batchSize = 400
|
||||
|
||||
for i := 0; i < len(fileIDs); i += batchSize {
|
||||
end := i + batchSize
|
||||
if end > len(fileIDs) {
|
||||
end = len(fileIDs)
|
||||
}
|
||||
end := min(i+batchSize, len(fileIDs))
|
||||
|
||||
batch := fileIDs[i:end]
|
||||
|
||||
query := "INSERT OR IGNORE INTO snapshot_files (snapshot_id, file_id) VALUES "
|
||||
args := make([]interface{}, 0, len(batch)*2)
|
||||
|
||||
args := make([]any, 0, len(batch)*2)
|
||||
|
||||
var querySb312 strings.Builder
|
||||
|
||||
for j, fileID := range batch {
|
||||
if j > 0 {
|
||||
query += ", "
|
||||
querySb312.WriteString(", ")
|
||||
}
|
||||
query += "(?, ?)"
|
||||
|
||||
querySb312.WriteString("(?, ?)")
|
||||
|
||||
args = append(args, snapshotID, fileID.String())
|
||||
}
|
||||
|
||||
query += querySb312.String()
|
||||
|
||||
var err error
|
||||
if tx != nil {
|
||||
_, err = tx.ExecContext(ctx, query, args...)
|
||||
} else {
|
||||
_, err = r.db.ExecWithLog(ctx, query, args...)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("batch adding files to snapshot: %w", err)
|
||||
}
|
||||
@@ -353,18 +372,22 @@ func (r *SnapshotRepository) PopulateReferencedBlobs(ctx context.Context, tx *sq
|
||||
AND blobs.blob_hash IS NOT NULL
|
||||
`
|
||||
|
||||
var result sql.Result
|
||||
var err error
|
||||
var (
|
||||
result sql.Result
|
||||
err error
|
||||
)
|
||||
if tx != nil {
|
||||
result, err = tx.ExecContext(ctx, query, snapshotID, snapshotID)
|
||||
} else {
|
||||
result, err = r.db.ExecWithLog(ctx, query, snapshotID, snapshotID)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("populating referenced blobs: %w", err)
|
||||
}
|
||||
|
||||
n, _ := result.RowsAffected()
|
||||
|
||||
return n, nil
|
||||
}
|
||||
|
||||
@@ -405,11 +428,15 @@ func (r *SnapshotRepository) GetBlobHashes(ctx context.Context, snapshotID strin
|
||||
defer CloseRows(rows)
|
||||
|
||||
var blobs []string
|
||||
|
||||
for rows.Next() {
|
||||
var blobHash string
|
||||
if err := rows.Scan(&blobHash); err != nil {
|
||||
|
||||
err := rows.Scan(&blobHash)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scanning blob hash: %w", err)
|
||||
}
|
||||
|
||||
blobs = append(blobs, blobHash)
|
||||
}
|
||||
|
||||
@@ -426,6 +453,7 @@ func (r *SnapshotRepository) GetSnapshotTotalCompressedSize(ctx context.Context,
|
||||
`
|
||||
|
||||
var totalSize int64
|
||||
|
||||
err := r.db.conn.QueryRowContext(ctx, query, snapshotID).Scan(&totalSize)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("querying total compressed size: %w", err)
|
||||
@@ -449,6 +477,7 @@ func (r *SnapshotRepository) GetSnapshotUncompressedChunkSize(ctx context.Contex
|
||||
`
|
||||
|
||||
var totalSize int64
|
||||
|
||||
err := r.db.conn.QueryRowContext(ctx, query, snapshotID).Scan(&totalSize)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("querying uncompressed chunk size: %w", err)
|
||||
@@ -485,6 +514,7 @@ func (r *SnapshotRepository) GetSnapshotNewChunkSize(ctx context.Context, snapsh
|
||||
`
|
||||
|
||||
var totalSize int64
|
||||
|
||||
err := r.db.conn.QueryRowContext(ctx, query, snapshotID, snapshotID, snapshotID).Scan(&totalSize)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("querying new chunk size: %w", err)
|
||||
@@ -509,10 +539,13 @@ func (r *SnapshotRepository) GetIncompleteSnapshots(ctx context.Context) ([]*Sna
|
||||
defer CloseRows(rows)
|
||||
|
||||
var snapshots []*Snapshot
|
||||
|
||||
for rows.Next() {
|
||||
var snapshot Snapshot
|
||||
var startedAtUnix int64
|
||||
var completedAtUnix *int64
|
||||
var (
|
||||
snapshot Snapshot
|
||||
startedAtUnix int64
|
||||
completedAtUnix *int64
|
||||
)
|
||||
|
||||
err := rows.Scan(
|
||||
&snapshot.ID,
|
||||
@@ -560,10 +593,13 @@ func (r *SnapshotRepository) GetIncompleteByHostname(ctx context.Context, hostna
|
||||
defer CloseRows(rows)
|
||||
|
||||
var snapshots []*Snapshot
|
||||
|
||||
for rows.Next() {
|
||||
var snapshot Snapshot
|
||||
var startedAtUnix int64
|
||||
var completedAtUnix *int64
|
||||
var (
|
||||
snapshot Snapshot
|
||||
startedAtUnix int64
|
||||
completedAtUnix *int64
|
||||
)
|
||||
|
||||
err := rows.Scan(
|
||||
&snapshot.ID,
|
||||
|
||||
@@ -52,15 +52,19 @@ func TestSnapshotRepository(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get snapshot: %v", err)
|
||||
}
|
||||
|
||||
if retrieved == nil {
|
||||
t.Fatal("expected snapshot, got nil")
|
||||
}
|
||||
|
||||
if retrieved.ID != snapshot.ID {
|
||||
t.Errorf("ID mismatch: got %s, want %s", retrieved.ID, snapshot.ID)
|
||||
}
|
||||
|
||||
if retrieved.Hostname != snapshot.Hostname {
|
||||
t.Errorf("hostname mismatch: got %s, want %s", retrieved.Hostname, snapshot.Hostname)
|
||||
}
|
||||
|
||||
if retrieved.FileCount != snapshot.FileCount {
|
||||
t.Errorf("file count mismatch: got %d, want %d", retrieved.FileCount, snapshot.FileCount)
|
||||
}
|
||||
@@ -75,21 +79,27 @@ func TestSnapshotRepository(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get updated snapshot: %v", err)
|
||||
}
|
||||
|
||||
if retrieved.FileCount != 200 {
|
||||
t.Errorf("file count not updated: got %d, want %d", retrieved.FileCount, 200)
|
||||
}
|
||||
|
||||
if retrieved.ChunkCount != 1000 {
|
||||
t.Errorf("chunk count not updated: got %d, want %d", retrieved.ChunkCount, 1000)
|
||||
}
|
||||
|
||||
if retrieved.BlobCount != 20 {
|
||||
t.Errorf("blob count not updated: got %d, want %d", retrieved.BlobCount, 20)
|
||||
}
|
||||
|
||||
if retrieved.TotalSize != twoHundredMebibytes {
|
||||
t.Errorf("total size not updated: got %d, want %d", retrieved.TotalSize, twoHundredMebibytes)
|
||||
}
|
||||
|
||||
if retrieved.BlobSize != sixtyMebibytes {
|
||||
t.Errorf("blob size not updated: got %d, want %d", retrieved.BlobSize, sixtyMebibytes)
|
||||
}
|
||||
|
||||
expectedRatio := compressionRatioPoint3 // 0.3
|
||||
if math.Abs(retrieved.CompressionRatio-expectedRatio) > 0.001 {
|
||||
t.Errorf("compression ratio not updated: got %f, want %f", retrieved.CompressionRatio, expectedRatio)
|
||||
@@ -108,6 +118,7 @@ func TestSnapshotRepository(t *testing.T) {
|
||||
ChunkCount: int64(500 * i),
|
||||
BlobCount: int64(10 * i),
|
||||
}
|
||||
|
||||
err := repo.Create(ctx, nil, s)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create snapshot %d: %v", i, err)
|
||||
@@ -119,12 +130,13 @@ func TestSnapshotRepository(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to list recent snapshots: %v", err)
|
||||
}
|
||||
|
||||
if len(recent) != 3 {
|
||||
t.Errorf("expected 3 recent snapshots, got %d", len(recent))
|
||||
}
|
||||
|
||||
// Verify order (most recent first)
|
||||
for i := 0; i < len(recent)-1; i++ {
|
||||
for i := range len(recent) - 1 {
|
||||
if recent[i].StartedAt.Before(recent[i+1].StartedAt) {
|
||||
t.Error("snapshots not in descending order")
|
||||
}
|
||||
@@ -143,6 +155,7 @@ func TestSnapshotRepositoryNotFound(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if snapshot != nil {
|
||||
t.Error("expected nil for non-existent snapshot")
|
||||
}
|
||||
|
||||
@@ -53,6 +53,7 @@ func (r *UploadRepository) GetByBlobHash(ctx context.Context, blobHash string) (
|
||||
`
|
||||
|
||||
var upload Upload
|
||||
|
||||
err := r.conn.QueryRowContext(ctx, query, blobHash).Scan(
|
||||
&upload.BlobHash,
|
||||
&upload.UploadedAt,
|
||||
@@ -63,6 +64,7 @@ func (r *UploadRepository) GetByBlobHash(ctx context.Context, blobHash string) (
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -84,17 +86,22 @@ func (r *UploadRepository) GetRecentUploads(ctx context.Context, limit int) ([]*
|
||||
return nil, err
|
||||
}
|
||||
defer func() {
|
||||
if err := rows.Close(); err != nil {
|
||||
err := rows.Close()
|
||||
if err != nil {
|
||||
log.Error("failed to close rows", "error", err)
|
||||
}
|
||||
}()
|
||||
|
||||
var uploads []*Upload
|
||||
|
||||
for rows.Next() {
|
||||
var upload Upload
|
||||
if err := rows.Scan(&upload.BlobHash, &upload.UploadedAt, &upload.Size, &upload.DurationMs); err != nil {
|
||||
|
||||
err := rows.Scan(&upload.BlobHash, &upload.UploadedAt, &upload.Size, &upload.DurationMs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
uploads = append(uploads, &upload)
|
||||
}
|
||||
|
||||
@@ -115,6 +122,7 @@ func (r *UploadRepository) GetUploadStats(ctx context.Context, since time.Time)
|
||||
`
|
||||
|
||||
var stats UploadStats
|
||||
|
||||
err := r.conn.QueryRowContext(ctx, query, since).Scan(
|
||||
&stats.Count,
|
||||
&stats.TotalSize,
|
||||
@@ -138,10 +146,13 @@ type UploadStats struct {
|
||||
// GetCountBySnapshot returns the count of uploads for a specific snapshot
|
||||
func (r *UploadRepository) GetCountBySnapshot(ctx context.Context, snapshotID string) (int64, error) {
|
||||
query := `SELECT COUNT(*) FROM uploads WHERE snapshot_id = ?`
|
||||
|
||||
var count int64
|
||||
|
||||
err := r.conn.QueryRowContext(ctx, query, snapshotID).Scan(&count)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
return count, nil
|
||||
}
|
||||
|
||||
@@ -50,5 +50,6 @@ func (g *Globals) ShortCommit() string {
|
||||
if len(g.Commit) > 12 {
|
||||
return g.Commit[:12]
|
||||
}
|
||||
|
||||
return g.Commit
|
||||
}
|
||||
|
||||
@@ -84,6 +84,7 @@ func getCaller(skip int) string {
|
||||
if !ok {
|
||||
return "unknown"
|
||||
}
|
||||
|
||||
return fmt.Sprintf("%s:%d", filepath.Base(file), line)
|
||||
}
|
||||
|
||||
@@ -94,6 +95,7 @@ func Fatal(msg string, args ...any) {
|
||||
args = append(args, "caller", getCaller(2))
|
||||
logger.Error(msg, args...)
|
||||
}
|
||||
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
@@ -172,6 +174,7 @@ func With(args ...any) *slog.Logger {
|
||||
if logger != nil {
|
||||
return logger.With(args...)
|
||||
}
|
||||
|
||||
return slog.Default()
|
||||
}
|
||||
|
||||
|
||||
@@ -33,6 +33,7 @@ func NewTTYHandler(out io.Writer, opts *slog.HandlerOptions) *TTYHandler {
|
||||
if opts == nil {
|
||||
opts = &slog.HandlerOptions{}
|
||||
}
|
||||
|
||||
return &TTYHandler{
|
||||
out: out,
|
||||
opts: *opts,
|
||||
@@ -54,7 +55,9 @@ func (h *TTYHandler) Handle(_ context.Context, r slog.Record) error {
|
||||
|
||||
// Level and color
|
||||
level := r.Level.String()
|
||||
|
||||
var levelColor string
|
||||
|
||||
switch r.Level {
|
||||
case slog.LevelDebug:
|
||||
levelColor = colorGray
|
||||
@@ -96,10 +99,12 @@ func (h *TTYHandler) Handle(_ context.Context, r slog.Record) error {
|
||||
_, _ = fmt.Fprintf(h.out, " %s%s%s=%s%s%s",
|
||||
colorCyan, a.Key, colorReset,
|
||||
colorBlue, value, colorReset)
|
||||
|
||||
return true
|
||||
})
|
||||
|
||||
_, _ = fmt.Fprintln(h.out)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -122,6 +127,7 @@ func formatDuration(d time.Duration) string {
|
||||
} else if d < time.Minute {
|
||||
return fmt.Sprintf("%.1fs", d.Seconds())
|
||||
}
|
||||
|
||||
return d.String()
|
||||
}
|
||||
|
||||
@@ -131,10 +137,12 @@ func formatBytes(b int64) string {
|
||||
if b < unit {
|
||||
return fmt.Sprintf("%d B", b)
|
||||
}
|
||||
|
||||
div, exp := int64(unit), 0
|
||||
for n := b / unit; n >= unit; n /= unit {
|
||||
div *= unit
|
||||
exp++
|
||||
}
|
||||
|
||||
return fmt.Sprintf("%.1f %cB", float64(b)/float64(div), "KMGTPE"[exp])
|
||||
}
|
||||
|
||||
@@ -77,6 +77,7 @@ func (l *Lock) Release() error {
|
||||
}
|
||||
|
||||
l.path = "" // Prevent double-release
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -104,5 +105,6 @@ func isProcessRunning(pid int) bool {
|
||||
|
||||
// On Unix, FindProcess always succeeds. We need to send signal 0 to check.
|
||||
err = process.Signal(syscall.Signal(0))
|
||||
|
||||
return err == nil
|
||||
}
|
||||
|
||||
@@ -40,6 +40,7 @@ func TestAcquireBlocksSecondInstance(t *testing.T) {
|
||||
// Acquire first lock
|
||||
lock1, err := Acquire(tmpDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.NotNil(t, lock1)
|
||||
defer func() { _ = lock1.Release() }()
|
||||
|
||||
@@ -61,6 +62,7 @@ func TestAcquireWithStaleLock(t *testing.T) {
|
||||
// Should be able to acquire lock (stale lock is cleaned up)
|
||||
lock, err := Acquire(tmpDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.NotNil(t, lock)
|
||||
defer func() { _ = lock.Release() }()
|
||||
|
||||
@@ -88,6 +90,7 @@ func TestReleaseIsIdempotent(t *testing.T) {
|
||||
|
||||
func TestReleaseNilLock(t *testing.T) {
|
||||
var lock *Lock
|
||||
|
||||
err := lock.Release()
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
@@ -98,6 +101,7 @@ func TestAcquireCreatesDirectory(t *testing.T) {
|
||||
|
||||
lock, err := Acquire(nestedDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.NotNil(t, lock)
|
||||
defer func() { _ = lock.Release() }()
|
||||
|
||||
|
||||
@@ -42,7 +42,7 @@ type Config struct {
|
||||
// Used to suppress SDK warnings about checksums.
|
||||
type nopLogger struct{}
|
||||
|
||||
func (nopLogger) Logf(classification logging.Classification, format string, v ...interface{}) {}
|
||||
func (nopLogger) Logf(classification logging.Classification, format string, v ...any) {}
|
||||
|
||||
// NewClient creates a new S3 client with the provided configuration.
|
||||
// It establishes a connection to the S3-compatible storage service and
|
||||
@@ -92,6 +92,7 @@ func (c *Client) PutObject(ctx context.Context, key string, data io.Reader) erro
|
||||
Key: aws.String(fullKey),
|
||||
Body: data,
|
||||
})
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -137,6 +138,7 @@ func (c *Client) PutObjectWithProgress(ctx context.Context, key string, data io.
|
||||
// close the returned reader when done to avoid resource leaks.
|
||||
func (c *Client) GetObject(ctx context.Context, key string) (io.ReadCloser, error) {
|
||||
fullKey := c.prefix + key
|
||||
|
||||
result, err := c.s3Client.GetObject(ctx, &s3.GetObjectInput{
|
||||
Bucket: aws.String(c.bucket),
|
||||
Key: aws.String(fullKey),
|
||||
@@ -144,6 +146,7 @@ func (c *Client) GetObject(ctx context.Context, key string) (io.ReadCloser, erro
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return result.Body, nil
|
||||
}
|
||||
|
||||
@@ -156,6 +159,7 @@ func (c *Client) DeleteObject(ctx context.Context, key string) error {
|
||||
Bucket: aws.String(c.bucket),
|
||||
Key: aws.String(fullKey),
|
||||
})
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -168,6 +172,7 @@ func (c *Client) ListObjects(ctx context.Context, prefix string) ([]string, erro
|
||||
fullPrefix := c.prefix + prefix
|
||||
|
||||
var keys []string
|
||||
|
||||
paginator := s3.NewListObjectsV2Paginator(c.s3Client, &s3.ListObjectsV2Input{
|
||||
Bucket: aws.String(c.bucket),
|
||||
Prefix: aws.String(fullPrefix),
|
||||
@@ -186,6 +191,7 @@ func (c *Client) ListObjects(ctx context.Context, prefix string) ([]string, erro
|
||||
if len(key) > len(c.prefix) {
|
||||
key = key[len(c.prefix):]
|
||||
}
|
||||
|
||||
keys = append(keys, key)
|
||||
}
|
||||
}
|
||||
@@ -200,18 +206,23 @@ func (c *Client) ListObjects(ctx context.Context, prefix string) ([]string, erro
|
||||
// Note: This method returns false for any error, not just "not found".
|
||||
func (c *Client) HeadObject(ctx context.Context, key string) (bool, error) {
|
||||
fullKey := c.prefix + key
|
||||
|
||||
_, err := c.s3Client.HeadObject(ctx, &s3.HeadObjectInput{
|
||||
Bucket: aws.String(c.bucket),
|
||||
Key: aws.String(fullKey),
|
||||
})
|
||||
if err != nil {
|
||||
var notFound *s3types.NotFound
|
||||
var noSuchKey *s3types.NoSuchKey
|
||||
var (
|
||||
notFound *s3types.NotFound
|
||||
noSuchKey *s3types.NoSuchKey
|
||||
)
|
||||
if errors.As(err, ¬Found) || errors.As(err, &noSuchKey) {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
return false, err
|
||||
}
|
||||
|
||||
return true, nil
|
||||
}
|
||||
|
||||
@@ -247,6 +258,7 @@ func (c *Client) ListObjectsStream(ctx context.Context, prefix string, recursive
|
||||
page, err := paginator.NextPage(ctx)
|
||||
if err != nil {
|
||||
ch <- ObjectInfo{Err: err}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
@@ -257,6 +269,7 @@ func (c *Client) ListObjectsStream(ctx context.Context, prefix string, recursive
|
||||
if len(key) > len(c.prefix) {
|
||||
key = key[len(c.prefix):]
|
||||
}
|
||||
|
||||
ch <- ObjectInfo{
|
||||
Key: key,
|
||||
Size: *obj.Size,
|
||||
@@ -275,6 +288,7 @@ func (c *Client) ListObjectsStream(ctx context.Context, prefix string, recursive
|
||||
// Returns an error if the object doesn't exist or if the operation fails.
|
||||
func (c *Client) StatObject(ctx context.Context, key string) (*ObjectInfo, error) {
|
||||
fullKey := c.prefix + key
|
||||
|
||||
result, err := c.s3Client.HeadObject(ctx, &s3.HeadObjectInput{
|
||||
Bucket: aws.String(c.bucket),
|
||||
Key: aws.String(fullKey),
|
||||
@@ -313,6 +327,7 @@ func (c *Client) Endpoint() string {
|
||||
if c.endpoint == "" {
|
||||
return "s3.amazonaws.com"
|
||||
}
|
||||
|
||||
return c.endpoint
|
||||
}
|
||||
|
||||
@@ -329,11 +344,14 @@ func (pr *progressReader) Read(p []byte) (int, error) {
|
||||
n, err := pr.reader.Read(p)
|
||||
if n > 0 {
|
||||
atomic.AddInt64(&pr.read, int64(n))
|
||||
|
||||
if pr.callback != nil {
|
||||
if callbackErr := pr.callback(atomic.LoadInt64(&pr.read)); callbackErr != nil {
|
||||
callbackErr := pr.callback(atomic.LoadInt64(&pr.read))
|
||||
if callbackErr != nil {
|
||||
return n, callbackErr
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return n, err
|
||||
}
|
||||
|
||||
@@ -12,7 +12,8 @@ import (
|
||||
func TestClient(t *testing.T) {
|
||||
ts := NewTestServer(t)
|
||||
defer func() {
|
||||
if err := ts.Cleanup(); err != nil {
|
||||
err := ts.Cleanup()
|
||||
if err != nil {
|
||||
t.Errorf("cleanup failed: %v", err)
|
||||
}
|
||||
}()
|
||||
@@ -35,6 +36,7 @@ func TestClient(t *testing.T) {
|
||||
// Test PutObject
|
||||
testKey := "foo/bar.txt"
|
||||
testData := []byte("test data")
|
||||
|
||||
err = client.PutObject(ctx, testKey, bytes.NewReader(testData))
|
||||
if err != nil {
|
||||
t.Fatalf("failed to put object: %v", err)
|
||||
@@ -46,7 +48,8 @@ func TestClient(t *testing.T) {
|
||||
t.Fatalf("failed to get object: %v", err)
|
||||
}
|
||||
defer func() {
|
||||
if err := reader.Close(); err != nil {
|
||||
err := reader.Close()
|
||||
if err != nil {
|
||||
t.Errorf("failed to close reader: %v", err)
|
||||
}
|
||||
}()
|
||||
@@ -65,6 +68,7 @@ func TestClient(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to head object: %v", err)
|
||||
}
|
||||
|
||||
if !exists {
|
||||
t.Error("expected object to exist")
|
||||
}
|
||||
@@ -74,9 +78,11 @@ func TestClient(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to list objects: %v", err)
|
||||
}
|
||||
|
||||
if len(keys) != 1 {
|
||||
t.Errorf("expected 1 key, got %d", len(keys))
|
||||
}
|
||||
|
||||
if keys[0] != testKey {
|
||||
t.Errorf("unexpected key: got %s, want %s", keys[0], testKey)
|
||||
}
|
||||
@@ -92,6 +98,7 @@ func TestClient(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to head object after deletion: %v", err)
|
||||
}
|
||||
|
||||
if exists {
|
||||
t.Error("expected object to not exist after deletion")
|
||||
}
|
||||
|
||||
@@ -57,7 +57,8 @@ func NewTestServer(t *testing.T) *TestServer {
|
||||
|
||||
// Start server in background
|
||||
go func() {
|
||||
if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
|
||||
err := server.ListenAndServe()
|
||||
if err != nil && err != http.ErrServerClosed {
|
||||
t.Logf("test server error: %v", err)
|
||||
}
|
||||
}()
|
||||
@@ -77,7 +78,7 @@ func NewTestServer(t *testing.T) *TestServer {
|
||||
"",
|
||||
)),
|
||||
config.WithClientLogMode(aws.LogRetries|aws.LogRequestWithBody|aws.LogResponseWithBody),
|
||||
config.WithLogger(logging.LoggerFunc(func(classification logging.Classification, format string, v ...interface{}) {
|
||||
config.WithLogger(logging.LoggerFunc(func(classification logging.Classification, format string, v ...any) {
|
||||
// Capture logs to buffer instead of stdout
|
||||
fmt.Fprintf(logBuf, "SDK %s %s %s\n",
|
||||
time.Now().Format("2006/01/02 15:04:05"),
|
||||
@@ -125,7 +126,8 @@ func (ts *TestServer) Cleanup() error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
if err := ts.server.Shutdown(ctx); err != nil {
|
||||
err := ts.server.Shutdown(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -141,7 +143,8 @@ func (ts *TestServer) Client() *s3.Client {
|
||||
func TestBasicS3Operations(t *testing.T) {
|
||||
ts := NewTestServer(t)
|
||||
defer func() {
|
||||
if err := ts.Cleanup(); err != nil {
|
||||
err := ts.Cleanup()
|
||||
if err != nil {
|
||||
t.Errorf("cleanup failed: %v", err)
|
||||
}
|
||||
}()
|
||||
@@ -172,7 +175,8 @@ func TestBasicS3Operations(t *testing.T) {
|
||||
t.Fatalf("failed to get object: %v", err)
|
||||
}
|
||||
defer func() {
|
||||
if err := result.Body.Close(); err != nil {
|
||||
err := result.Body.Close()
|
||||
if err != nil {
|
||||
t.Errorf("failed to close body: %v", err)
|
||||
}
|
||||
}()
|
||||
@@ -192,7 +196,8 @@ func TestBasicS3Operations(t *testing.T) {
|
||||
func TestBlobOperations(t *testing.T) {
|
||||
ts := NewTestServer(t)
|
||||
defer func() {
|
||||
if err := ts.Cleanup(); err != nil {
|
||||
err := ts.Cleanup()
|
||||
if err != nil {
|
||||
t.Errorf("cleanup failed: %v", err)
|
||||
}
|
||||
}()
|
||||
@@ -255,7 +260,8 @@ func TestBlobOperations(t *testing.T) {
|
||||
func TestMetadataOperations(t *testing.T) {
|
||||
ts := NewTestServer(t)
|
||||
defer func() {
|
||||
if err := ts.Cleanup(); err != nil {
|
||||
err := ts.Cleanup()
|
||||
if err != nil {
|
||||
t.Errorf("cleanup failed: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
@@ -4,6 +4,8 @@ import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
@@ -30,6 +32,7 @@ func NewMockS3Client() *MockS3Client {
|
||||
|
||||
func (m *MockS3Client) PutBlob(ctx context.Context, hash string, data []byte) error {
|
||||
m.storage[hash] = data
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -38,11 +41,13 @@ func (m *MockS3Client) GetBlob(ctx context.Context, hash string) ([]byte, error)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("blob not found: %s", hash)
|
||||
}
|
||||
|
||||
return data, nil
|
||||
}
|
||||
|
||||
func (m *MockS3Client) BlobExists(ctx context.Context, hash string) (bool, error) {
|
||||
_, ok := m.storage[hash]
|
||||
|
||||
return ok, nil
|
||||
}
|
||||
|
||||
@@ -81,12 +86,15 @@ func TestBackupWithInMemoryFS(t *testing.T) {
|
||||
|
||||
// Initialize the database
|
||||
ctx := context.Background()
|
||||
|
||||
db, err := database.New(ctx, dbPath)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create database: %v", err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if err := db.Close(); err != nil {
|
||||
err := db.Close()
|
||||
if err != nil {
|
||||
t.Logf("Failed to close database: %v", err)
|
||||
}
|
||||
}()
|
||||
@@ -142,12 +150,14 @@ func TestBackupWithInMemoryFS(t *testing.T) {
|
||||
if !expectedFiles[file.Path.String()] {
|
||||
t.Errorf("Unexpected file in database: %s", file.Path)
|
||||
}
|
||||
|
||||
delete(expectedFiles, file.Path.String())
|
||||
|
||||
// Verify file metadata
|
||||
fsFile := testFS[file.Path.String()]
|
||||
if fsFile == nil {
|
||||
t.Errorf("File %s not found in test filesystem", file.Path)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -187,6 +197,7 @@ func TestBackupWithInMemoryFS(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to get blob hashes: %v", err)
|
||||
}
|
||||
|
||||
if len(blobHashes) == 0 {
|
||||
t.Error("Expected at least one blob to be created")
|
||||
}
|
||||
@@ -197,6 +208,7 @@ func TestBackupWithInMemoryFS(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Errorf("Failed to check blob %s: %v", blobHash, err)
|
||||
}
|
||||
|
||||
if !exists {
|
||||
t.Errorf("Blob %s not found in S3", blobHash)
|
||||
}
|
||||
@@ -229,12 +241,15 @@ func TestBackupDeduplication(t *testing.T) {
|
||||
|
||||
// Initialize the database
|
||||
ctx := context.Background()
|
||||
|
||||
db, err := database.New(ctx, dbPath)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create database: %v", err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if err := db.Close(); err != nil {
|
||||
err := db.Close()
|
||||
if err != nil {
|
||||
t.Logf("Failed to close database: %v", err)
|
||||
}
|
||||
}()
|
||||
@@ -348,6 +363,7 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
|
||||
UID: 1000, // Default UID for test
|
||||
GID: 1000, // Default GID for test
|
||||
}
|
||||
|
||||
err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
|
||||
return b.repos.Files.Create(ctx, tx, file)
|
||||
})
|
||||
@@ -364,7 +380,8 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
if err := f.Close(); err != nil {
|
||||
err := f.Close()
|
||||
if err != nil {
|
||||
// Log but don't fail since we're already in an error path potentially
|
||||
fmt.Fprintf(os.Stderr, "Failed to close file: %v\n", err)
|
||||
}
|
||||
@@ -376,9 +393,10 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
|
||||
|
||||
for {
|
||||
n, err := f.Read(buffer)
|
||||
if err != nil && err != io.EOF {
|
||||
if err != nil && !errors.Is(err, io.EOF) {
|
||||
return err
|
||||
}
|
||||
|
||||
if n == 0 {
|
||||
break
|
||||
}
|
||||
@@ -395,11 +413,13 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
|
||||
ChunkHash: types.ChunkHash(chunkHash),
|
||||
Size: int64(n),
|
||||
}
|
||||
|
||||
return b.repos.Chunks.Create(ctx, tx, chunk)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
processedChunks[chunkHash] = true
|
||||
}
|
||||
|
||||
@@ -410,6 +430,7 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
|
||||
Idx: chunkIndex,
|
||||
ChunkHash: types.ChunkHash(chunkHash),
|
||||
}
|
||||
|
||||
return b.repos.FileChunks.Create(ctx, tx, fileChunk)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -424,6 +445,7 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
|
||||
FileOffset: int64(chunkIndex * defaultChunkSize),
|
||||
Length: int64(n),
|
||||
}
|
||||
|
||||
return b.repos.ChunkFiles.Create(ctx, tx, chunkFile)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -435,7 +457,6 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
@@ -464,12 +485,14 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
|
||||
|
||||
// Create blob entry in a short transaction
|
||||
blobID := types.NewBlobID()
|
||||
|
||||
err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
|
||||
blob := &database.Blob{
|
||||
ID: blobID,
|
||||
Hash: types.BlobHash(blobHash),
|
||||
CreatedTS: time.Now(),
|
||||
}
|
||||
|
||||
return b.repos.Blobs.Create(ctx, tx, blob)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -487,6 +510,7 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
|
||||
Offset: 0,
|
||||
Length: chunk.Size,
|
||||
}
|
||||
|
||||
return b.repos.BlobChunks.Create(ctx, tx, blobChunk)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -506,7 +530,6 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
|
||||
err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
|
||||
return b.repos.Snapshots.UpdateCounts(ctx, tx, snapshotID, fileCount, chunkCount, blobCount, totalSize, blobSize)
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
@@ -517,16 +540,18 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
|
||||
func calculateHash(data []byte) string {
|
||||
h := sha256.New()
|
||||
h.Write(data)
|
||||
return fmt.Sprintf("%x", h.Sum(nil))
|
||||
|
||||
return hex.EncodeToString(h.Sum(nil))
|
||||
}
|
||||
|
||||
func generateLargeFileContent(size int) []byte {
|
||||
data := make([]byte, size)
|
||||
// Fill with pattern that changes every chunk to avoid deduplication
|
||||
for i := 0; i < size; i++ {
|
||||
for i := range size {
|
||||
chunkNum := i / defaultChunkSize
|
||||
data[i] = byte((i + chunkNum) % 256)
|
||||
}
|
||||
|
||||
return data
|
||||
}
|
||||
|
||||
|
||||
@@ -63,6 +63,7 @@ func setupExcludeTestFS(t *testing.T) afero.Fs {
|
||||
}
|
||||
|
||||
testTime := time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC)
|
||||
|
||||
for path, content := range files {
|
||||
dir := filepath.Dir(path)
|
||||
err := fs.MkdirAll(dir, 0755)
|
||||
@@ -107,6 +108,7 @@ func createTestScanner(t *testing.T, fs afero.Fs, excludePatterns []string) (*sn
|
||||
|
||||
func createSnapshotRecord(t *testing.T, ctx context.Context, repos *database.Repositories, snapshotID string) {
|
||||
t.Helper()
|
||||
|
||||
err := repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
|
||||
snap := &database.Snapshot{
|
||||
ID: types.SnapshotID(snapshotID),
|
||||
@@ -121,6 +123,7 @@ func createSnapshotRecord(t *testing.T, ctx context.Context, repos *database.Rep
|
||||
BlobSize: 0,
|
||||
CompressionRatio: 1.0,
|
||||
}
|
||||
|
||||
return repos.Snapshots.Create(ctx, tx, snap)
|
||||
})
|
||||
require.NoError(t, err)
|
||||
@@ -128,8 +131,10 @@ func createSnapshotRecord(t *testing.T, ctx context.Context, repos *database.Rep
|
||||
|
||||
func TestExcludePatterns_ExcludeGitDirectory(t *testing.T) {
|
||||
fs := setupExcludeTestFS(t)
|
||||
|
||||
scanner, repos, cleanup := createTestScanner(t, fs, []string{".git"})
|
||||
defer cleanup()
|
||||
|
||||
require.NotNil(t, scanner)
|
||||
|
||||
ctx := context.Background()
|
||||
@@ -148,8 +153,10 @@ func TestExcludePatterns_ExcludeGitDirectory(t *testing.T) {
|
||||
|
||||
func TestExcludePatterns_ExcludeByExtension(t *testing.T) {
|
||||
fs := setupExcludeTestFS(t)
|
||||
|
||||
scanner, repos, cleanup := createTestScanner(t, fs, []string{"*.log"})
|
||||
defer cleanup()
|
||||
|
||||
require.NotNil(t, scanner)
|
||||
|
||||
ctx := context.Background()
|
||||
@@ -165,8 +172,10 @@ func TestExcludePatterns_ExcludeByExtension(t *testing.T) {
|
||||
|
||||
func TestExcludePatterns_ExcludeNodeModules(t *testing.T) {
|
||||
fs := setupExcludeTestFS(t)
|
||||
|
||||
scanner, repos, cleanup := createTestScanner(t, fs, []string{"node_modules"})
|
||||
defer cleanup()
|
||||
|
||||
require.NotNil(t, scanner)
|
||||
|
||||
ctx := context.Background()
|
||||
@@ -182,8 +191,10 @@ func TestExcludePatterns_ExcludeNodeModules(t *testing.T) {
|
||||
|
||||
func TestExcludePatterns_MultiplePatterns(t *testing.T) {
|
||||
fs := setupExcludeTestFS(t)
|
||||
|
||||
scanner, repos, cleanup := createTestScanner(t, fs, []string{".git", "node_modules", "*.log", ".DS_Store", "thumbs.db", "cache", "build"})
|
||||
defer cleanup()
|
||||
|
||||
require.NotNil(t, scanner)
|
||||
|
||||
ctx := context.Background()
|
||||
@@ -199,8 +210,10 @@ func TestExcludePatterns_MultiplePatterns(t *testing.T) {
|
||||
|
||||
func TestExcludePatterns_NoExclusions(t *testing.T) {
|
||||
fs := setupExcludeTestFS(t)
|
||||
|
||||
scanner, repos, cleanup := createTestScanner(t, fs, []string{})
|
||||
defer cleanup()
|
||||
|
||||
require.NotNil(t, scanner)
|
||||
|
||||
ctx := context.Background()
|
||||
@@ -215,8 +228,10 @@ func TestExcludePatterns_NoExclusions(t *testing.T) {
|
||||
|
||||
func TestExcludePatterns_ExcludeHiddenFiles(t *testing.T) {
|
||||
fs := setupExcludeTestFS(t)
|
||||
|
||||
scanner, repos, cleanup := createTestScanner(t, fs, []string{".*"})
|
||||
defer cleanup()
|
||||
|
||||
require.NotNil(t, scanner)
|
||||
|
||||
ctx := context.Background()
|
||||
@@ -232,8 +247,10 @@ func TestExcludePatterns_ExcludeHiddenFiles(t *testing.T) {
|
||||
|
||||
func TestExcludePatterns_DoubleStarGlob(t *testing.T) {
|
||||
fs := setupExcludeTestFS(t)
|
||||
|
||||
scanner, repos, cleanup := createTestScanner(t, fs, []string{"**/*.pack"})
|
||||
defer cleanup()
|
||||
|
||||
require.NotNil(t, scanner)
|
||||
|
||||
ctx := context.Background()
|
||||
@@ -249,8 +266,10 @@ func TestExcludePatterns_DoubleStarGlob(t *testing.T) {
|
||||
|
||||
func TestExcludePatterns_ExactFileName(t *testing.T) {
|
||||
fs := setupExcludeTestFS(t)
|
||||
|
||||
scanner, repos, cleanup := createTestScanner(t, fs, []string{"thumbs.db", ".DS_Store"})
|
||||
defer cleanup()
|
||||
|
||||
require.NotNil(t, scanner)
|
||||
|
||||
ctx := context.Background()
|
||||
@@ -267,8 +286,10 @@ func TestExcludePatterns_ExactFileName(t *testing.T) {
|
||||
func TestExcludePatterns_CaseSensitive(t *testing.T) {
|
||||
// Pattern matching should be case-sensitive
|
||||
fs := setupExcludeTestFS(t)
|
||||
|
||||
scanner, repos, cleanup := createTestScanner(t, fs, []string{"THUMBS.DB"})
|
||||
defer cleanup()
|
||||
|
||||
require.NotNil(t, scanner)
|
||||
|
||||
ctx := context.Background()
|
||||
@@ -287,6 +308,7 @@ func TestExcludePatterns_DirectoryWithTrailingSlash(t *testing.T) {
|
||||
// Some users might add trailing slashes to directory patterns
|
||||
scanner, repos, cleanup := createTestScanner(t, fs, []string{"cache/", "build/"})
|
||||
defer cleanup()
|
||||
|
||||
require.NotNil(t, scanner)
|
||||
|
||||
ctx := context.Background()
|
||||
@@ -305,6 +327,7 @@ func TestExcludePatterns_PatternInSubdirectory(t *testing.T) {
|
||||
// Exclude .hidden file specifically in src directory
|
||||
scanner, repos, cleanup := createTestScanner(t, fs, []string{"src/.hidden"})
|
||||
defer cleanup()
|
||||
|
||||
require.NotNil(t, scanner)
|
||||
|
||||
ctx := context.Background()
|
||||
@@ -343,6 +366,7 @@ func setupAnchoredTestFS(t *testing.T) afero.Fs {
|
||||
}
|
||||
|
||||
testTime := time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC)
|
||||
|
||||
for path, content := range files {
|
||||
dir := filepath.Dir(path)
|
||||
err := fs.MkdirAll(dir, 0755)
|
||||
@@ -359,8 +383,10 @@ func setupAnchoredTestFS(t *testing.T) afero.Fs {
|
||||
func TestExcludePatterns_AnchoredPattern(t *testing.T) {
|
||||
// Pattern starting with / should only match from root of source dir
|
||||
fs := setupAnchoredTestFS(t)
|
||||
|
||||
scanner, repos, cleanup := createTestScanner(t, fs, []string{"/projectname"})
|
||||
defer cleanup()
|
||||
|
||||
require.NotNil(t, scanner)
|
||||
|
||||
ctx := context.Background()
|
||||
@@ -378,8 +404,10 @@ func TestExcludePatterns_AnchoredPattern(t *testing.T) {
|
||||
func TestExcludePatterns_UnanchoredPattern(t *testing.T) {
|
||||
// Pattern without leading / should match anywhere in path
|
||||
fs := setupAnchoredTestFS(t)
|
||||
|
||||
scanner, repos, cleanup := createTestScanner(t, fs, []string{"projectname"})
|
||||
defer cleanup()
|
||||
|
||||
require.NotNil(t, scanner)
|
||||
|
||||
ctx := context.Background()
|
||||
@@ -398,8 +426,10 @@ func TestExcludePatterns_UnanchoredPattern(t *testing.T) {
|
||||
func TestExcludePatterns_AnchoredPatternWithGlob(t *testing.T) {
|
||||
// Anchored pattern with glob
|
||||
fs := setupAnchoredTestFS(t)
|
||||
|
||||
scanner, repos, cleanup := createTestScanner(t, fs, []string{"/src/*.go"})
|
||||
defer cleanup()
|
||||
|
||||
require.NotNil(t, scanner)
|
||||
|
||||
ctx := context.Background()
|
||||
@@ -416,8 +446,10 @@ func TestExcludePatterns_AnchoredPatternWithGlob(t *testing.T) {
|
||||
func TestExcludePatterns_AnchoredPatternFile(t *testing.T) {
|
||||
// Anchored pattern for exact file at root
|
||||
fs := setupAnchoredTestFS(t)
|
||||
|
||||
scanner, repos, cleanup := createTestScanner(t, fs, []string{"/file.txt"})
|
||||
defer cleanup()
|
||||
|
||||
require.NotNil(t, scanner)
|
||||
|
||||
ctx := context.Background()
|
||||
@@ -435,8 +467,10 @@ func TestExcludePatterns_AnchoredPatternFile(t *testing.T) {
|
||||
func TestExcludePatterns_UnanchoredPatternFile(t *testing.T) {
|
||||
// Unanchored pattern for file should match anywhere
|
||||
fs := setupAnchoredTestFS(t)
|
||||
|
||||
scanner, repos, cleanup := createTestScanner(t, fs, []string{"file.txt"})
|
||||
defer cleanup()
|
||||
|
||||
require.NotNil(t, scanner)
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
@@ -30,9 +30,11 @@ func TestFileContentChange(t *testing.T) {
|
||||
|
||||
// Create test database
|
||||
db, err := database.NewTestDB()
|
||||
|
||||
require.NoError(t, err)
|
||||
defer func() {
|
||||
if err := db.Close(); err != nil {
|
||||
err := db.Close()
|
||||
if err != nil {
|
||||
t.Errorf("failed to close database: %v", err)
|
||||
}
|
||||
}()
|
||||
@@ -59,6 +61,7 @@ func TestFileContentChange(t *testing.T) {
|
||||
VaultikVersion: "test",
|
||||
StartedAt: time.Now(),
|
||||
}
|
||||
|
||||
return repos.Snapshots.Create(ctx, tx, snapshot)
|
||||
})
|
||||
require.NoError(t, err)
|
||||
@@ -81,6 +84,7 @@ func TestFileContentChange(t *testing.T) {
|
||||
|
||||
// Modify the file
|
||||
time.Sleep(10 * time.Millisecond) // Ensure mtime changes
|
||||
|
||||
err = afero.WriteFile(fs, "/test.txt", []byte("Modified content with different data"), 0644)
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -93,6 +97,7 @@ func TestFileContentChange(t *testing.T) {
|
||||
VaultikVersion: "test",
|
||||
StartedAt: time.Now(),
|
||||
}
|
||||
|
||||
return repos.Snapshots.Create(ctx, tx, snapshot)
|
||||
})
|
||||
require.NoError(t, err)
|
||||
@@ -130,6 +135,7 @@ func TestFileContentChange(t *testing.T) {
|
||||
// Verify that chunk_files for old chunk no longer references this file
|
||||
oldChunkFiles, err := repos.ChunkFiles.GetByChunkHash(ctx, oldChunkHash)
|
||||
require.NoError(t, err)
|
||||
|
||||
for _, cf := range oldChunkFiles {
|
||||
file, err := repos.Files.GetByID(ctx, cf.FileID)
|
||||
require.NoError(t, err)
|
||||
@@ -159,9 +165,11 @@ func TestMultipleFileChanges(t *testing.T) {
|
||||
|
||||
// Create test database
|
||||
db, err := database.NewTestDB()
|
||||
|
||||
require.NoError(t, err)
|
||||
defer func() {
|
||||
if err := db.Close(); err != nil {
|
||||
err := db.Close()
|
||||
if err != nil {
|
||||
t.Errorf("failed to close database: %v", err)
|
||||
}
|
||||
}()
|
||||
@@ -188,6 +196,7 @@ func TestMultipleFileChanges(t *testing.T) {
|
||||
VaultikVersion: "test",
|
||||
StartedAt: time.Now(),
|
||||
}
|
||||
|
||||
return repos.Snapshots.Create(ctx, tx, snapshot)
|
||||
})
|
||||
require.NoError(t, err)
|
||||
@@ -200,6 +209,7 @@ func TestMultipleFileChanges(t *testing.T) {
|
||||
|
||||
// Modify two files
|
||||
time.Sleep(10 * time.Millisecond) // Ensure mtime changes
|
||||
|
||||
err = afero.WriteFile(fs, "/file1.txt", []byte("Modified content 1"), 0644)
|
||||
require.NoError(t, err)
|
||||
err = afero.WriteFile(fs, "/file3.txt", []byte("Modified content 3"), 0644)
|
||||
@@ -214,6 +224,7 @@ func TestMultipleFileChanges(t *testing.T) {
|
||||
VaultikVersion: "test",
|
||||
StartedAt: time.Now(),
|
||||
}
|
||||
|
||||
return repos.Snapshots.Create(ctx, tx, snapshot)
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -52,6 +52,7 @@ func EncodeManifest(manifest *Manifest, compressionLevel int) ([]byte, error) {
|
||||
|
||||
// Compress using zstd
|
||||
var compressedBuf bytes.Buffer
|
||||
|
||||
writer, err := zstd.NewWriter(&compressedBuf, zstd.WithEncoderLevel(zstd.EncoderLevelFromZstd(compressionLevel)))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("creating zstd writer: %w", err)
|
||||
@@ -59,6 +60,7 @@ func EncodeManifest(manifest *Manifest, compressionLevel int) ([]byte, error) {
|
||||
|
||||
if _, err := writer.Write(jsonData); err != nil {
|
||||
_ = writer.Close()
|
||||
|
||||
return nil, fmt.Errorf("writing compressed data: %w", err)
|
||||
}
|
||||
|
||||
|
||||
@@ -12,7 +12,9 @@ import (
|
||||
func TestWrapPermissionError(t *testing.T) {
|
||||
// Non-permission errors pass through unchanged.
|
||||
plain := errors.New("disk on fire")
|
||||
if got := wrapPermissionError("/some/path", plain); got != plain {
|
||||
|
||||
got := wrapPermissionError("/some/path", plain)
|
||||
if !errors.Is(got, plain) {
|
||||
t.Errorf("non-permission error should pass through, got %v", got)
|
||||
}
|
||||
|
||||
@@ -23,6 +25,7 @@ func TestWrapPermissionError(t *testing.T) {
|
||||
if !errors.Is(wrapped, os.ErrPermission) {
|
||||
t.Error("wrapped error should still match os.ErrPermission")
|
||||
}
|
||||
|
||||
if !strings.Contains(wrapped.Error(), "/Users/u/Library/Calendars") {
|
||||
t.Error("wrapped error should name the offending path")
|
||||
}
|
||||
@@ -31,6 +34,7 @@ func TestWrapPermissionError(t *testing.T) {
|
||||
if !strings.Contains(wrapped.Error(), "Full Disk Access") {
|
||||
t.Errorf("macOS permission error should mention Full Disk Access:\n%s", wrapped.Error())
|
||||
}
|
||||
|
||||
if !strings.Contains(wrapped.Error(), "System Settings") {
|
||||
t.Errorf("macOS permission error should point at System Settings:\n%s", wrapped.Error())
|
||||
}
|
||||
|
||||
@@ -153,6 +153,7 @@ func (pr *ProgressReporter) printSummaryStatus() {
|
||||
if uploadInfo, ok := pr.stats.CurrentUpload.Load().(*UploadInfo); ok && uploadInfo != nil {
|
||||
// Show upload progress instead
|
||||
pr.printUploadProgress(uploadInfo)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
@@ -175,16 +176,18 @@ func (pr *ProgressReporter) printSummaryStatus() {
|
||||
|
||||
// Calculate ETA if we have total size and are processing
|
||||
etaStr := ""
|
||||
|
||||
if totalSize > 0 && bytesProcessed > 0 {
|
||||
processStart, ok := pr.stats.ProcessStartTime.Load().(time.Time)
|
||||
if ok && !processStart.IsZero() {
|
||||
processElapsed := time.Since(processStart)
|
||||
|
||||
rate := float64(bytesProcessed) / processElapsed.Seconds()
|
||||
if rate > 0 {
|
||||
remainingBytes := totalSize - bytesProcessed
|
||||
remainingSeconds := float64(remainingBytes) / rate
|
||||
eta := time.Duration(remainingSeconds * float64(time.Second))
|
||||
etaStr = fmt.Sprintf(" | ETA: %s", formatDuration(eta))
|
||||
etaStr = " | ETA: " + formatDuration(eta)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -206,7 +209,7 @@ func (pr *ProgressReporter) printSummaryStatus() {
|
||||
)
|
||||
|
||||
if currentFile != "" {
|
||||
status += fmt.Sprintf(" | Current: %s", truncatePath(currentFile, 40))
|
||||
status += " | Current: " + truncatePath(currentFile, 40)
|
||||
}
|
||||
|
||||
log.Info(status)
|
||||
@@ -242,6 +245,7 @@ func (pr *ProgressReporter) printDetailedStatus() {
|
||||
processStart, ok := pr.stats.ProcessStartTime.Load().(time.Time)
|
||||
if ok && !processStart.IsZero() {
|
||||
processElapsed := time.Since(processStart)
|
||||
|
||||
processRate := float64(bytesProcessed) / processElapsed.Seconds()
|
||||
if processRate > 0 {
|
||||
remainingBytes := totalSize - bytesProcessed
|
||||
@@ -276,9 +280,11 @@ func (pr *ProgressReporter) printDetailedStatus() {
|
||||
log.Info("Total uploaded to remote",
|
||||
"uploaded", humanize.Bytes(uint64(bytesUploaded)),
|
||||
"compression_ratio", formatRatio(bytesUploaded, bytesScanned))
|
||||
|
||||
if currentFile != "" {
|
||||
log.Info("Current file", "path", currentFile)
|
||||
}
|
||||
|
||||
log.Notice("=============================")
|
||||
}
|
||||
|
||||
@@ -288,12 +294,15 @@ func formatDuration(d time.Duration) string {
|
||||
if d < 0 {
|
||||
return "unknown"
|
||||
}
|
||||
|
||||
if d < time.Minute {
|
||||
return fmt.Sprintf("%ds", int(d.Seconds()))
|
||||
}
|
||||
|
||||
if d < time.Hour {
|
||||
return fmt.Sprintf("%dm%ds", int(d.Minutes()), int(d.Seconds())%60)
|
||||
}
|
||||
|
||||
return fmt.Sprintf("%dh%dm", int(d.Hours()), int(d.Minutes())%60)
|
||||
}
|
||||
|
||||
@@ -301,6 +310,7 @@ func formatPercent(numerator, denominator int64) string {
|
||||
if denominator == 0 {
|
||||
return "0.0%"
|
||||
}
|
||||
|
||||
return fmt.Sprintf("%.1f%%", float64(numerator)/float64(denominator)*100)
|
||||
}
|
||||
|
||||
@@ -308,7 +318,9 @@ func formatRatio(compressed, uncompressed int64) string {
|
||||
if uncompressed == 0 {
|
||||
return "1.00"
|
||||
}
|
||||
|
||||
ratio := float64(compressed) / float64(uncompressed)
|
||||
|
||||
return fmt.Sprintf("%.2f", ratio)
|
||||
}
|
||||
|
||||
@@ -353,6 +365,7 @@ func (pr *ProgressReporter) ReportUploadComplete(blobHash string, size int64, du
|
||||
if duration < time.Millisecond {
|
||||
duration = time.Millisecond
|
||||
}
|
||||
|
||||
bytesPerSec := float64(size) / duration.Seconds()
|
||||
bitsPerSec := bytesPerSec * 8
|
||||
|
||||
@@ -398,6 +411,7 @@ func (pr *ProgressReporter) ReportUploadProgress(blobHash string, bytesUploaded,
|
||||
|
||||
// Calculate ETA based on current speed
|
||||
etaStr := "unknown"
|
||||
|
||||
if instantSpeed > 0 && bytesUploaded < totalSize {
|
||||
remainingBytes := totalSize - bytesUploaded
|
||||
remainingSeconds := float64(remainingBytes) / instantSpeed
|
||||
|
||||
@@ -36,5 +36,6 @@ const remoteKeyPrefix = "vaultik|"
|
||||
func RemoteSnapshotKey(snapshotID string) string {
|
||||
first := sha256.Sum256([]byte(remoteKeyPrefix + snapshotID))
|
||||
second := sha256.Sum256(first[:])
|
||||
|
||||
return hex.EncodeToString(second[:])
|
||||
}
|
||||
|
||||
@@ -119,6 +119,7 @@ func NewScanner(cfg ScannerConfig) *Scanner {
|
||||
// Create encryptor (required for blob packing)
|
||||
if len(cfg.AgeRecipients) == 0 {
|
||||
log.Error("No age recipients configured - encryption is required")
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -130,9 +131,11 @@ func NewScanner(cfg ScannerConfig) *Scanner {
|
||||
Repositories: cfg.Repositories,
|
||||
Fs: cfg.FS,
|
||||
}
|
||||
|
||||
packer, err := blob.NewPacker(packerCfg)
|
||||
if err != nil {
|
||||
log.Error("Failed to create packer", "error", err)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -199,11 +202,14 @@ func (s *Scanner) Scan(ctx context.Context, path string, snapshotID string) (*Sc
|
||||
// Phase 1: Scan directory, collect files to process, and track existing files
|
||||
// (builds existingFiles map during walk to avoid double traversal)
|
||||
log.Info("Phase 1/3: Scanning directory structure")
|
||||
|
||||
existingFiles := make(map[string]struct{})
|
||||
|
||||
scanResult, err := s.scanPhase(ctx, path, result, existingFiles, knownFiles)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scan phase failed: %w", err)
|
||||
}
|
||||
|
||||
filesToProcess := scanResult.FilesToProcess
|
||||
|
||||
// Phase 1b: Detect deleted files by comparing DB against scanned files
|
||||
@@ -213,8 +219,10 @@ func (s *Scanner) Scan(ctx context.Context, path string, snapshotID string) (*Sc
|
||||
|
||||
// Phase 1c: Associate unchanged files with this snapshot (no new records needed)
|
||||
if len(scanResult.UnchangedFileIDs) > 0 {
|
||||
s.ui.Begin("Associating %s unchanged files with the snapshot.", s.ui.Count(len(scanResult.UnchangedFileIDs)))
|
||||
if err := s.batchAddFilesToSnapshot(ctx, scanResult.UnchangedFileIDs); err != nil {
|
||||
s.ui.Beginf("Associating %s unchanged files with the snapshot.", s.ui.Count(len(scanResult.UnchangedFileIDs)))
|
||||
|
||||
err := s.batchAddFilesToSnapshot(ctx, scanResult.UnchangedFileIDs)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("associating unchanged files: %w", err)
|
||||
}
|
||||
}
|
||||
@@ -224,13 +232,15 @@ func (s *Scanner) Scan(ctx context.Context, path string, snapshotID string) (*Sc
|
||||
|
||||
// Phase 2: Process files and create chunks
|
||||
if len(filesToProcess) > 0 {
|
||||
s.ui.Begin("Backing up %s snapshot source files (chunking, compressing, encrypting, uploading).", s.ui.Count(len(filesToProcess)))
|
||||
s.ui.Beginf("Backing up %s snapshot source files (chunking, compressing, encrypting, uploading).", s.ui.Count(len(filesToProcess)))
|
||||
log.Info("Phase 2/3: Creating snapshot (chunking, compressing, encrypting, and uploading blobs)")
|
||||
if err := s.processPhase(ctx, filesToProcess, result); err != nil {
|
||||
|
||||
err := s.processPhase(ctx, filesToProcess, result)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("process phase failed: %w", err)
|
||||
}
|
||||
} else {
|
||||
s.ui.Info("Snapshot file backup skipped: no changed files (creating metadata-only snapshot).")
|
||||
s.ui.Infof("Snapshot file backup skipped: no changed files (creating metadata-only snapshot).")
|
||||
log.Info("Phase 2/3: Skipping (no files need processing, metadata-only snapshot)")
|
||||
}
|
||||
|
||||
@@ -243,18 +253,22 @@ func (s *Scanner) Scan(ctx context.Context, path string, snapshotID string) (*Sc
|
||||
// loadDatabaseState loads known files and chunks from the database into memory for fast lookup
|
||||
// This avoids per-file and per-chunk database queries during the scan and process phases
|
||||
func (s *Scanner) loadDatabaseState(ctx context.Context, path string) (map[string]*database.File, error) {
|
||||
s.ui.Begin("Loading known files from local index database.")
|
||||
s.ui.Beginf("Loading known files from local index database.")
|
||||
|
||||
knownFiles, err := s.loadKnownFiles(ctx, path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("loading known files: %w", err)
|
||||
}
|
||||
s.ui.Complete("Loaded %s known files from local index database.", s.ui.Count(len(knownFiles)))
|
||||
|
||||
s.ui.Begin("Loading known chunks from local index database.")
|
||||
s.ui.Completef("Loaded %s known files from local index database.", s.ui.Count(len(knownFiles)))
|
||||
|
||||
s.ui.Beginf("Loading known chunks from local index database.")
|
||||
|
||||
if err := s.loadKnownChunks(ctx); err != nil {
|
||||
return nil, fmt.Errorf("loading known chunks: %w", err)
|
||||
}
|
||||
s.ui.Complete("Loaded %s known chunks from local index database.", s.ui.Count(len(s.knownChunks)))
|
||||
|
||||
s.ui.Completef("Loaded %s known chunks from local index database.", s.ui.Count(len(s.knownChunks)))
|
||||
|
||||
return knownFiles, nil
|
||||
}
|
||||
@@ -288,7 +302,8 @@ func (s *Scanner) summarizeScanPhase(result *ScanResult, filesToProcess []*FileT
|
||||
s.ui.Count(result.FilesDeleted),
|
||||
s.ui.Size(result.BytesDeleted))
|
||||
}
|
||||
s.ui.Complete("%s.", msg)
|
||||
|
||||
s.ui.Completef("%s.", msg)
|
||||
}
|
||||
|
||||
// finalizeScanResult populates final blob statistics in the scan result
|
||||
@@ -337,6 +352,7 @@ func (s *Scanner) loadKnownChunks(ctx context.Context) error {
|
||||
}
|
||||
|
||||
s.knownChunksMu.Lock()
|
||||
|
||||
s.knownChunks = make(map[string]struct{}, len(chunks))
|
||||
for _, c := range chunks {
|
||||
s.knownChunks[c.ChunkHash.String()] = struct{}{}
|
||||
@@ -351,6 +367,7 @@ func (s *Scanner) chunkExists(hash string) bool {
|
||||
s.knownChunksMu.RLock()
|
||||
_, exists := s.knownChunks[hash]
|
||||
s.knownChunksMu.RUnlock()
|
||||
|
||||
return exists
|
||||
}
|
||||
|
||||
@@ -371,7 +388,9 @@ func (s *Scanner) addPendingChunkHash(hash string) {
|
||||
// removePendingChunkHashes removes committed chunk hashes from the pending set
|
||||
func (s *Scanner) removePendingChunkHashes(hashes []string) {
|
||||
log.Debug("removePendingChunkHashes: starting", "count", len(hashes))
|
||||
|
||||
start := time.Now()
|
||||
|
||||
s.pendingChunkHashesMu.Lock()
|
||||
for _, hash := range hashes {
|
||||
delete(s.pendingChunkHashes, hash)
|
||||
@@ -385,6 +404,7 @@ func (s *Scanner) isChunkPending(hash string) bool {
|
||||
s.pendingChunkHashesMu.Lock()
|
||||
_, pending := s.pendingChunkHashes[hash]
|
||||
s.pendingChunkHashesMu.Unlock()
|
||||
|
||||
return pending
|
||||
}
|
||||
|
||||
@@ -411,37 +431,45 @@ func (s *Scanner) flushPendingFiles(ctx context.Context) error {
|
||||
return s.repos.WithTx(ctx, func(txCtx context.Context, tx *sql.Tx) error {
|
||||
for _, data := range files {
|
||||
// Create or update the file record
|
||||
if err := s.repos.Files.Create(txCtx, tx, data.file); err != nil {
|
||||
err := s.repos.Files.Create(txCtx, tx, data.file)
|
||||
if err != nil {
|
||||
return fmt.Errorf("creating file record: %w", err)
|
||||
}
|
||||
|
||||
// Delete any existing file_chunks and chunk_files for this file
|
||||
if err := s.repos.FileChunks.DeleteByFileID(txCtx, tx, data.file.ID); err != nil {
|
||||
err = s.repos.FileChunks.DeleteByFileID(txCtx, tx, data.file.ID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("deleting old file chunks: %w", err)
|
||||
}
|
||||
if err := s.repos.ChunkFiles.DeleteByFileID(txCtx, tx, data.file.ID); err != nil {
|
||||
|
||||
err = s.repos.ChunkFiles.DeleteByFileID(txCtx, tx, data.file.ID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("deleting old chunk files: %w", err)
|
||||
}
|
||||
|
||||
// Create file-chunk mappings
|
||||
for i := range data.fileChunks {
|
||||
if err := s.repos.FileChunks.Create(txCtx, tx, &data.fileChunks[i]); err != nil {
|
||||
err := s.repos.FileChunks.Create(txCtx, tx, &data.fileChunks[i])
|
||||
if err != nil {
|
||||
return fmt.Errorf("creating file chunk: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Create chunk-file mappings
|
||||
for i := range data.chunkFiles {
|
||||
if err := s.repos.ChunkFiles.Create(txCtx, tx, &data.chunkFiles[i]); err != nil {
|
||||
err := s.repos.ChunkFiles.Create(txCtx, tx, &data.chunkFiles[i])
|
||||
if err != nil {
|
||||
return fmt.Errorf("creating chunk file: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Add file to snapshot
|
||||
if err := s.repos.Snapshots.AddFileByID(txCtx, tx, s.snapshotID, data.file.ID); err != nil {
|
||||
err = s.repos.Snapshots.AddFileByID(txCtx, tx, s.snapshotID, data.file.ID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("adding file to snapshot: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
@@ -455,6 +483,7 @@ func (s *Scanner) flushAllPending(ctx context.Context) error {
|
||||
// Files with pending chunks are kept in the queue for later flushing
|
||||
func (s *Scanner) flushCompletedPendingFiles(ctx context.Context) error {
|
||||
flushStart := time.Now()
|
||||
|
||||
log.Debug("flushCompletedPendingFiles: starting")
|
||||
|
||||
// Partition pending files into those ready to flush and those still waiting
|
||||
@@ -462,6 +491,7 @@ func (s *Scanner) flushCompletedPendingFiles(ctx context.Context) error {
|
||||
|
||||
if len(canFlush) == 0 {
|
||||
log.Debug("flushCompletedPendingFiles: nothing to flush")
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -474,10 +504,13 @@ func (s *Scanner) flushCompletedPendingFiles(ctx context.Context) error {
|
||||
|
||||
// Execute the batch flush in a single transaction
|
||||
log.Debug("flushCompletedPendingFiles: starting transaction")
|
||||
|
||||
txStart := time.Now()
|
||||
err := s.executeBatchFileFlush(ctx, allFiles, allFileIDs, allFileChunks, allChunkFiles)
|
||||
|
||||
log.Debug("flushCompletedPendingFiles: transaction done", "duration", time.Since(txStart))
|
||||
log.Debug("flushCompletedPendingFiles: total duration", "duration", time.Since(flushStart))
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -492,21 +525,27 @@ func (s *Scanner) partitionPendingByChunkStatus() (canFlush []pendingFileData, s
|
||||
var stillPending []pendingFileData
|
||||
|
||||
log.Debug("flushCompletedPendingFiles: checking which files can flush")
|
||||
|
||||
checkStart := time.Now()
|
||||
|
||||
for _, data := range s.pendingFiles {
|
||||
allChunksCommitted := true
|
||||
|
||||
for _, fc := range data.fileChunks {
|
||||
if s.isChunkPending(fc.ChunkHash.String()) {
|
||||
allChunksCommitted = false
|
||||
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if allChunksCommitted {
|
||||
canFlush = append(canFlush, data)
|
||||
} else {
|
||||
stillPending = append(stillPending, data)
|
||||
}
|
||||
}
|
||||
|
||||
log.Debug("flushCompletedPendingFiles: check done", "duration", time.Since(checkStart), "can_flush", len(canFlush), "still_pending", len(stillPending))
|
||||
|
||||
s.pendingFiles = stillPending
|
||||
@@ -520,12 +559,15 @@ func (s *Scanner) partitionPendingByChunkStatus() (canFlush []pendingFileData, s
|
||||
// mappings from the given pending file data for efficient batch database operations
|
||||
func (s *Scanner) collectBatchFlushData(canFlush []pendingFileData) ([]*database.File, []types.FileID, []database.FileChunk, []database.ChunkFile) {
|
||||
log.Debug("flushCompletedPendingFiles: collecting data for batch ops")
|
||||
|
||||
collectStart := time.Now()
|
||||
|
||||
var allFileChunks []database.FileChunk
|
||||
var allChunkFiles []database.ChunkFile
|
||||
var allFileIDs []types.FileID
|
||||
var allFiles []*database.File
|
||||
var (
|
||||
allFileChunks []database.FileChunk
|
||||
allChunkFiles []database.ChunkFile
|
||||
allFileIDs []types.FileID
|
||||
allFiles []*database.File
|
||||
)
|
||||
|
||||
for _, data := range canFlush {
|
||||
allFileChunks = append(allFileChunks, data.fileChunks...)
|
||||
@@ -551,52 +593,77 @@ func (s *Scanner) executeBatchFileFlush(ctx context.Context, allFiles []*databas
|
||||
|
||||
// Batch delete old file_chunks and chunk_files
|
||||
log.Debug("flushCompletedPendingFiles: deleting old file_chunks")
|
||||
|
||||
opStart := time.Now()
|
||||
if err := s.repos.FileChunks.DeleteByFileIDs(txCtx, tx, allFileIDs); err != nil {
|
||||
|
||||
err := s.repos.FileChunks.DeleteByFileIDs(txCtx, tx, allFileIDs)
|
||||
if err != nil {
|
||||
return fmt.Errorf("batch deleting old file chunks: %w", err)
|
||||
}
|
||||
|
||||
log.Debug("flushCompletedPendingFiles: deleted file_chunks", "duration", time.Since(opStart))
|
||||
|
||||
log.Debug("flushCompletedPendingFiles: deleting old chunk_files")
|
||||
|
||||
opStart = time.Now()
|
||||
if err := s.repos.ChunkFiles.DeleteByFileIDs(txCtx, tx, allFileIDs); err != nil {
|
||||
|
||||
err = s.repos.ChunkFiles.DeleteByFileIDs(txCtx, tx, allFileIDs)
|
||||
if err != nil {
|
||||
return fmt.Errorf("batch deleting old chunk files: %w", err)
|
||||
}
|
||||
|
||||
log.Debug("flushCompletedPendingFiles: deleted chunk_files", "duration", time.Since(opStart))
|
||||
|
||||
// Batch create/update file records
|
||||
log.Debug("flushCompletedPendingFiles: creating files")
|
||||
|
||||
opStart = time.Now()
|
||||
if err := s.repos.Files.CreateBatch(txCtx, tx, allFiles); err != nil {
|
||||
|
||||
err = s.repos.Files.CreateBatch(txCtx, tx, allFiles)
|
||||
if err != nil {
|
||||
return fmt.Errorf("batch creating file records: %w", err)
|
||||
}
|
||||
|
||||
log.Debug("flushCompletedPendingFiles: created files", "duration", time.Since(opStart))
|
||||
|
||||
// Batch insert file_chunks
|
||||
log.Debug("flushCompletedPendingFiles: inserting file_chunks")
|
||||
|
||||
opStart = time.Now()
|
||||
if err := s.repos.FileChunks.CreateBatch(txCtx, tx, allFileChunks); err != nil {
|
||||
|
||||
err = s.repos.FileChunks.CreateBatch(txCtx, tx, allFileChunks)
|
||||
if err != nil {
|
||||
return fmt.Errorf("batch creating file chunks: %w", err)
|
||||
}
|
||||
|
||||
log.Debug("flushCompletedPendingFiles: inserted file_chunks", "duration", time.Since(opStart))
|
||||
|
||||
// Batch insert chunk_files
|
||||
log.Debug("flushCompletedPendingFiles: inserting chunk_files")
|
||||
|
||||
opStart = time.Now()
|
||||
if err := s.repos.ChunkFiles.CreateBatch(txCtx, tx, allChunkFiles); err != nil {
|
||||
|
||||
err = s.repos.ChunkFiles.CreateBatch(txCtx, tx, allChunkFiles)
|
||||
if err != nil {
|
||||
return fmt.Errorf("batch creating chunk files: %w", err)
|
||||
}
|
||||
|
||||
log.Debug("flushCompletedPendingFiles: inserted chunk_files", "duration", time.Since(opStart))
|
||||
|
||||
// Batch add files to snapshot
|
||||
log.Debug("flushCompletedPendingFiles: adding files to snapshot")
|
||||
|
||||
opStart = time.Now()
|
||||
if err := s.repos.Snapshots.AddFilesByIDBatch(txCtx, tx, s.snapshotID, allFileIDs); err != nil {
|
||||
|
||||
err = s.repos.Snapshots.AddFilesByIDBatch(txCtx, tx, s.snapshotID, allFileIDs)
|
||||
if err != nil {
|
||||
return fmt.Errorf("batch adding files to snapshot: %w", err)
|
||||
}
|
||||
|
||||
log.Debug("flushCompletedPendingFiles: added files to snapshot", "duration", time.Since(opStart))
|
||||
|
||||
log.Debug("flushCompletedPendingFiles: transaction complete")
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
@@ -616,24 +683,31 @@ func (s *Scanner) scanPhase(ctx context.Context, path string, result *ScanResult
|
||||
estimatedTotal := int64(len(knownFiles))
|
||||
|
||||
var filesToProcess []*FileToProcess
|
||||
|
||||
var unchangedFileIDs []types.FileID // Just IDs - no new records needed
|
||||
|
||||
var mu sync.Mutex
|
||||
|
||||
// Set up periodic status output
|
||||
startTime := time.Now()
|
||||
lastStatusTime := time.Now()
|
||||
statusInterval := 15 * time.Second
|
||||
|
||||
var filesScanned int64
|
||||
|
||||
log.Debug("Starting directory walk", "path", path)
|
||||
|
||||
err := afero.Walk(s.fs, path, func(filePath string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
if s.skipErrors {
|
||||
log.Error("Failed to access file (skipping due to --skip-errors)", "path", filePath, "error", err)
|
||||
s.ui.Error("Failed to access %s: %v. Skipping (--skip-errors).", s.ui.Path(filePath), err)
|
||||
s.ui.Errorf("Failed to access %s: %v. Skipping (--skip-errors).", s.ui.Path(filePath), err)
|
||||
|
||||
return nil // Continue scanning
|
||||
}
|
||||
|
||||
log.Debug("Error accessing filesystem entry", "path", filePath, "error", err)
|
||||
|
||||
return wrapPermissionError(filePath, err)
|
||||
}
|
||||
|
||||
@@ -649,6 +723,7 @@ func (s *Scanner) scanPhase(ctx context.Context, path string, result *ScanResult
|
||||
if info.IsDir() {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -657,7 +732,9 @@ func (s *Scanner) scanPhase(ctx context.Context, path string, result *ScanResult
|
||||
file := s.buildSymlinkEntry(filePath, info)
|
||||
if file != nil {
|
||||
existingFiles[filePath] = struct{}{}
|
||||
|
||||
mu.Lock()
|
||||
|
||||
filesToProcess = append(filesToProcess, &FileToProcess{
|
||||
Path: filePath,
|
||||
FileInfo: info,
|
||||
@@ -667,6 +744,7 @@ func (s *Scanner) scanPhase(ctx context.Context, path string, result *ScanResult
|
||||
mu.Unlock()
|
||||
s.updateScanEntryStats(result, true, info)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -674,7 +752,9 @@ func (s *Scanner) scanPhase(ctx context.Context, path string, result *ScanResult
|
||||
if info.IsDir() {
|
||||
file := s.buildDirectoryEntry(filePath, info)
|
||||
existingFiles[filePath] = struct{}{}
|
||||
|
||||
mu.Lock()
|
||||
|
||||
filesToProcess = append(filesToProcess, &FileToProcess{
|
||||
Path: filePath,
|
||||
FileInfo: info,
|
||||
@@ -682,6 +762,7 @@ func (s *Scanner) scanPhase(ctx context.Context, path string, result *ScanResult
|
||||
})
|
||||
filesScanned++
|
||||
mu.Unlock()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -708,6 +789,7 @@ func (s *Scanner) scanPhase(ctx context.Context, path string, result *ScanResult
|
||||
// Unchanged file with existing ID - just need snapshot association
|
||||
unchangedFileIDs = append(unchangedFileIDs, file.ID)
|
||||
}
|
||||
|
||||
filesScanned++
|
||||
changedCount := len(filesToProcess)
|
||||
mu.Unlock()
|
||||
@@ -718,12 +800,12 @@ func (s *Scanner) scanPhase(ctx context.Context, path string, result *ScanResult
|
||||
// Output periodic status
|
||||
if time.Since(lastStatusTime) >= statusInterval {
|
||||
s.printScanProgressLine(filesScanned, changedCount, estimatedTotal, startTime)
|
||||
|
||||
lastStatusTime = time.Now()
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -745,11 +827,13 @@ func (s *Scanner) updateScanEntryStats(result *ScanResult, needsProcessing bool,
|
||||
} else {
|
||||
result.FilesSkipped++
|
||||
result.BytesSkipped += info.Size()
|
||||
|
||||
if s.progress != nil {
|
||||
s.progress.GetStats().FilesSkipped.Add(1)
|
||||
s.progress.GetStats().BytesSkipped.Add(info.Size())
|
||||
}
|
||||
}
|
||||
|
||||
result.FilesScanned++
|
||||
if s.progress != nil {
|
||||
s.progress.GetStats().FilesScanned.Add(1)
|
||||
@@ -768,16 +852,16 @@ func (s *Scanner) printScanProgressLine(filesScanned int64, changedCount int, es
|
||||
if pct > 100 {
|
||||
pct = 100 // Cap at 100% for display
|
||||
}
|
||||
remaining := estimatedTotal - filesScanned
|
||||
if remaining < 0 {
|
||||
remaining = 0
|
||||
}
|
||||
|
||||
remaining := max(estimatedTotal-filesScanned, 0)
|
||||
|
||||
var eta time.Duration
|
||||
if rate > 0 && remaining > 0 {
|
||||
eta = time.Duration(float64(remaining)/rate) * time.Second
|
||||
}
|
||||
|
||||
if eta > 0 {
|
||||
s.ui.Progress("Snapshot source files enumeration: %s files (~%s), %s changed or new, %.0f files/sec, enumeration elapsed: %s, enumeration ETA: %s (est remain %s).",
|
||||
s.ui.Progressf("Snapshot source files enumeration: %s files (~%s), %s changed or new, %.0f files/sec, enumeration elapsed: %s, enumeration ETA: %s (est remain %s).",
|
||||
s.ui.Count(int(filesScanned)),
|
||||
s.ui.Percent(pct),
|
||||
s.ui.Count(changedCount),
|
||||
@@ -786,7 +870,7 @@ func (s *Scanner) printScanProgressLine(filesScanned int64, changedCount int, es
|
||||
s.ui.Time(time.Now().Add(eta)),
|
||||
s.ui.Duration(eta))
|
||||
} else {
|
||||
s.ui.Progress("Snapshot source files enumeration: %s files (~%s), %s changed or new, %.0f files/sec, enumeration elapsed: %s.",
|
||||
s.ui.Progressf("Snapshot source files enumeration: %s files (~%s), %s changed or new, %.0f files/sec, enumeration elapsed: %s.",
|
||||
s.ui.Count(int(filesScanned)),
|
||||
s.ui.Percent(pct),
|
||||
s.ui.Count(changedCount),
|
||||
@@ -794,7 +878,7 @@ func (s *Scanner) printScanProgressLine(filesScanned int64, changedCount int, es
|
||||
s.ui.Duration(elapsed))
|
||||
}
|
||||
} else {
|
||||
s.ui.Progress("Snapshot source files enumeration: %s files seen, %s changed or new, %.0f files/sec, enumeration elapsed: %s.",
|
||||
s.ui.Progressf("Snapshot source files enumeration: %s files seen, %s changed or new, %.0f files/sec, enumeration elapsed: %s.",
|
||||
s.ui.Count(int(filesScanned)),
|
||||
s.ui.Count(changedCount),
|
||||
rate,
|
||||
@@ -808,6 +892,7 @@ func (s *Scanner) buildSymlinkEntry(path string, info os.FileInfo) *database.Fil
|
||||
target, err := os.Readlink(path)
|
||||
if err != nil {
|
||||
log.Debug("Cannot read symlink target", "path", path, "error", err)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -860,9 +945,11 @@ func (s *Scanner) buildDirectoryEntry(path string, info os.FileInfo) *database.F
|
||||
// and associates it with the current snapshot. No chunking is performed.
|
||||
func (s *Scanner) recordNonRegularFile(ctx context.Context, ftp *FileToProcess) error {
|
||||
return s.repos.WithTx(ctx, func(txCtx context.Context, tx *sql.Tx) error {
|
||||
if err := s.repos.Files.Create(txCtx, tx, ftp.File); err != nil {
|
||||
err := s.repos.Files.Create(txCtx, tx, ftp.File)
|
||||
if err != nil {
|
||||
return fmt.Errorf("creating non-regular file record: %w", err)
|
||||
}
|
||||
|
||||
return s.repos.Snapshots.AddFileByID(txCtx, tx, s.snapshotID, ftp.File.ID)
|
||||
})
|
||||
}
|
||||
@@ -941,18 +1028,18 @@ func (s *Scanner) batchAddFilesToSnapshot(ctx context.Context, fileIDs []types.F
|
||||
default:
|
||||
}
|
||||
|
||||
end := i + batchSize
|
||||
if end > len(fileIDs) {
|
||||
end = len(fileIDs)
|
||||
}
|
||||
end := min(i+batchSize, len(fileIDs))
|
||||
|
||||
batch := fileIDs[i:end]
|
||||
|
||||
err := s.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
|
||||
for _, fileID := range batch {
|
||||
if err := s.repos.Snapshots.AddFileByID(ctx, tx, s.snapshotID, fileID); err != nil {
|
||||
err := s.repos.Snapshots.AddFileByID(ctx, tx, s.snapshotID, fileID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("adding file to snapshot: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
@@ -964,15 +1051,16 @@ func (s *Scanner) batchAddFilesToSnapshot(ctx context.Context, fileIDs []types.F
|
||||
elapsed := time.Since(startTime)
|
||||
rate := float64(end) / elapsed.Seconds()
|
||||
pct := float64(end) / float64(len(fileIDs)) * 100
|
||||
s.ui.Progress("Snapshot unchanged-file association: %s/%s (%s), %.0f files/sec.",
|
||||
s.ui.Progressf("Snapshot unchanged-file association: %s/%s (%s), %.0f files/sec.",
|
||||
s.ui.Count(end), s.ui.Count(len(fileIDs)), s.ui.Percent(pct), rate)
|
||||
|
||||
lastStatusTime = time.Now()
|
||||
}
|
||||
}
|
||||
|
||||
elapsed := time.Since(startTime)
|
||||
rate := float64(len(fileIDs)) / elapsed.Seconds()
|
||||
s.ui.Complete("Associated %s unchanged files with the snapshot in %s (%.0f files/sec).",
|
||||
s.ui.Completef("Associated %s unchanged files with the snapshot in %s (%.0f files/sec).",
|
||||
s.ui.Count(len(fileIDs)), s.ui.Duration(elapsed), rate)
|
||||
|
||||
return nil
|
||||
@@ -991,7 +1079,9 @@ func (s *Scanner) processPhase(ctx context.Context, filesToProcess []*FileToProc
|
||||
statusInterval := 15 * time.Second
|
||||
startTime := time.Now()
|
||||
filesProcessed := 0
|
||||
|
||||
var bytesProcessed int64
|
||||
|
||||
totalFiles := len(filesToProcess)
|
||||
|
||||
// Process each file
|
||||
@@ -1006,6 +1096,7 @@ func (s *Scanner) processPhase(ctx context.Context, filesToProcess []*FileToProc
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if skipped {
|
||||
continue
|
||||
}
|
||||
@@ -1021,6 +1112,7 @@ func (s *Scanner) processPhase(ctx context.Context, filesToProcess []*FileToProc
|
||||
// Output periodic status
|
||||
if time.Since(lastStatusTime) >= statusInterval {
|
||||
s.printProcessingProgress(filesProcessed, totalFiles, bytesProcessed, totalBytes, startTime)
|
||||
|
||||
lastStatusTime = time.Now()
|
||||
}
|
||||
}
|
||||
@@ -1032,22 +1124,29 @@ func (s *Scanner) processPhase(ctx context.Context, filesToProcess []*FileToProc
|
||||
// processFileWithErrorHandling wraps processFileStreaming with error recovery for
|
||||
// deleted files and skip-errors mode. Returns (skipped, error).
|
||||
func (s *Scanner) processFileWithErrorHandling(ctx context.Context, fileToProcess *FileToProcess, result *ScanResult) (bool, error) {
|
||||
if err := s.processFileStreaming(ctx, fileToProcess, result); err != nil {
|
||||
err := s.processFileStreaming(ctx, fileToProcess, result)
|
||||
if err != nil {
|
||||
// Handle files that were deleted between scan and process phases
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
log.Warn("File was deleted during backup, skipping", "path", fileToProcess.Path)
|
||||
|
||||
result.FilesSkipped++
|
||||
|
||||
return true, nil
|
||||
}
|
||||
// Skip file read errors if --skip-errors is enabled
|
||||
if s.skipErrors {
|
||||
log.Error("Failed to process file (skipping due to --skip-errors)", "path", fileToProcess.Path, "error", err)
|
||||
s.ui.Error("Failed to process %s: %v. Skipping (--skip-errors).", s.ui.Path(fileToProcess.Path), err)
|
||||
s.ui.Errorf("Failed to process %s: %v. Skipping (--skip-errors).", s.ui.Path(fileToProcess.Path), err)
|
||||
|
||||
result.FilesSkipped++
|
||||
|
||||
return true, nil
|
||||
}
|
||||
|
||||
return false, fmt.Errorf("processing file %s: %w", fileToProcess.Path, err)
|
||||
}
|
||||
|
||||
return false, nil
|
||||
}
|
||||
|
||||
@@ -1061,13 +1160,14 @@ func (s *Scanner) printProcessingProgress(filesProcessed, totalFiles int, bytesP
|
||||
|
||||
// Calculate ETA based on bytes (more accurate than files)
|
||||
remainingBytes := totalBytes - bytesProcessed
|
||||
|
||||
var eta time.Duration
|
||||
if byteRate > 0 {
|
||||
eta = time.Duration(float64(remainingBytes)/byteRate) * time.Second
|
||||
}
|
||||
|
||||
if eta > 0 {
|
||||
s.ui.Progress("Snapshot backup: %s/%s files (%s), %s/%s, %s, %.0f files/sec, backup elapsed: %s, backup ETA: %s (est remain %s).",
|
||||
s.ui.Progressf("Snapshot backup: %s/%s files (%s), %s/%s, %s, %.0f files/sec, backup elapsed: %s, backup ETA: %s (est remain %s).",
|
||||
s.ui.Count(filesProcessed),
|
||||
s.ui.Count(totalFiles),
|
||||
s.ui.Percent(pct),
|
||||
@@ -1079,7 +1179,7 @@ func (s *Scanner) printProcessingProgress(filesProcessed, totalFiles int, bytesP
|
||||
s.ui.Time(time.Now().Add(eta)),
|
||||
s.ui.Duration(eta))
|
||||
} else {
|
||||
s.ui.Progress("Snapshot backup: %s/%s files (%s), %s/%s, %s, %.0f files/sec, backup elapsed: %s.",
|
||||
s.ui.Progressf("Snapshot backup: %s/%s files (%s), %s/%s, %s, %.0f files/sec, backup elapsed: %s.",
|
||||
s.ui.Count(filesProcessed),
|
||||
s.ui.Count(totalFiles),
|
||||
s.ui.Percent(pct),
|
||||
@@ -1097,15 +1197,19 @@ func (s *Scanner) finalizeProcessPhase(ctx context.Context, result *ScanResult)
|
||||
// Final packer flush first - this commits remaining chunks to DB
|
||||
// and handleBlobReady will flush files whose chunks are now committed
|
||||
s.packerMu.Lock()
|
||||
if err := s.packer.Flush(); err != nil {
|
||||
|
||||
err := s.packer.Flush()
|
||||
if err != nil {
|
||||
s.packerMu.Unlock()
|
||||
|
||||
return fmt.Errorf("flushing packer: %w", err)
|
||||
}
|
||||
s.packerMu.Unlock()
|
||||
|
||||
// Flush any remaining pending files (e.g., files with only pre-existing chunks
|
||||
// that didn't trigger a blob finalize)
|
||||
if err := s.flushAllPending(ctx); err != nil {
|
||||
err = s.flushAllPending(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("flushing remaining pending files: %w", err)
|
||||
}
|
||||
|
||||
@@ -1119,6 +1223,7 @@ func (s *Scanner) finalizeProcessPhase(ctx context.Context, result *ScanResult)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parsing blob ID: %w", err)
|
||||
}
|
||||
|
||||
err = s.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
|
||||
return s.repos.Snapshots.AddBlob(ctx, tx, s.snapshotID, blobID, types.BlobHash(b.Hash))
|
||||
})
|
||||
@@ -1126,6 +1231,7 @@ func (s *Scanner) finalizeProcessPhase(ctx context.Context, result *ScanResult)
|
||||
return fmt.Errorf("storing blob metadata: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
result.BlobsCreated += len(blobs)
|
||||
}
|
||||
|
||||
@@ -1148,14 +1254,17 @@ func (s *Scanner) handleBlobReady(blobWithReader *blob.BlobWithReader) error {
|
||||
}
|
||||
|
||||
blobPath := fmt.Sprintf("blobs/%s/%s/%s", finishedBlob.Hash[:2], finishedBlob.Hash[2:4], finishedBlob.Hash)
|
||||
|
||||
blobExists, err := s.uploadBlobIfNeeded(ctx, blobPath, blobWithReader, startTime)
|
||||
if err != nil {
|
||||
s.cleanupBlobTempFile(blobWithReader)
|
||||
|
||||
return fmt.Errorf("uploading blob %s: %w", finishedBlob.Hash, err)
|
||||
}
|
||||
|
||||
if err := s.recordBlobMetadata(ctx, finishedBlob, blobExists, startTime); err != nil {
|
||||
s.cleanupBlobTempFile(blobWithReader)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -1181,25 +1290,28 @@ func (s *Scanner) uploadBlobIfNeeded(ctx context.Context, blobPath string, blobW
|
||||
if _, err := s.storage.Stat(ctx, blobPath); err == nil {
|
||||
log.Info("Blob already exists in storage, skipping upload",
|
||||
"hash", finishedBlob.Hash, "size", humanize.Bytes(uint64(finishedBlob.Compressed)))
|
||||
s.ui.Info("Blob %s (%s) already exists at %s. Skipping upload.",
|
||||
s.ui.Infof("Blob %s (%s) already exists at %s. Skipping upload.",
|
||||
s.ui.Hex(finishedBlob.Hash), s.ui.Size(finishedBlob.Compressed), s.ui.Path(destination))
|
||||
|
||||
return true, nil
|
||||
}
|
||||
|
||||
s.ui.Begin("Uploading blob %s (%s) to %s.",
|
||||
s.ui.Beginf("Uploading blob %s (%s) to %s.",
|
||||
s.ui.Hex(finishedBlob.Hash), s.ui.Size(finishedBlob.Compressed), s.ui.Path(destination))
|
||||
|
||||
progressCallback := s.makeUploadProgressCallback(ctx, finishedBlob, startTime)
|
||||
|
||||
if err := s.storage.PutWithProgress(ctx, blobPath, blobWithReader.Reader, finishedBlob.Compressed, progressCallback); err != nil {
|
||||
err := s.storage.PutWithProgress(ctx, blobPath, blobWithReader.Reader, finishedBlob.Compressed, progressCallback)
|
||||
if err != nil {
|
||||
log.Error("Failed to upload blob", "hash", finishedBlob.Hash, "error", err)
|
||||
|
||||
return false, fmt.Errorf("uploading blob to storage: %w", err)
|
||||
}
|
||||
|
||||
uploadDuration := time.Since(startTime)
|
||||
uploadSpeedBps := float64(finishedBlob.Compressed) / uploadDuration.Seconds()
|
||||
|
||||
s.ui.Complete("Uploaded blob %s (%s) in %s at %s.",
|
||||
s.ui.Completef("Uploaded blob %s (%s) in %s at %s.",
|
||||
s.ui.Hex(finishedBlob.Hash),
|
||||
s.ui.Size(finishedBlob.Compressed),
|
||||
s.ui.Duration(uploadDuration),
|
||||
@@ -1228,17 +1340,21 @@ func (s *Scanner) makeUploadProgressCallback(ctx context.Context, finishedBlob *
|
||||
lastProgressTime := time.Now()
|
||||
lastProgressBytes := int64(0)
|
||||
lastStdoutTime := time.Now()
|
||||
|
||||
const stdoutInterval = 15 * time.Second
|
||||
|
||||
return func(uploaded int64) error {
|
||||
now := time.Now()
|
||||
|
||||
elapsed := now.Sub(lastProgressTime).Seconds()
|
||||
if elapsed > 0.5 {
|
||||
bytesSinceLastUpdate := uploaded - lastProgressBytes
|
||||
|
||||
speed := float64(bytesSinceLastUpdate) / elapsed
|
||||
if s.progress != nil {
|
||||
s.progress.ReportUploadProgress(finishedBlob.Hash, uploaded, finishedBlob.Compressed, speed)
|
||||
}
|
||||
|
||||
lastProgressTime = now
|
||||
lastProgressBytes = uploaded
|
||||
}
|
||||
@@ -1248,11 +1364,13 @@ func (s *Scanner) makeUploadProgressCallback(ctx context.Context, finishedBlob *
|
||||
totalElapsed := now.Sub(uploadStart)
|
||||
pct := float64(uploaded) / float64(finishedBlob.Compressed) * 100
|
||||
avgSpeed := float64(uploaded) / totalElapsed.Seconds()
|
||||
|
||||
var eta time.Duration
|
||||
if avgSpeed > 0 {
|
||||
eta = time.Duration(float64(finishedBlob.Compressed-uploaded)/avgSpeed) * time.Second
|
||||
}
|
||||
s.ui.Progress("Blob upload %s: %s / %s (%s) at %s, blob upload elapsed: %s, blob upload ETA: %s (est remain %s).",
|
||||
|
||||
s.ui.Progressf("Blob upload %s: %s / %s (%s) at %s, blob upload elapsed: %s, blob upload ETA: %s (est remain %s).",
|
||||
s.ui.Hex(finishedBlob.Hash),
|
||||
s.ui.Size(uploaded),
|
||||
s.ui.Size(finishedBlob.Compressed),
|
||||
@@ -1283,11 +1401,13 @@ func (s *Scanner) recordBlobMetadata(ctx context.Context, finishedBlob *blob.Fin
|
||||
uploadDuration := time.Since(startTime)
|
||||
|
||||
return s.repos.WithTx(ctx, func(txCtx context.Context, tx *sql.Tx) error {
|
||||
if err := s.repos.Blobs.UpdateUploaded(txCtx, tx, finishedBlob.ID); err != nil {
|
||||
err := s.repos.Blobs.UpdateUploaded(txCtx, tx, finishedBlob.ID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("updating blob upload timestamp: %w", err)
|
||||
}
|
||||
|
||||
if err := s.repos.Snapshots.AddBlob(txCtx, tx, s.snapshotID, finishedBlobID, types.BlobHash(finishedBlob.Hash)); err != nil {
|
||||
err = s.repos.Snapshots.AddBlob(txCtx, tx, s.snapshotID, finishedBlobID, types.BlobHash(finishedBlob.Hash))
|
||||
if err != nil {
|
||||
return fmt.Errorf("adding blob to snapshot: %w", err)
|
||||
}
|
||||
|
||||
@@ -1299,7 +1419,9 @@ func (s *Scanner) recordBlobMetadata(ctx context.Context, finishedBlob *blob.Fin
|
||||
Size: finishedBlob.Compressed,
|
||||
DurationMs: uploadDuration.Milliseconds(),
|
||||
}
|
||||
if err := s.repos.Uploads.Create(txCtx, tx, upload); err != nil {
|
||||
|
||||
err := s.repos.Uploads.Create(txCtx, tx, upload)
|
||||
if err != nil {
|
||||
return fmt.Errorf("recording upload metrics: %w", err)
|
||||
}
|
||||
}
|
||||
@@ -1312,10 +1434,14 @@ func (s *Scanner) recordBlobMetadata(ctx context.Context, finishedBlob *blob.Fin
|
||||
func (s *Scanner) cleanupBlobTempFile(blobWithReader *blob.BlobWithReader) {
|
||||
if blobWithReader.TempFile != nil {
|
||||
tempName := blobWithReader.TempFile.Name()
|
||||
if err := blobWithReader.TempFile.Close(); err != nil {
|
||||
|
||||
err := blobWithReader.TempFile.Close()
|
||||
if err != nil {
|
||||
log.Fatal("Failed to close temp file", "file", tempName, "error", err)
|
||||
}
|
||||
if err := s.fs.Remove(tempName); err != nil {
|
||||
|
||||
err = s.fs.Remove(tempName)
|
||||
if err != nil {
|
||||
log.Fatal("Failed to remove temp file", "file", tempName, "error", err)
|
||||
}
|
||||
}
|
||||
@@ -1343,6 +1469,7 @@ func (s *Scanner) processFileStreaming(ctx context.Context, fileToProcess *FileT
|
||||
defer func() { _ = file.Close() }()
|
||||
|
||||
var chunks []streamingChunkInfo
|
||||
|
||||
chunkIndex := 0
|
||||
|
||||
fileHash, err := s.chunker.ChunkReaderStreaming(file, func(chunk chunker.Chunk) error {
|
||||
@@ -1372,16 +1499,17 @@ func (s *Scanner) processFileStreaming(ctx context.Context, fileToProcess *FileT
|
||||
s.updateChunkStats(chunkExists, chunk.Size, result)
|
||||
|
||||
if !chunkExists {
|
||||
if err := s.addChunkToPacker(chunk); err != nil {
|
||||
err := s.addChunkToPacker(chunk)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
chunk.Data = nil
|
||||
chunkIndex++
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("chunking file: %w", err)
|
||||
}
|
||||
@@ -1390,6 +1518,7 @@ func (s *Scanner) processFileStreaming(ctx context.Context, fileToProcess *FileT
|
||||
"path", fileToProcess.Path, "file_hash", fileHash, "chunks", len(chunks))
|
||||
|
||||
s.queueFileForBatchInsert(ctx, fileToProcess, chunks)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1397,6 +1526,7 @@ func (s *Scanner) processFileStreaming(ctx context.Context, fileToProcess *FileT
|
||||
func (s *Scanner) updateChunkStats(chunkExists bool, chunkSize int64, result *ScanResult) {
|
||||
if chunkExists {
|
||||
result.FilesSkipped++
|
||||
|
||||
result.BytesSkipped += chunkSize
|
||||
if s.progress != nil {
|
||||
s.progress.GetStats().BytesSkipped.Add(chunkSize)
|
||||
@@ -1404,6 +1534,7 @@ func (s *Scanner) updateChunkStats(chunkExists bool, chunkSize int64, result *Sc
|
||||
} else {
|
||||
result.ChunksCreated++
|
||||
result.BytesScanned += chunkSize
|
||||
|
||||
if s.progress != nil {
|
||||
s.progress.GetStats().ChunksCreated.Add(1)
|
||||
s.progress.GetStats().BytesProcessed.Add(chunkSize)
|
||||
@@ -1415,27 +1546,36 @@ func (s *Scanner) updateChunkStats(chunkExists bool, chunkSize int64, result *Sc
|
||||
// addChunkToPacker adds a chunk to the blob packer, finalizing the current blob if needed
|
||||
func (s *Scanner) addChunkToPacker(chunk chunker.Chunk) error {
|
||||
s.packerMu.Lock()
|
||||
|
||||
err := s.packer.AddChunk(&blob.ChunkRef{Hash: chunk.Hash, Data: chunk.Data})
|
||||
if err == blob.ErrBlobSizeLimitExceeded {
|
||||
if err := s.packer.FinalizeBlob(); err != nil {
|
||||
if errors.Is(err, blob.ErrBlobSizeLimitExceeded) {
|
||||
err := s.packer.FinalizeBlob()
|
||||
if err != nil {
|
||||
s.packerMu.Unlock()
|
||||
|
||||
return fmt.Errorf("finalizing blob: %w", err)
|
||||
}
|
||||
if err := s.packer.AddChunk(&blob.ChunkRef{Hash: chunk.Hash, Data: chunk.Data}); err != nil {
|
||||
|
||||
err = s.packer.AddChunk(&blob.ChunkRef{Hash: chunk.Hash, Data: chunk.Data})
|
||||
if err != nil {
|
||||
s.packerMu.Unlock()
|
||||
|
||||
return fmt.Errorf("adding chunk after finalize: %w", err)
|
||||
}
|
||||
} else if err != nil {
|
||||
s.packerMu.Unlock()
|
||||
|
||||
return fmt.Errorf("adding chunk to packer: %w", err)
|
||||
}
|
||||
s.packerMu.Unlock()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// queueFileForBatchInsert builds file/chunk associations and queues the file for batch DB insert
|
||||
func (s *Scanner) queueFileForBatchInsert(ctx context.Context, fileToProcess *FileToProcess, chunks []streamingChunkInfo) {
|
||||
fileChunks := make([]database.FileChunk, len(chunks))
|
||||
|
||||
chunkFiles := make([]database.ChunkFile, len(chunks))
|
||||
for i, ci := range chunks {
|
||||
fileChunks[i] = database.FileChunk{
|
||||
@@ -1489,7 +1629,7 @@ func (s *Scanner) detectDeletedFilesFromMap(ctx context.Context, knownFiles map[
|
||||
}
|
||||
|
||||
if result.FilesDeleted > 0 {
|
||||
s.ui.Info("Snapshot source files enumeration detected %s deleted files.", s.ui.Count(result.FilesDeleted))
|
||||
s.ui.Infof("Snapshot source files enumeration detected %s deleted files.", s.ui.Count(result.FilesDeleted))
|
||||
}
|
||||
|
||||
return nil
|
||||
@@ -1503,6 +1643,7 @@ func wrapPermissionError(path string, err error) error {
|
||||
if !errors.Is(err, os.ErrPermission) {
|
||||
return err
|
||||
}
|
||||
|
||||
if runtime.GOOS == "darwin" {
|
||||
return fmt.Errorf("cannot read %s: %w\n\n"+
|
||||
"macOS is blocking access to this path. Grant Full Disk Access to your\n"+
|
||||
@@ -1510,12 +1651,14 @@ func wrapPermissionError(path string, err error) error {
|
||||
" System Settings → Privacy & Security → Full Disk Access\n\n"+
|
||||
"then quit and reopen the terminal and re-run the backup", path, err)
|
||||
}
|
||||
|
||||
return fmt.Errorf("cannot read %s: %w (check file permissions, or run with --skip-errors to continue past unreadable files)", path, err)
|
||||
}
|
||||
|
||||
// compileExcludePatterns compiles the exclude patterns into glob matchers
|
||||
func compileExcludePatterns(patterns []string) []compiledPattern {
|
||||
var compiled []compiledPattern
|
||||
|
||||
for _, p := range patterns {
|
||||
if p == "" {
|
||||
continue
|
||||
@@ -1523,6 +1666,7 @@ func compileExcludePatterns(patterns []string) []compiledPattern {
|
||||
|
||||
// Check if pattern is anchored (starts with /)
|
||||
anchored := strings.HasPrefix(p, "/")
|
||||
|
||||
pattern := p
|
||||
if anchored {
|
||||
pattern = p[1:] // Remove leading /
|
||||
@@ -1537,6 +1681,7 @@ func compileExcludePatterns(patterns []string) []compiledPattern {
|
||||
g, err := glob.Compile(pattern, '/')
|
||||
if err != nil {
|
||||
log.Warn("Invalid exclude pattern, skipping", "pattern", p, "error", err)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -1546,6 +1691,7 @@ func compileExcludePatterns(patterns []string) []compiledPattern {
|
||||
original: p,
|
||||
})
|
||||
}
|
||||
|
||||
return compiled
|
||||
}
|
||||
|
||||
|
||||
@@ -33,16 +33,22 @@ func TestScannerSimpleDirectory(t *testing.T) {
|
||||
|
||||
// Create files with specific times
|
||||
testTime := time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC)
|
||||
|
||||
for path, content := range testFiles {
|
||||
dir := filepath.Dir(path)
|
||||
if err := fs.MkdirAll(dir, 0755); err != nil {
|
||||
|
||||
err := fs.MkdirAll(dir, 0755)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create directory %s: %v", dir, err)
|
||||
}
|
||||
if err := afero.WriteFile(fs, path, []byte(content), 0644); err != nil {
|
||||
|
||||
err = afero.WriteFile(fs, path, []byte(content), 0644)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to write file %s: %v", path, err)
|
||||
}
|
||||
// Set times
|
||||
if err := fs.Chtimes(path, testTime, testTime); err != nil {
|
||||
err = fs.Chtimes(path, testTime, testTime)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to set times for %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
@@ -53,7 +59,8 @@ func TestScannerSimpleDirectory(t *testing.T) {
|
||||
t.Fatalf("failed to create test database: %v", err)
|
||||
}
|
||||
defer func() {
|
||||
if err := db.Close(); err != nil {
|
||||
err := db.Close()
|
||||
if err != nil {
|
||||
t.Errorf("failed to close database: %v", err)
|
||||
}
|
||||
}()
|
||||
@@ -73,6 +80,7 @@ func TestScannerSimpleDirectory(t *testing.T) {
|
||||
// Create a snapshot record for testing
|
||||
ctx := context.Background()
|
||||
snapshotID := "test-snapshot-001"
|
||||
|
||||
err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
|
||||
snapshot := &database.Snapshot{
|
||||
ID: types.SnapshotID(snapshotID),
|
||||
@@ -87,6 +95,7 @@ func TestScannerSimpleDirectory(t *testing.T) {
|
||||
BlobSize: 0,
|
||||
CompressionRatio: 1.0,
|
||||
}
|
||||
|
||||
return repos.Snapshots.Create(ctx, tx, snapshot)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -95,6 +104,7 @@ func TestScannerSimpleDirectory(t *testing.T) {
|
||||
|
||||
// Scan the directory
|
||||
var result *snapshot.ScanResult
|
||||
|
||||
result, err = scanner.Scan(ctx, "/source", snapshotID)
|
||||
if err != nil {
|
||||
t.Fatalf("scan failed: %v", err)
|
||||
@@ -170,7 +180,7 @@ func TestScannerLargeFile(t *testing.T) {
|
||||
// Use random content to ensure good chunk boundaries
|
||||
largeContent := make([]byte, 1024*1024) // 1MB
|
||||
// Fill with pseudo-random data to ensure chunk boundaries
|
||||
for i := 0; i < len(largeContent); i++ {
|
||||
for i := range largeContent {
|
||||
// Simple pseudo-random generator for deterministic tests
|
||||
largeContent[i] = byte((i * 7919) ^ (i >> 3))
|
||||
}
|
||||
@@ -178,6 +188,7 @@ func TestScannerLargeFile(t *testing.T) {
|
||||
if err := fs.MkdirAll("/source", 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := afero.WriteFile(fs, "/source/large.bin", largeContent, 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -188,7 +199,8 @@ func TestScannerLargeFile(t *testing.T) {
|
||||
t.Fatalf("failed to create test database: %v", err)
|
||||
}
|
||||
defer func() {
|
||||
if err := db.Close(); err != nil {
|
||||
err := db.Close()
|
||||
if err != nil {
|
||||
t.Errorf("failed to close database: %v", err)
|
||||
}
|
||||
}()
|
||||
@@ -208,6 +220,7 @@ func TestScannerLargeFile(t *testing.T) {
|
||||
// Create a snapshot record for testing
|
||||
ctx := context.Background()
|
||||
snapshotID := "test-snapshot-001"
|
||||
|
||||
err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
|
||||
snapshot := &database.Snapshot{
|
||||
ID: types.SnapshotID(snapshotID),
|
||||
@@ -222,6 +235,7 @@ func TestScannerLargeFile(t *testing.T) {
|
||||
BlobSize: 0,
|
||||
CompressionRatio: 1.0,
|
||||
}
|
||||
|
||||
return repos.Snapshots.Create(ctx, tx, snapshot)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -230,6 +244,7 @@ func TestScannerLargeFile(t *testing.T) {
|
||||
|
||||
// Scan the directory
|
||||
var result *snapshot.ScanResult
|
||||
|
||||
result, err = scanner.Scan(ctx, "/source", snapshotID)
|
||||
if err != nil {
|
||||
t.Fatalf("scan failed: %v", err)
|
||||
|
||||
@@ -37,6 +37,8 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os/exec"
|
||||
@@ -97,12 +99,13 @@ func (sm *SnapshotManager) CreateSnapshot(ctx context.Context, hostname, version
|
||||
func (sm *SnapshotManager) CreateSnapshotWithName(ctx context.Context, hostname, name, version, gitRevision string) (string, error) {
|
||||
// Use short hostname (strip domain if present)
|
||||
shortHostname := hostname
|
||||
if idx := strings.Index(hostname, "."); idx != -1 {
|
||||
shortHostname = hostname[:idx]
|
||||
if before, _, ok := strings.Cut(hostname, "."); ok {
|
||||
shortHostname = before
|
||||
}
|
||||
|
||||
// Build snapshot ID with optional name
|
||||
timestamp := time.Now().UTC().Format("2006-01-02T15:04:05Z")
|
||||
|
||||
var snapshotID string
|
||||
if name != "" {
|
||||
snapshotID = fmt.Sprintf("%s_%s_%s", shortHostname, name, timestamp)
|
||||
@@ -128,12 +131,12 @@ func (sm *SnapshotManager) CreateSnapshotWithName(ctx context.Context, hostname,
|
||||
err := sm.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
|
||||
return sm.repos.Snapshots.Create(ctx, tx, snapshot)
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("creating snapshot: %w", err)
|
||||
}
|
||||
|
||||
log.Info("Created snapshot", "snapshot_id", snapshotID)
|
||||
|
||||
return snapshotID, nil
|
||||
}
|
||||
|
||||
@@ -148,7 +151,6 @@ func (sm *SnapshotManager) UpdateSnapshotStats(ctx context.Context, snapshotID s
|
||||
stats.BytesUploaded,
|
||||
)
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("updating snapshot stats: %w", err)
|
||||
}
|
||||
@@ -161,13 +163,14 @@ func (sm *SnapshotManager) UpdateSnapshotStats(ctx context.Context, snapshotID s
|
||||
func (sm *SnapshotManager) UpdateSnapshotStatsExtended(ctx context.Context, snapshotID string, stats ExtendedBackupStats) error {
|
||||
return sm.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
|
||||
// First update basic stats
|
||||
if err := sm.repos.Snapshots.UpdateCounts(ctx, tx, snapshotID,
|
||||
err := sm.repos.Snapshots.UpdateCounts(ctx, tx, snapshotID,
|
||||
int64(stats.FilesScanned),
|
||||
int64(stats.ChunksCreated),
|
||||
int64(stats.BlobsCreated),
|
||||
stats.BytesScanned,
|
||||
stats.BytesUploaded,
|
||||
); err != nil {
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -190,18 +193,20 @@ func (sm *SnapshotManager) CompleteSnapshot(ctx context.Context, snapshotID stri
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if added > 0 {
|
||||
log.Info("Populated snapshot_blobs with dedup-referenced blobs",
|
||||
"snapshot_id", snapshotID, "added", added)
|
||||
}
|
||||
|
||||
return sm.repos.Snapshots.MarkComplete(ctx, tx, snapshotID)
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("marking snapshot complete: %w", err)
|
||||
}
|
||||
|
||||
log.Info("Completed snapshot", "snapshot_id", snapshotID)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -229,10 +234,13 @@ func (sm *SnapshotManager) ExportSnapshotMetadata(ctx context.Context, dbPath st
|
||||
if err != nil {
|
||||
return fmt.Errorf("creating temp dir: %w", err)
|
||||
}
|
||||
|
||||
log.Debug("Created temporary directory", "path", tempDir)
|
||||
defer func() {
|
||||
log.Debug("Cleaning up temporary directory", "path", tempDir)
|
||||
if err := sm.fs.RemoveAll(tempDir); err != nil {
|
||||
|
||||
err := sm.fs.RemoveAll(tempDir)
|
||||
if err != nil {
|
||||
log.Debug("Failed to remove temp dir", "path", tempDir, "error", err)
|
||||
}
|
||||
}()
|
||||
@@ -258,6 +266,7 @@ func (sm *SnapshotManager) ExportSnapshotMetadata(ctx context.Context, dbPath st
|
||||
"snapshot_id", snapshotID,
|
||||
"db_size", len(finalData),
|
||||
"manifest_size", len(blobManifest))
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -268,17 +277,21 @@ func (sm *SnapshotManager) prepareExportDB(ctx context.Context, dbPath, snapshot
|
||||
// The main database should be closed at this point
|
||||
tempDBPath := filepath.Join(tempDir, "snapshot.db")
|
||||
log.Debug("Copying database to temporary location", "source", dbPath, "destination", tempDBPath)
|
||||
|
||||
if err := sm.copyFile(dbPath, tempDBPath); err != nil {
|
||||
return nil, "", fmt.Errorf("copying database: %w", err)
|
||||
}
|
||||
|
||||
log.Debug("Database copy complete", "size", sm.getFileSize(tempDBPath))
|
||||
|
||||
// Step 2: Clean the temp database to only contain current snapshot data
|
||||
log.Debug("Cleaning temporary database", "snapshot_id", snapshotID)
|
||||
|
||||
stats, err := sm.cleanSnapshotDB(ctx, tempDBPath, snapshotID)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("cleaning snapshot database: %w", err)
|
||||
}
|
||||
|
||||
log.Info("Temporary database cleanup complete",
|
||||
"db_path", tempDBPath,
|
||||
"size_after_clean", humanize.Bytes(uint64(sm.getFileSize(tempDBPath))),
|
||||
@@ -294,6 +307,7 @@ func (sm *SnapshotManager) prepareExportDB(ctx context.Context, dbPath, snapshot
|
||||
if err := sm.vacuumDatabase(tempDBPath); err != nil {
|
||||
return nil, "", fmt.Errorf("vacuuming database: %w", err)
|
||||
}
|
||||
|
||||
log.Debug("Database vacuumed", "size", humanize.Bytes(uint64(sm.getFileSize(tempDBPath))))
|
||||
|
||||
// Step 4: Compress and encrypt the binary database file
|
||||
@@ -301,6 +315,7 @@ func (sm *SnapshotManager) prepareExportDB(ctx context.Context, dbPath, snapshot
|
||||
if err := sm.compressFile(tempDBPath, compressedPath); err != nil {
|
||||
return nil, "", fmt.Errorf("compressing database: %w", err)
|
||||
}
|
||||
|
||||
log.Debug("Compression complete",
|
||||
"original_size", humanize.Bytes(uint64(sm.getFileSize(tempDBPath))),
|
||||
"compressed_size", humanize.Bytes(uint64(sm.getFileSize(compressedPath))))
|
||||
@@ -327,9 +342,12 @@ func (sm *SnapshotManager) uploadSnapshotArtifacts(ctx context.Context, snapshot
|
||||
dbKey := fmt.Sprintf("metadata/%s/db.zst.age", remoteKey)
|
||||
|
||||
dbUploadStart := time.Now()
|
||||
if err := sm.storage.Put(ctx, dbKey, bytes.NewReader(dbData)); err != nil {
|
||||
|
||||
err := sm.storage.Put(ctx, dbKey, bytes.NewReader(dbData))
|
||||
if err != nil {
|
||||
return fmt.Errorf("uploading snapshot database: %w", err)
|
||||
}
|
||||
|
||||
dbUploadDuration := time.Since(dbUploadStart)
|
||||
dbUploadSpeed := float64(len(dbData)) * 8 / dbUploadDuration.Seconds() // bits per second
|
||||
log.Info("Uploaded snapshot database",
|
||||
@@ -341,9 +359,12 @@ func (sm *SnapshotManager) uploadSnapshotArtifacts(ctx context.Context, snapshot
|
||||
// Upload blob manifest (compressed only, not encrypted)
|
||||
manifestKey := fmt.Sprintf("metadata/%s/manifest.json.zst", remoteKey)
|
||||
manifestUploadStart := time.Now()
|
||||
if err := sm.storage.Put(ctx, manifestKey, bytes.NewReader(manifestData)); err != nil {
|
||||
|
||||
err = sm.storage.Put(ctx, manifestKey, bytes.NewReader(manifestData))
|
||||
if err != nil {
|
||||
return fmt.Errorf("uploading blob manifest: %w", err)
|
||||
}
|
||||
|
||||
manifestUploadDuration := time.Since(manifestUploadStart)
|
||||
manifestUploadSpeed := float64(len(manifestData)) * 8 / manifestUploadDuration.Seconds() // bits per second
|
||||
log.Info("Uploaded blob manifest",
|
||||
@@ -383,7 +404,8 @@ func (sm *SnapshotManager) cleanSnapshotDB(ctx context.Context, dbPath string, s
|
||||
return nil, fmt.Errorf("opening temp database: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
if err := db.Close(); err != nil {
|
||||
err := db.Close()
|
||||
if err != nil {
|
||||
log.Debug("Failed to close temp database", "error", err)
|
||||
}
|
||||
}()
|
||||
@@ -394,7 +416,8 @@ func (sm *SnapshotManager) cleanSnapshotDB(ctx context.Context, dbPath string, s
|
||||
return nil, fmt.Errorf("beginning transaction: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
if rbErr := tx.Rollback(); rbErr != nil && rbErr != sql.ErrTxDone {
|
||||
rbErr := tx.Rollback()
|
||||
if rbErr != nil && !errors.Is(rbErr, sql.ErrTxDone) {
|
||||
log.Debug("Failed to rollback transaction", "error", rbErr)
|
||||
}
|
||||
}()
|
||||
@@ -430,6 +453,7 @@ func (sm *SnapshotManager) cleanSnapshotDB(ctx context.Context, dbPath string, s
|
||||
|
||||
// Commit transaction
|
||||
log.Debug("[Temp DB Cleanup] Committing cleanup transaction")
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
return nil, fmt.Errorf("committing transaction: %w", err)
|
||||
}
|
||||
@@ -439,23 +463,30 @@ func (sm *SnapshotManager) cleanSnapshotDB(ctx context.Context, dbPath string, s
|
||||
|
||||
// Count files
|
||||
var fileCount int
|
||||
|
||||
err = db.QueryRowWithLog(ctx, "SELECT COUNT(*) FROM files").Scan(&fileCount)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("counting files: %w", err)
|
||||
}
|
||||
|
||||
stats.FileCount = fileCount
|
||||
|
||||
// Count chunks
|
||||
var chunkCount int
|
||||
|
||||
err = db.QueryRowWithLog(ctx, "SELECT COUNT(*) FROM chunks").Scan(&chunkCount)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("counting chunks: %w", err)
|
||||
}
|
||||
|
||||
stats.ChunkCount = chunkCount
|
||||
|
||||
// Count blobs and get sizes
|
||||
var blobCount int
|
||||
var compressedSize, uncompressedSize sql.NullInt64
|
||||
var (
|
||||
blobCount int
|
||||
compressedSize, uncompressedSize sql.NullInt64
|
||||
)
|
||||
|
||||
err = db.QueryRowWithLog(ctx, `
|
||||
SELECT COUNT(*), COALESCE(SUM(compressed_size), 0), COALESCE(SUM(uncompressed_size), 0)
|
||||
FROM blobs
|
||||
@@ -464,6 +495,7 @@ func (sm *SnapshotManager) cleanSnapshotDB(ctx context.Context, dbPath string, s
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("counting blobs and sizes: %w", err)
|
||||
}
|
||||
|
||||
stats.BlobCount = blobCount
|
||||
stats.CompressedSize = compressedSize.Int64
|
||||
stats.UncompressedSize = uncompressedSize.Int64
|
||||
@@ -491,7 +523,8 @@ func (sm *SnapshotManager) compressFile(inputPath, outputPath string) error {
|
||||
return fmt.Errorf("opening input file: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
if err := input.Close(); err != nil {
|
||||
err := input.Close()
|
||||
if err != nil {
|
||||
log.Debug("Failed to close input file", "path", inputPath, "error", err)
|
||||
}
|
||||
}()
|
||||
@@ -501,13 +534,15 @@ func (sm *SnapshotManager) compressFile(inputPath, outputPath string) error {
|
||||
return fmt.Errorf("creating output file: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
if err := output.Close(); err != nil {
|
||||
err := output.Close()
|
||||
if err != nil {
|
||||
log.Debug("Failed to close output file", "path", outputPath, "error", err)
|
||||
}
|
||||
}()
|
||||
|
||||
// Use blobgen for compression and encryption
|
||||
log.Debug("Compressing and encrypting data")
|
||||
|
||||
writer, err := blobgen.NewWriter(output, sm.config.CompressionLevel, sm.config.AgeRecipients)
|
||||
if err != nil {
|
||||
return fmt.Errorf("creating blobgen writer: %w", err)
|
||||
@@ -517,7 +552,8 @@ func (sm *SnapshotManager) compressFile(inputPath, outputPath string) error {
|
||||
writerClosed := false
|
||||
defer func() {
|
||||
if !writerClosed {
|
||||
if err := writer.Close(); err != nil {
|
||||
err := writer.Close()
|
||||
if err != nil {
|
||||
log.Debug("Failed to close writer", "error", err)
|
||||
}
|
||||
}
|
||||
@@ -531,9 +567,10 @@ func (sm *SnapshotManager) compressFile(inputPath, outputPath string) error {
|
||||
if err := writer.Close(); err != nil {
|
||||
return fmt.Errorf("closing writer: %w", err)
|
||||
}
|
||||
|
||||
writerClosed = true
|
||||
|
||||
log.Debug("Compression complete", "hash", fmt.Sprintf("%x", writer.Sum256()))
|
||||
log.Debug("Compression complete", "hash", hex.EncodeToString(writer.Sum256()))
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -541,34 +578,42 @@ func (sm *SnapshotManager) compressFile(inputPath, outputPath string) error {
|
||||
// copyFile copies a file from src to dst
|
||||
func (sm *SnapshotManager) copyFile(src, dst string) error {
|
||||
log.Debug("Opening source file for copy", "path", src)
|
||||
|
||||
sourceFile, err := sm.fs.Open(src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
log.Debug("Closing source file", "path", src)
|
||||
if err := sourceFile.Close(); err != nil {
|
||||
|
||||
err := sourceFile.Close()
|
||||
if err != nil {
|
||||
log.Debug("Failed to close source file", "path", src, "error", err)
|
||||
}
|
||||
}()
|
||||
|
||||
log.Debug("Creating destination file", "path", dst)
|
||||
|
||||
destFile, err := sm.fs.Create(dst)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
log.Debug("Closing destination file", "path", dst)
|
||||
if err := destFile.Close(); err != nil {
|
||||
|
||||
err := destFile.Close()
|
||||
if err != nil {
|
||||
log.Debug("Failed to close destination file", "path", dst, "error", err)
|
||||
}
|
||||
}()
|
||||
|
||||
log.Debug("Copying file data")
|
||||
|
||||
n, err := io.Copy(destFile, sourceFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Debug("File copy complete", "bytes_copied", n)
|
||||
|
||||
return nil
|
||||
@@ -576,7 +621,6 @@ func (sm *SnapshotManager) copyFile(src, dst string) error {
|
||||
|
||||
// generateBlobManifest creates a compressed JSON list of all blobs in the snapshot
|
||||
func (sm *SnapshotManager) generateBlobManifest(ctx context.Context, dbPath string, snapshotID string) ([]byte, error) {
|
||||
|
||||
// Open the cleaned database using the database package
|
||||
db, err := database.New(ctx, dbPath)
|
||||
if err != nil {
|
||||
@@ -589,10 +633,12 @@ func (sm *SnapshotManager) generateBlobManifest(ctx context.Context, dbPath stri
|
||||
|
||||
// Get all blobs for this snapshot
|
||||
log.Debug("Querying blobs for snapshot", "snapshot_id", snapshotID)
|
||||
|
||||
blobHashes, err := repos.Snapshots.GetBlobHashes(ctx, snapshotID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("getting snapshot blobs: %w", err)
|
||||
}
|
||||
|
||||
log.Debug("Found blobs", "count", len(blobHashes))
|
||||
|
||||
// Get blob details including sizes
|
||||
@@ -603,8 +649,10 @@ func (sm *SnapshotManager) generateBlobManifest(ctx context.Context, dbPath stri
|
||||
blob, err := repos.Blobs.GetByHash(ctx, hash)
|
||||
if err != nil {
|
||||
log.Warn("Failed to get blob details", "hash", hash, "error", err)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
if blob != nil {
|
||||
blobs = append(blobs, BlobInfo{
|
||||
Hash: hash,
|
||||
@@ -648,6 +696,7 @@ func (sm *SnapshotManager) getFileSize(path string) int64 {
|
||||
if err != nil {
|
||||
return -1
|
||||
}
|
||||
|
||||
return info.Size()
|
||||
}
|
||||
|
||||
@@ -663,6 +712,7 @@ type BackupStats struct {
|
||||
// ExtendedBackupStats contains additional statistics for comprehensive tracking
|
||||
type ExtendedBackupStats struct {
|
||||
BackupStats
|
||||
|
||||
BlobUncompressedSize int64 // Total uncompressed size of all referenced blobs
|
||||
CompressionLevel int // Compression level used for this snapshot
|
||||
UploadDurationMs int64 // Total milliseconds spent uploading to S3
|
||||
@@ -682,6 +732,7 @@ func (sm *SnapshotManager) CleanupIncompleteSnapshots(ctx context.Context, hostn
|
||||
|
||||
if len(incompleteSnapshots) == 0 {
|
||||
log.Debug("No incomplete snapshots found")
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -692,14 +743,15 @@ func (sm *SnapshotManager) CleanupIncompleteSnapshots(ctx context.Context, hostn
|
||||
// Check if metadata exists in storage (paths use the hashed
|
||||
// remote key so we don't leak host info to the listing).
|
||||
metadataKey := fmt.Sprintf("metadata/%s/db.zst", RemoteSnapshotKey(snapshot.ID.String()))
|
||||
_, err := sm.storage.Stat(ctx, metadataKey)
|
||||
|
||||
_, err := sm.storage.Stat(ctx, metadataKey)
|
||||
if err != nil {
|
||||
// Metadata doesn't exist in S3 - this is an incomplete snapshot
|
||||
log.Info("Cleaning up incomplete snapshot record", "snapshot_id", snapshot.ID, "started_at", snapshot.StartedAt)
|
||||
|
||||
// Delete the snapshot and all its associations
|
||||
if err := sm.deleteSnapshot(ctx, snapshot.ID.String()); err != nil {
|
||||
err := sm.deleteSnapshot(ctx, snapshot.ID.String())
|
||||
if err != nil {
|
||||
return fmt.Errorf("deleting incomplete snapshot %s: %w", snapshot.ID, err)
|
||||
}
|
||||
|
||||
@@ -708,7 +760,9 @@ func (sm *SnapshotManager) CleanupIncompleteSnapshots(ctx context.Context, hostn
|
||||
// Metadata exists - this snapshot was completed but database wasn't updated
|
||||
// This shouldn't happen in normal operation, but mark it complete
|
||||
log.Warn("Found snapshot with remote metadata but incomplete in database", "snapshot_id", snapshot.ID)
|
||||
if err := sm.repos.Snapshots.MarkComplete(ctx, nil, snapshot.ID.String()); err != nil {
|
||||
|
||||
err := sm.repos.Snapshots.MarkComplete(ctx, nil, snapshot.ID.String())
|
||||
if err != nil {
|
||||
log.Error("Failed to mark snapshot as complete in database", "snapshot_id", snapshot.ID, "error", err)
|
||||
}
|
||||
}
|
||||
@@ -720,28 +774,34 @@ func (sm *SnapshotManager) CleanupIncompleteSnapshots(ctx context.Context, hostn
|
||||
// deleteSnapshot removes a snapshot and all its associations from the database
|
||||
func (sm *SnapshotManager) deleteSnapshot(ctx context.Context, snapshotID string) error {
|
||||
// Delete snapshot_files entries
|
||||
if err := sm.repos.Snapshots.DeleteSnapshotFiles(ctx, snapshotID); err != nil {
|
||||
err := sm.repos.Snapshots.DeleteSnapshotFiles(ctx, snapshotID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("deleting snapshot files: %w", err)
|
||||
}
|
||||
|
||||
// Delete snapshot_blobs entries
|
||||
if err := sm.repos.Snapshots.DeleteSnapshotBlobs(ctx, snapshotID); err != nil {
|
||||
err = sm.repos.Snapshots.DeleteSnapshotBlobs(ctx, snapshotID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("deleting snapshot blobs: %w", err)
|
||||
}
|
||||
|
||||
// Delete uploads entries (has foreign key to snapshots without CASCADE)
|
||||
if err := sm.repos.Snapshots.DeleteSnapshotUploads(ctx, snapshotID); err != nil {
|
||||
err = sm.repos.Snapshots.DeleteSnapshotUploads(ctx, snapshotID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("deleting snapshot uploads: %w", err)
|
||||
}
|
||||
|
||||
// Delete the snapshot itself
|
||||
if err := sm.repos.Snapshots.Delete(ctx, snapshotID); err != nil {
|
||||
err = sm.repos.Snapshots.Delete(ctx, snapshotID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("deleting snapshot: %w", err)
|
||||
}
|
||||
|
||||
// Clean up orphaned data
|
||||
log.Debug("Cleaning up orphaned records in main database")
|
||||
if err := sm.CleanupOrphanedData(ctx); err != nil {
|
||||
|
||||
err = sm.CleanupOrphanedData(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cleaning up orphaned data: %w", err)
|
||||
}
|
||||
|
||||
@@ -759,28 +819,36 @@ func (sm *SnapshotManager) CleanupOrphanedData(ctx context.Context) error {
|
||||
|
||||
// Delete orphaned files (files not in any snapshot)
|
||||
log.Debug("Deleting orphaned file records from database")
|
||||
if err := sm.repos.Files.DeleteOrphaned(ctx); err != nil {
|
||||
|
||||
err := sm.repos.Files.DeleteOrphaned(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("deleting orphaned files: %w", err)
|
||||
}
|
||||
|
||||
// Delete orphaned blobs (blobs not in any snapshot)
|
||||
// This will cascade delete blob_chunks for deleted blobs
|
||||
log.Debug("Deleting orphaned blob records from database")
|
||||
if err := sm.repos.Blobs.DeleteOrphaned(ctx); err != nil {
|
||||
|
||||
err = sm.repos.Blobs.DeleteOrphaned(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("deleting orphaned blobs: %w", err)
|
||||
}
|
||||
|
||||
// Delete orphaned blob_chunks entries
|
||||
// This handles cases where the blob still exists but chunks were deleted
|
||||
log.Debug("Deleting orphaned blob_chunks associations from database")
|
||||
if err := sm.repos.BlobChunks.DeleteOrphaned(ctx); err != nil {
|
||||
|
||||
err = sm.repos.BlobChunks.DeleteOrphaned(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("deleting orphaned blob_chunks: %w", err)
|
||||
}
|
||||
|
||||
// Delete orphaned chunks (chunks not referenced by any file)
|
||||
// This must come after cleaning up blob_chunks to avoid foreign key violations
|
||||
log.Debug("Deleting orphaned chunk records from database")
|
||||
if err := sm.repos.Chunks.DeleteOrphaned(ctx); err != nil {
|
||||
|
||||
err = sm.repos.Chunks.DeleteOrphaned(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("deleting orphaned chunks: %w", err)
|
||||
}
|
||||
|
||||
@@ -793,21 +861,26 @@ func (sm *SnapshotManager) deleteOtherSnapshots(ctx context.Context, tx *sql.Tx,
|
||||
|
||||
// First delete uploads that reference other snapshots (no CASCADE DELETE on this FK)
|
||||
database.LogSQL("Execute", "DELETE FROM uploads WHERE snapshot_id != ?", currentSnapshotID)
|
||||
|
||||
uploadResult, err := tx.ExecContext(ctx, "DELETE FROM uploads WHERE snapshot_id != ?", currentSnapshotID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("deleting uploads for other snapshots: %w", err)
|
||||
}
|
||||
|
||||
uploadsDeleted, _ := uploadResult.RowsAffected()
|
||||
log.Debug("[Temp DB Cleanup] Deleted upload records", "count", uploadsDeleted)
|
||||
|
||||
// Now we can safely delete the snapshots
|
||||
database.LogSQL("Execute", "DELETE FROM snapshots WHERE id != ?", currentSnapshotID)
|
||||
|
||||
result, err := tx.ExecContext(ctx, "DELETE FROM snapshots WHERE id != ?", currentSnapshotID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("deleting other snapshots: %w", err)
|
||||
}
|
||||
|
||||
rowsAffected, _ := result.RowsAffected()
|
||||
log.Debug("[Temp DB Cleanup] Deleted snapshot records from database", "count", rowsAffected)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -816,22 +889,27 @@ func (sm *SnapshotManager) deleteOrphanedSnapshotAssociations(ctx context.Contex
|
||||
// Delete orphaned snapshot_files
|
||||
log.Debug("[Temp DB Cleanup] Deleting orphaned snapshot_files associations")
|
||||
database.LogSQL("Execute", "DELETE FROM snapshot_files WHERE snapshot_id != ?", currentSnapshotID)
|
||||
|
||||
result, err := tx.ExecContext(ctx, "DELETE FROM snapshot_files WHERE snapshot_id != ?", currentSnapshotID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("deleting orphaned snapshot_files: %w", err)
|
||||
}
|
||||
|
||||
rowsAffected, _ := result.RowsAffected()
|
||||
log.Debug("[Temp DB Cleanup] Deleted snapshot_files associations", "count", rowsAffected)
|
||||
|
||||
// Delete orphaned snapshot_blobs
|
||||
log.Debug("[Temp DB Cleanup] Deleting orphaned snapshot_blobs associations")
|
||||
database.LogSQL("Execute", "DELETE FROM snapshot_blobs WHERE snapshot_id != ?", currentSnapshotID)
|
||||
|
||||
result, err = tx.ExecContext(ctx, "DELETE FROM snapshot_blobs WHERE snapshot_id != ?", currentSnapshotID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("deleting orphaned snapshot_blobs: %w", err)
|
||||
}
|
||||
|
||||
rowsAffected, _ = result.RowsAffected()
|
||||
log.Debug("[Temp DB Cleanup] Deleted snapshot_blobs associations", "count", rowsAffected)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -839,6 +917,7 @@ func (sm *SnapshotManager) deleteOrphanedSnapshotAssociations(ctx context.Contex
|
||||
func (sm *SnapshotManager) deleteOrphanedFiles(ctx context.Context, tx *sql.Tx, currentSnapshotID string) error {
|
||||
log.Debug("[Temp DB Cleanup] Deleting file records not referenced by current snapshot")
|
||||
database.LogSQL("Execute", `DELETE FROM files WHERE NOT EXISTS (SELECT 1 FROM snapshot_files WHERE snapshot_files.file_id = files.id AND snapshot_files.snapshot_id = ?)`, currentSnapshotID)
|
||||
|
||||
result, err := tx.ExecContext(ctx, `
|
||||
DELETE FROM files
|
||||
WHERE NOT EXISTS (
|
||||
@@ -849,11 +928,13 @@ func (sm *SnapshotManager) deleteOrphanedFiles(ctx context.Context, tx *sql.Tx,
|
||||
if err != nil {
|
||||
return fmt.Errorf("deleting orphaned files: %w", err)
|
||||
}
|
||||
|
||||
rowsAffected, _ := result.RowsAffected()
|
||||
log.Debug("[Temp DB Cleanup] Deleted file records from database", "count", rowsAffected)
|
||||
|
||||
// Note: file_chunks will be deleted via CASCADE
|
||||
log.Debug("[Temp DB Cleanup] file_chunks associations deleted via CASCADE")
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -861,6 +942,7 @@ func (sm *SnapshotManager) deleteOrphanedFiles(ctx context.Context, tx *sql.Tx,
|
||||
func (sm *SnapshotManager) deleteOrphanedChunkToFileMappings(ctx context.Context, tx *sql.Tx) error {
|
||||
log.Debug("[Temp DB Cleanup] Deleting orphaned chunk_files associations")
|
||||
database.LogSQL("Execute", `DELETE FROM chunk_files WHERE NOT EXISTS (SELECT 1 FROM files WHERE files.id = chunk_files.file_id)`)
|
||||
|
||||
result, err := tx.ExecContext(ctx, `
|
||||
DELETE FROM chunk_files
|
||||
WHERE NOT EXISTS (
|
||||
@@ -870,8 +952,10 @@ func (sm *SnapshotManager) deleteOrphanedChunkToFileMappings(ctx context.Context
|
||||
if err != nil {
|
||||
return fmt.Errorf("deleting orphaned chunk_files: %w", err)
|
||||
}
|
||||
|
||||
rowsAffected, _ := result.RowsAffected()
|
||||
log.Debug("[Temp DB Cleanup] Deleted chunk_files associations", "count", rowsAffected)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -879,6 +963,7 @@ func (sm *SnapshotManager) deleteOrphanedChunkToFileMappings(ctx context.Context
|
||||
func (sm *SnapshotManager) deleteOrphanedBlobs(ctx context.Context, tx *sql.Tx, currentSnapshotID string) error {
|
||||
log.Debug("[Temp DB Cleanup] Deleting blob records not referenced by current snapshot")
|
||||
database.LogSQL("Execute", `DELETE FROM blobs WHERE NOT EXISTS (SELECT 1 FROM snapshot_blobs WHERE snapshot_blobs.blob_hash = blobs.blob_hash AND snapshot_blobs.snapshot_id = ?)`, currentSnapshotID)
|
||||
|
||||
result, err := tx.ExecContext(ctx, `
|
||||
DELETE FROM blobs
|
||||
WHERE NOT EXISTS (
|
||||
@@ -889,8 +974,10 @@ func (sm *SnapshotManager) deleteOrphanedBlobs(ctx context.Context, tx *sql.Tx,
|
||||
if err != nil {
|
||||
return fmt.Errorf("deleting orphaned blobs: %w", err)
|
||||
}
|
||||
|
||||
rowsAffected, _ := result.RowsAffected()
|
||||
log.Debug("[Temp DB Cleanup] Deleted blob records from database", "count", rowsAffected)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -898,6 +985,7 @@ func (sm *SnapshotManager) deleteOrphanedBlobs(ctx context.Context, tx *sql.Tx,
|
||||
func (sm *SnapshotManager) deleteOrphanedBlobToChunkMappings(ctx context.Context, tx *sql.Tx) error {
|
||||
log.Debug("[Temp DB Cleanup] Deleting orphaned blob_chunks associations")
|
||||
database.LogSQL("Execute", `DELETE FROM blob_chunks WHERE NOT EXISTS (SELECT 1 FROM blobs WHERE blobs.id = blob_chunks.blob_id)`)
|
||||
|
||||
result, err := tx.ExecContext(ctx, `
|
||||
DELETE FROM blob_chunks
|
||||
WHERE NOT EXISTS (
|
||||
@@ -907,14 +995,17 @@ func (sm *SnapshotManager) deleteOrphanedBlobToChunkMappings(ctx context.Context
|
||||
if err != nil {
|
||||
return fmt.Errorf("deleting orphaned blob_chunks: %w", err)
|
||||
}
|
||||
|
||||
rowsAffected, _ := result.RowsAffected()
|
||||
log.Debug("[Temp DB Cleanup] Deleted blob_chunks associations", "count", rowsAffected)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// deleteOrphanedChunks deletes chunks not referenced by any file or blob
|
||||
func (sm *SnapshotManager) deleteOrphanedChunks(ctx context.Context, tx *sql.Tx) error {
|
||||
log.Debug("[Temp DB Cleanup] Deleting orphaned chunk records")
|
||||
|
||||
query := `
|
||||
DELETE FROM chunks
|
||||
WHERE NOT EXISTS (
|
||||
@@ -926,11 +1017,14 @@ func (sm *SnapshotManager) deleteOrphanedChunks(ctx context.Context, tx *sql.Tx)
|
||||
WHERE blob_chunks.chunk_hash = chunks.chunk_hash
|
||||
)`
|
||||
database.LogSQL("Execute", query)
|
||||
|
||||
result, err := tx.ExecContext(ctx, query)
|
||||
if err != nil {
|
||||
return fmt.Errorf("deleting orphaned chunks: %w", err)
|
||||
}
|
||||
|
||||
rowsAffected, _ := result.RowsAffected()
|
||||
log.Debug("[Temp DB Cleanup] Deleted chunk records from database", "count", rowsAffected)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -33,6 +33,7 @@ func copyFile(fs afero.Fs, src, dst string) error {
|
||||
defer func() { _ = destFile.Close() }()
|
||||
|
||||
_, err = io.Copy(destFile, sourceFile)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -46,6 +47,7 @@ func TestCleanSnapshotDBEmptySnapshot(t *testing.T) {
|
||||
// Create a test database
|
||||
tempDir := t.TempDir()
|
||||
dbPath := filepath.Join(tempDir, "test.db")
|
||||
|
||||
db, err := database.New(ctx, dbPath)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create database: %v", err)
|
||||
@@ -71,9 +73,11 @@ func TestCleanSnapshotDBEmptySnapshot(t *testing.T) {
|
||||
chunk := &database.Chunk{ChunkHash: "orphan-chunk", Size: 500}
|
||||
|
||||
err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
|
||||
if err := repos.Files.Create(ctx, tx, file); err != nil {
|
||||
err := repos.Files.Create(ctx, tx, file)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return repos.Chunks.Create(ctx, tx, chunk)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -111,7 +115,8 @@ func TestCleanSnapshotDBEmptySnapshot(t *testing.T) {
|
||||
t.Fatalf("failed to open cleaned database: %v", err)
|
||||
}
|
||||
defer func() {
|
||||
if err := cleanedDB.Close(); err != nil {
|
||||
err := cleanedDB.Close()
|
||||
if err != nil {
|
||||
t.Errorf("failed to close database: %v", err)
|
||||
}
|
||||
}()
|
||||
@@ -123,6 +128,7 @@ func TestCleanSnapshotDBEmptySnapshot(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get snapshot: %v", err)
|
||||
}
|
||||
|
||||
if verifySnapshot == nil {
|
||||
t.Error("snapshot should exist")
|
||||
}
|
||||
@@ -132,6 +138,7 @@ func TestCleanSnapshotDBEmptySnapshot(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to check file: %v", err)
|
||||
}
|
||||
|
||||
if f != nil {
|
||||
t.Error("orphan file should not exist")
|
||||
}
|
||||
@@ -141,6 +148,7 @@ func TestCleanSnapshotDBEmptySnapshot(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to check chunk: %v", err)
|
||||
}
|
||||
|
||||
if c != nil {
|
||||
t.Error("orphan chunk should not exist")
|
||||
}
|
||||
@@ -156,6 +164,7 @@ func TestCleanSnapshotDBNonExistentSnapshot(t *testing.T) {
|
||||
// Create a test database
|
||||
tempDir := t.TempDir()
|
||||
dbPath := filepath.Join(tempDir, "test.db")
|
||||
|
||||
db, err := database.New(ctx, dbPath)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create database: %v", err)
|
||||
|
||||
@@ -102,26 +102,32 @@ func (f *FileStorer) PutWithProgress(ctx context.Context, key string, data io.Re
|
||||
// Get retrieves data from the specified key.
|
||||
func (f *FileStorer) Get(ctx context.Context, key string) (io.ReadCloser, error) {
|
||||
path := f.fullPath(key)
|
||||
|
||||
file, err := f.fs.Open(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("opening file: %w", err)
|
||||
}
|
||||
|
||||
return file, nil
|
||||
}
|
||||
|
||||
// Stat returns metadata about an object without retrieving its contents.
|
||||
func (f *FileStorer) Stat(ctx context.Context, key string) (*ObjectInfo, error) {
|
||||
path := f.fullPath(key)
|
||||
|
||||
info, err := f.fs.Stat(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("stat file: %w", err)
|
||||
}
|
||||
|
||||
return &ObjectInfo{
|
||||
Key: key,
|
||||
Size: info.Size(),
|
||||
@@ -131,19 +137,23 @@ func (f *FileStorer) Stat(ctx context.Context, key string) (*ObjectInfo, error)
|
||||
// Delete removes an object.
|
||||
func (f *FileStorer) Delete(ctx context.Context, key string) error {
|
||||
path := f.fullPath(key)
|
||||
|
||||
err := f.fs.Remove(path)
|
||||
if os.IsNotExist(err) {
|
||||
return nil // Match S3 behavior: no error if doesn't exist
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("removing file: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// List returns all keys with the given prefix.
|
||||
func (f *FileStorer) List(ctx context.Context, prefix string) ([]string, error) {
|
||||
var keys []string
|
||||
|
||||
basePath := f.fullPath(prefix)
|
||||
|
||||
// Check if base path exists
|
||||
@@ -151,6 +161,7 @@ func (f *FileStorer) List(ctx context.Context, prefix string) ([]string, error)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("checking path: %w", err)
|
||||
}
|
||||
|
||||
if !exists {
|
||||
return keys, nil // Empty list for non-existent prefix
|
||||
}
|
||||
@@ -177,9 +188,9 @@ func (f *FileStorer) List(ctx context.Context, prefix string) ([]string, error)
|
||||
relPath = strings.ReplaceAll(relPath, string(filepath.Separator), "/")
|
||||
keys = append(keys, relPath)
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("walking directory: %w", err)
|
||||
}
|
||||
@@ -192,14 +203,17 @@ func (f *FileStorer) ListStream(ctx context.Context, prefix string) <-chan Objec
|
||||
ch := make(chan ObjectInfo)
|
||||
go func() {
|
||||
defer close(ch)
|
||||
|
||||
basePath := f.fullPath(prefix)
|
||||
|
||||
// Check if base path exists
|
||||
exists, err := afero.Exists(f.fs, basePath)
|
||||
if err != nil {
|
||||
ch <- ObjectInfo{Err: fmt.Errorf("checking path: %w", err)}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if !exists {
|
||||
return // Empty channel for non-existent prefix
|
||||
}
|
||||
@@ -209,12 +223,14 @@ func (f *FileStorer) ListStream(ctx context.Context, prefix string) <-chan Objec
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
ch <- ObjectInfo{Err: ctx.Err()}
|
||||
|
||||
return ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
ch <- ObjectInfo{Err: err}
|
||||
|
||||
return nil // Continue walking despite errors
|
||||
}
|
||||
|
||||
@@ -222,6 +238,7 @@ func (f *FileStorer) ListStream(ctx context.Context, prefix string) <-chan Objec
|
||||
relPath, err := filepath.Rel(f.basePath, path)
|
||||
if err != nil {
|
||||
ch <- ObjectInfo{Err: fmt.Errorf("computing relative path: %w", err)}
|
||||
|
||||
return nil
|
||||
}
|
||||
// Normalize path separators
|
||||
@@ -231,9 +248,11 @@ func (f *FileStorer) ListStream(ctx context.Context, prefix string) <-chan Objec
|
||||
Size: info.Size(),
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}()
|
||||
|
||||
return ch
|
||||
}
|
||||
|
||||
@@ -257,10 +276,12 @@ func (pw *progressWriter) Write(p []byte) (int, error) {
|
||||
if n > 0 {
|
||||
pw.written += int64(n)
|
||||
if pw.callback != nil {
|
||||
if callbackErr := pw.callback(pw.written); callbackErr != nil {
|
||||
callbackErr := pw.callback(pw.written)
|
||||
if callbackErr != nil {
|
||||
return n, callbackErr
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return n, err
|
||||
}
|
||||
|
||||
@@ -24,6 +24,7 @@ func NewStorer(cfg *config.Config) (Storer, error) {
|
||||
if cfg.StorageURL != "" {
|
||||
return storerFromURL(cfg.StorageURL, cfg)
|
||||
}
|
||||
|
||||
return storerFromLegacyS3Config(cfg)
|
||||
}
|
||||
|
||||
@@ -71,6 +72,7 @@ func storerFromURL(rawURL string, cfg *config.Config) (Storer, error) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("creating S3 client: %w", err)
|
||||
}
|
||||
|
||||
return NewS3Storer(client), nil
|
||||
|
||||
case "rclone":
|
||||
@@ -109,5 +111,6 @@ func storerFromLegacyS3Config(cfg *config.Config) (Storer, error) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("creating S3 client: %w", err)
|
||||
}
|
||||
|
||||
return NewS3Storer(client), nil
|
||||
}
|
||||
|
||||
@@ -49,6 +49,7 @@ func NewRcloneStorer(ctx context.Context, remote, path string) (*RcloneStorer, e
|
||||
strings.Contains(err.Error(), "failed to find remote") {
|
||||
return nil, fmt.Errorf("%w: %s", ErrRemoteNotFound, remote)
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("creating rclone filesystem: %w", err)
|
||||
}
|
||||
|
||||
@@ -101,9 +102,11 @@ func (r *RcloneStorer) Get(ctx context.Context, key string) (io.ReadCloser, erro
|
||||
if errors.Is(err, fs.ErrorObjectNotFound) {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
|
||||
if errors.Is(err, fs.ErrorDirNotFound) {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("getting object: %w", err)
|
||||
}
|
||||
|
||||
@@ -123,9 +126,11 @@ func (r *RcloneStorer) Stat(ctx context.Context, key string) (*ObjectInfo, error
|
||||
if errors.Is(err, fs.ErrorObjectNotFound) {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
|
||||
if errors.Is(err, fs.ErrorDirNotFound) {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("getting object: %w", err)
|
||||
}
|
||||
|
||||
@@ -142,9 +147,11 @@ func (r *RcloneStorer) Delete(ctx context.Context, key string) error {
|
||||
if errors.Is(err, fs.ErrorObjectNotFound) {
|
||||
return nil // Match S3 behavior: no error if doesn't exist
|
||||
}
|
||||
|
||||
if errors.Is(err, fs.ErrorDirNotFound) {
|
||||
return nil
|
||||
}
|
||||
|
||||
return fmt.Errorf("getting object: %w", err)
|
||||
}
|
||||
|
||||
@@ -209,6 +216,7 @@ func (r *RcloneStorer) Info() StorageInfo {
|
||||
if r.path != "" {
|
||||
location += ":" + r.path
|
||||
}
|
||||
|
||||
return StorageInfo{
|
||||
Type: "rclone",
|
||||
Location: location,
|
||||
@@ -227,10 +235,12 @@ func (pr *progressReader) Read(p []byte) (int, error) {
|
||||
if n > 0 {
|
||||
pr.read += int64(n)
|
||||
if pr.callback != nil {
|
||||
if callbackErr := pr.callback(pr.read); callbackErr != nil {
|
||||
callbackErr := pr.callback(pr.read)
|
||||
if callbackErr != nil {
|
||||
return n, callbackErr
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return n, err
|
||||
}
|
||||
|
||||
@@ -30,6 +30,7 @@ func (s *S3Storer) PutWithProgress(ctx context.Context, key string, data io.Read
|
||||
if progress != nil {
|
||||
s3Progress = s3.ProgressCallback(progress)
|
||||
}
|
||||
|
||||
return s.client.PutObjectWithProgress(ctx, key, data, size, s3Progress)
|
||||
}
|
||||
|
||||
@@ -44,6 +45,7 @@ func (s *S3Storer) Stat(ctx context.Context, key string) (*ObjectInfo, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &ObjectInfo{
|
||||
Key: info.Key,
|
||||
Size: info.Size,
|
||||
@@ -65,6 +67,7 @@ func (s *S3Storer) ListStream(ctx context.Context, prefix string) <-chan ObjectI
|
||||
ch := make(chan ObjectInfo)
|
||||
go func() {
|
||||
defer close(ch)
|
||||
|
||||
for info := range s.client.ListObjectsStream(ctx, prefix, false) {
|
||||
ch <- ObjectInfo{
|
||||
Key: info.Key,
|
||||
@@ -73,6 +76,7 @@ func (s *S3Storer) ListStream(ctx context.Context, prefix string) <-chan ObjectI
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
return ch
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"strings"
|
||||
@@ -24,15 +25,16 @@ type StorageURL struct {
|
||||
// - rclone://remote/path/to/backups
|
||||
func ParseStorageURL(rawURL string) (*StorageURL, error) {
|
||||
if rawURL == "" {
|
||||
return nil, fmt.Errorf("storage URL is empty")
|
||||
return nil, errors.New("storage URL is empty")
|
||||
}
|
||||
|
||||
// Handle file:// URLs
|
||||
if strings.HasPrefix(rawURL, "file://") {
|
||||
path := strings.TrimPrefix(rawURL, "file://")
|
||||
if after, ok := strings.CutPrefix(rawURL, "file://"); ok {
|
||||
path := after
|
||||
if path == "" {
|
||||
return nil, fmt.Errorf("file URL path is empty")
|
||||
return nil, errors.New("file URL path is empty")
|
||||
}
|
||||
|
||||
return &StorageURL{
|
||||
Scheme: "file",
|
||||
Prefix: path,
|
||||
@@ -48,12 +50,13 @@ func ParseStorageURL(rawURL string) (*StorageURL, error) {
|
||||
|
||||
bucket := u.Host
|
||||
if bucket == "" {
|
||||
return nil, fmt.Errorf("s3 URL missing bucket name")
|
||||
return nil, errors.New("s3 URL missing bucket name")
|
||||
}
|
||||
|
||||
prefix := strings.TrimPrefix(u.Path, "/")
|
||||
|
||||
query := u.Query()
|
||||
|
||||
useSSL := true
|
||||
if query.Get("ssl") == "false" {
|
||||
useSSL = false
|
||||
@@ -78,7 +81,7 @@ func ParseStorageURL(rawURL string) (*StorageURL, error) {
|
||||
|
||||
remote := u.Host
|
||||
if remote == "" {
|
||||
return nil, fmt.Errorf("rclone URL missing remote name")
|
||||
return nil, errors.New("rclone URL missing remote name")
|
||||
}
|
||||
|
||||
path := strings.TrimPrefix(u.Path, "/")
|
||||
@@ -90,29 +93,32 @@ func ParseStorageURL(rawURL string) (*StorageURL, error) {
|
||||
}, nil
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("unsupported URL scheme: must start with s3://, file://, or rclone://")
|
||||
return nil, errors.New("unsupported URL scheme: must start with s3://, file://, or rclone://")
|
||||
}
|
||||
|
||||
// String returns a human-readable representation of the storage URL.
|
||||
func (u *StorageURL) String() string {
|
||||
switch u.Scheme {
|
||||
case "file":
|
||||
return fmt.Sprintf("file://%s", u.Prefix)
|
||||
return "file://" + u.Prefix
|
||||
case "s3":
|
||||
endpoint := u.Endpoint
|
||||
if endpoint == "" {
|
||||
endpoint = "s3.amazonaws.com"
|
||||
}
|
||||
|
||||
if u.Prefix != "" {
|
||||
return fmt.Sprintf("s3://%s/%s (endpoint: %s)", u.Bucket, u.Prefix, endpoint)
|
||||
}
|
||||
|
||||
return fmt.Sprintf("s3://%s (endpoint: %s)", u.Bucket, endpoint)
|
||||
case "rclone":
|
||||
if u.Prefix != "" {
|
||||
return fmt.Sprintf("rclone://%s/%s", u.RcloneRemote, u.Prefix)
|
||||
}
|
||||
return fmt.Sprintf("rclone://%s", u.RcloneRemote)
|
||||
|
||||
return "rclone://" + u.RcloneRemote
|
||||
default:
|
||||
return fmt.Sprintf("%s://?", u.Scheme)
|
||||
return u.Scheme + "://?"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -24,6 +24,7 @@ func ParseFileID(s string) (FileID, error) {
|
||||
if err != nil {
|
||||
return FileID{}, err
|
||||
}
|
||||
|
||||
return FileID(id), nil
|
||||
}
|
||||
|
||||
@@ -38,13 +39,15 @@ func (id FileID) Value() (driver.Value, error) {
|
||||
}
|
||||
|
||||
// Scan implements sql.Scanner for database deserialization.
|
||||
func (id *FileID) Scan(src interface{}) error {
|
||||
func (id *FileID) Scan(src any) error {
|
||||
if src == nil {
|
||||
*id = FileID{}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
var s string
|
||||
|
||||
switch v := src.(type) {
|
||||
case string:
|
||||
s = v
|
||||
@@ -58,7 +61,9 @@ func (id *FileID) Scan(src interface{}) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid FileID: %w", err)
|
||||
}
|
||||
|
||||
*id = FileID(parsed)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -77,6 +82,7 @@ func ParseBlobID(s string) (BlobID, error) {
|
||||
if err != nil {
|
||||
return BlobID{}, err
|
||||
}
|
||||
|
||||
return BlobID(id), nil
|
||||
}
|
||||
|
||||
@@ -91,13 +97,15 @@ func (id BlobID) Value() (driver.Value, error) {
|
||||
}
|
||||
|
||||
// Scan implements sql.Scanner for database deserialization.
|
||||
func (id *BlobID) Scan(src interface{}) error {
|
||||
func (id *BlobID) Scan(src any) error {
|
||||
if src == nil {
|
||||
*id = BlobID{}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
var s string
|
||||
|
||||
switch v := src.(type) {
|
||||
case string:
|
||||
s = v
|
||||
@@ -111,7 +119,9 @@ func (id *BlobID) Scan(src interface{}) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid BlobID: %w", err)
|
||||
}
|
||||
|
||||
*id = BlobID(parsed)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -94,10 +94,12 @@ func shouldColor(w io.Writer) bool {
|
||||
if os.Getenv("NO_COLOR") != "" {
|
||||
return false
|
||||
}
|
||||
|
||||
f, ok := w.(*os.File)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
|
||||
return term.IsTerminal(int(f.Fd()))
|
||||
}
|
||||
|
||||
@@ -106,67 +108,73 @@ func (w *Writer) paint(color, s string) string {
|
||||
if !w.color {
|
||||
return s
|
||||
}
|
||||
|
||||
return color + s + ansiReset
|
||||
}
|
||||
|
||||
// ───────────────────────── message methods ─────────────────────────
|
||||
|
||||
// Begin prints an operation-start line, left-aligned with a white marker.
|
||||
func (w *Writer) Begin(format string, args ...any) {
|
||||
// Beginf prints an operation-start line, left-aligned with a white marker.
|
||||
func (w *Writer) Beginf(format string, args ...any) {
|
||||
if w.quiet {
|
||||
return
|
||||
}
|
||||
|
||||
w.emit(ansiWhite, Marker, "", format, args)
|
||||
}
|
||||
|
||||
// Complete prints an operation-completion line in green, left-aligned.
|
||||
func (w *Writer) Complete(format string, args ...any) {
|
||||
// Completef prints an operation-completion line in green, left-aligned.
|
||||
func (w *Writer) Completef(format string, args ...any) {
|
||||
if w.quiet {
|
||||
return
|
||||
}
|
||||
|
||||
w.emit(ansiGreen, Marker, ansiGreen, format, args)
|
||||
}
|
||||
|
||||
// Info prints a neutral status line, left-aligned with a white marker.
|
||||
func (w *Writer) Info(format string, args ...any) {
|
||||
// Infof prints a neutral status line, left-aligned with a white marker.
|
||||
func (w *Writer) Infof(format string, args ...any) {
|
||||
if w.quiet {
|
||||
return
|
||||
}
|
||||
|
||||
w.emit(ansiWhite, Marker, "", format, args)
|
||||
}
|
||||
|
||||
// Notice prints an attention-worthy informational line, marker in cyan.
|
||||
func (w *Writer) Notice(format string, args ...any) {
|
||||
// Noticef prints an attention-worthy informational line, marker in cyan.
|
||||
func (w *Writer) Noticef(format string, args ...any) {
|
||||
if w.quiet {
|
||||
return
|
||||
}
|
||||
|
||||
w.emit(ansiCyan, Marker, "", format, args)
|
||||
}
|
||||
|
||||
// Warning prints "⚠️ Warning: " in orange/yellow followed by the message.
|
||||
func (w *Writer) Warning(format string, args ...any) {
|
||||
// Warningf prints "⚠️ Warning: " in orange/yellow followed by the message.
|
||||
func (w *Writer) Warningf(format string, args ...any) {
|
||||
w.warnings++
|
||||
prefix := "⚠️ " + w.paint(ansiYellow+ansiBold, "Warning: ")
|
||||
_, _ = fmt.Fprintln(w.out, prefix+fmt.Sprintf(format, args...))
|
||||
}
|
||||
|
||||
// Error prints "🛑 ERROR: " in red followed by the message. Goes to the
|
||||
// Errorf prints "🛑 ERROR: " in red followed by the message. Goes to the
|
||||
// same writer as everything else; callers that want stderr should
|
||||
// construct a separate Writer for it.
|
||||
func (w *Writer) Error(format string, args ...any) {
|
||||
func (w *Writer) Errorf(format string, args ...any) {
|
||||
w.errors++
|
||||
prefix := "🛑 " + w.paint(ansiRed+ansiBold, "ERROR: ")
|
||||
_, _ = fmt.Fprintln(w.out, prefix+fmt.Sprintf(format, args...))
|
||||
}
|
||||
|
||||
// Detail prints an indented continuation line under a preceding Complete
|
||||
// Detailf prints an indented continuation line under a preceding Completef
|
||||
// (or other top-level message). Marker " 》" (white) at column 2.
|
||||
// Distinct from Progress (semantically a "heartbeat") in usage but
|
||||
// Distinct from Progressf (semantically a "heartbeat") in usage but
|
||||
// visually identical.
|
||||
func (w *Writer) Detail(format string, args ...any) {
|
||||
func (w *Writer) Detailf(format string, args ...any) {
|
||||
if w.quiet {
|
||||
return
|
||||
}
|
||||
|
||||
w.emit(ansiWhite, " "+Marker, "", format, args)
|
||||
}
|
||||
|
||||
@@ -176,24 +184,27 @@ func (w *Writer) WarningCount() int { return w.warnings }
|
||||
// ErrorCount returns the number of Error() calls this writer has emitted.
|
||||
func (w *Writer) ErrorCount() int { return w.errors }
|
||||
|
||||
// Progress prints an indented heartbeat / per-item update, marker in white.
|
||||
func (w *Writer) Progress(format string, args ...any) {
|
||||
// Progressf prints an indented heartbeat / per-item update, marker in white.
|
||||
func (w *Writer) Progressf(format string, args ...any) {
|
||||
if w.quiet {
|
||||
return
|
||||
}
|
||||
|
||||
w.emit(ansiWhite, " "+Marker, "", format, args)
|
||||
}
|
||||
|
||||
// Banner prints a line with no marker, left-aligned. Bold when color
|
||||
// Bannerf prints a line with no marker, left-aligned. Bold when color
|
||||
// is enabled. Used for the application startup banner only.
|
||||
func (w *Writer) Banner(format string, args ...any) {
|
||||
func (w *Writer) Bannerf(format string, args ...any) {
|
||||
if w.quiet {
|
||||
return
|
||||
}
|
||||
|
||||
body := fmt.Sprintf(format, args...)
|
||||
if w.color {
|
||||
body = ansiBold + body + ansiReset
|
||||
}
|
||||
|
||||
_, _ = fmt.Fprintln(w.out, body)
|
||||
}
|
||||
|
||||
@@ -204,6 +215,7 @@ func (w *Writer) emit(prefixColor, prefix, bodyColor, format string, args []any)
|
||||
if bodyColor != "" {
|
||||
body = w.paint(bodyColor, body)
|
||||
}
|
||||
|
||||
_, _ = fmt.Fprintln(w.out, w.paint(prefixColor, prefix)+" "+body)
|
||||
}
|
||||
|
||||
@@ -219,6 +231,7 @@ func (w *Writer) Hex(s string) string {
|
||||
if len(s) > 12 {
|
||||
short = s[:12] + "..."
|
||||
}
|
||||
|
||||
return w.paint(ansiCyan, short)
|
||||
}
|
||||
|
||||
@@ -244,8 +257,11 @@ func (w *Writer) Speed(bytesPerSec float64) string {
|
||||
if bytesPerSec <= 0 {
|
||||
return w.paint(ansiMagenta, "N/A")
|
||||
}
|
||||
|
||||
bitsPerSec := bytesPerSec * 8
|
||||
|
||||
var s string
|
||||
|
||||
switch {
|
||||
case bitsPerSec >= 1e9:
|
||||
s = fmt.Sprintf("%.1f Gbit/sec", bitsPerSec/1e9)
|
||||
@@ -256,6 +272,7 @@ func (w *Writer) Speed(bytesPerSec float64) string {
|
||||
default:
|
||||
s = fmt.Sprintf("%.0f bit/sec", bitsPerSec)
|
||||
}
|
||||
|
||||
return w.paint(ansiMagenta, s)
|
||||
}
|
||||
|
||||
@@ -270,10 +287,12 @@ func (w *Writer) Duration(d time.Duration) string {
|
||||
// displayed in the process's local zone.
|
||||
func (w *Writer) Time(t time.Time) string {
|
||||
t = t.Local()
|
||||
|
||||
now := time.Now()
|
||||
if t.Year() == now.Year() && t.YearDay() == now.YearDay() {
|
||||
return w.paint(ansiYellow, t.Format("15:04:05"))
|
||||
}
|
||||
|
||||
return w.paint(ansiYellow, t.Format("2006-01-02 15:04:05"))
|
||||
}
|
||||
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
|
||||
func newTestWriter(color bool) (*Writer, *bytes.Buffer) {
|
||||
buf := &bytes.Buffer{}
|
||||
|
||||
return NewWithColor(buf, color), buf
|
||||
}
|
||||
|
||||
@@ -18,21 +19,22 @@ func TestMessageMethodsPlain(t *testing.T) {
|
||||
fn func(*Writer)
|
||||
want string
|
||||
}{
|
||||
{"Begin", func(w *Writer) { w.Begin("starting %s", "thing") }, "》 starting thing\n"},
|
||||
{"Complete", func(w *Writer) { w.Complete("done %s", "thing") }, "》 done thing\n"},
|
||||
{"Info", func(w *Writer) { w.Info("status") }, "》 status\n"},
|
||||
{"Notice", func(w *Writer) { w.Notice("note") }, "》 note\n"},
|
||||
{"Warning", func(w *Writer) { w.Warning("oops") }, "⚠️ Warning: oops\n"},
|
||||
{"Error", func(w *Writer) { w.Error("boom") }, "🛑 ERROR: boom\n"},
|
||||
{"Progress", func(w *Writer) { w.Progress("p") }, " 》 p\n"},
|
||||
{"Detail", func(w *Writer) { w.Detail("d") }, " 》 d\n"},
|
||||
{"Banner", func(w *Writer) { w.Banner("hello") }, "hello\n"}, // plain mode, no bold
|
||||
{"Begin", func(w *Writer) { w.Beginf("starting %s", "thing") }, "》 starting thing\n"},
|
||||
{"Complete", func(w *Writer) { w.Completef("done %s", "thing") }, "》 done thing\n"},
|
||||
{"Info", func(w *Writer) { w.Infof("status") }, "》 status\n"},
|
||||
{"Notice", func(w *Writer) { w.Noticef("note") }, "》 note\n"},
|
||||
{"Warning", func(w *Writer) { w.Warningf("oops") }, "⚠️ Warning: oops\n"},
|
||||
{"Error", func(w *Writer) { w.Errorf("boom") }, "🛑 ERROR: boom\n"},
|
||||
{"Progress", func(w *Writer) { w.Progressf("p") }, " 》 p\n"},
|
||||
{"Detail", func(w *Writer) { w.Detailf("d") }, " 》 d\n"},
|
||||
{"Banner", func(w *Writer) { w.Bannerf("hello") }, "hello\n"}, // plain mode, no bold
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.method, func(t *testing.T) {
|
||||
w, buf := newTestWriter(false)
|
||||
tt.fn(w)
|
||||
|
||||
if got := buf.String(); got != tt.want {
|
||||
t.Errorf("got %q, want %q", got, tt.want)
|
||||
}
|
||||
@@ -45,13 +47,16 @@ func TestWarningErrorCounters(t *testing.T) {
|
||||
if w.WarningCount() != 0 || w.ErrorCount() != 0 {
|
||||
t.Fatalf("expected fresh writer to have zero counts")
|
||||
}
|
||||
w.Info("normal")
|
||||
w.Warning("first warn")
|
||||
w.Warning("second warn")
|
||||
w.Error("only error")
|
||||
|
||||
w.Infof("normal")
|
||||
w.Warningf("first warn")
|
||||
w.Warningf("second warn")
|
||||
w.Errorf("only error")
|
||||
|
||||
if got, want := w.WarningCount(), 2; got != want {
|
||||
t.Errorf("WarningCount: got %d, want %d", got, want)
|
||||
}
|
||||
|
||||
if got, want := w.ErrorCount(), 1; got != want {
|
||||
t.Errorf("ErrorCount: got %d, want %d", got, want)
|
||||
}
|
||||
@@ -59,11 +64,13 @@ func TestWarningErrorCounters(t *testing.T) {
|
||||
|
||||
func TestColorOutputContainsANSI(t *testing.T) {
|
||||
w, buf := newTestWriter(true)
|
||||
w.Error("boom")
|
||||
w.Errorf("boom")
|
||||
|
||||
out := buf.String()
|
||||
if !strings.Contains(out, "\033[") {
|
||||
t.Errorf("expected ANSI escapes in color output, got %q", out)
|
||||
}
|
||||
|
||||
if !strings.Contains(out, "ERROR: ") {
|
||||
t.Errorf("expected 'ERROR: ' text in output, got %q", out)
|
||||
}
|
||||
@@ -71,7 +78,8 @@ func TestColorOutputContainsANSI(t *testing.T) {
|
||||
|
||||
func TestBannerBoldWhenColor(t *testing.T) {
|
||||
w, buf := newTestWriter(true)
|
||||
w.Banner("hello")
|
||||
w.Bannerf("hello")
|
||||
|
||||
out := buf.String()
|
||||
if !strings.Contains(out, "\033[1m") {
|
||||
t.Errorf("expected bold ANSI escape in colored Banner output, got %q", out)
|
||||
@@ -84,18 +92,23 @@ func TestValueFormattersPlain(t *testing.T) {
|
||||
if got := w.Hex("0123456789abcdef0123"); got != "0123456789ab..." {
|
||||
t.Errorf("Hex long: got %q", got)
|
||||
}
|
||||
|
||||
if got := w.Hex("short"); got != "short" {
|
||||
t.Errorf("Hex short: got %q", got)
|
||||
}
|
||||
|
||||
if got := w.Size(1024); got != "1.0 kB" {
|
||||
t.Errorf("Size: got %q", got)
|
||||
}
|
||||
|
||||
if got := w.Duration(90 * time.Second); got != "1m30s" {
|
||||
t.Errorf("Duration: got %q", got)
|
||||
}
|
||||
|
||||
if got := w.Count(12345); got != "12,345" {
|
||||
t.Errorf("Count: got %q", got)
|
||||
}
|
||||
|
||||
if got := w.Percent(12.34); got != "12.3%" {
|
||||
t.Errorf("Percent: got %q", got)
|
||||
}
|
||||
@@ -104,9 +117,11 @@ func TestValueFormattersPlain(t *testing.T) {
|
||||
if got := w.Speed(0); got != "N/A" {
|
||||
t.Errorf("Speed(0): got %q, want N/A", got)
|
||||
}
|
||||
|
||||
if got := w.Speed(125_000_000); got != "1.0 Gbit/sec" { // 1 Gbit/s = 125 MB/s
|
||||
t.Errorf("Speed(125e6): got %q", got)
|
||||
}
|
||||
|
||||
if got := w.Speed(125_000); got != "1 Mbit/sec" {
|
||||
t.Errorf("Speed(125e3): got %q", got)
|
||||
}
|
||||
@@ -116,6 +131,7 @@ func TestValueFormattersPlain(t *testing.T) {
|
||||
if got := w.Time(today); got != "14:30:45" {
|
||||
t.Errorf("Time today: got %q, want 14:30:45", got)
|
||||
}
|
||||
|
||||
other := time.Date(2030, 1, 2, 3, 4, 5, 0, time.Local)
|
||||
if got := w.Time(other); got != "2030-01-02 03:04:05" {
|
||||
t.Errorf("Time other day: got %q", got)
|
||||
@@ -124,10 +140,12 @@ func TestValueFormattersPlain(t *testing.T) {
|
||||
|
||||
func TestValueFormattersColored(t *testing.T) {
|
||||
w, _ := newTestWriter(true)
|
||||
|
||||
hex := w.Hex("0123456789abcdef0123")
|
||||
if !strings.Contains(hex, "\033[") {
|
||||
t.Errorf("expected ANSI in colored Hex output, got %q", hex)
|
||||
}
|
||||
|
||||
if !strings.Contains(hex, "0123456789ab") {
|
||||
t.Errorf("expected hex content in output, got %q", hex)
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"time"
|
||||
@@ -26,9 +27,10 @@ type hashVerifyReader struct {
|
||||
|
||||
func (h *hashVerifyReader) Read(p []byte) (int, error) {
|
||||
n, err := h.reader.Read(p)
|
||||
if err == io.EOF {
|
||||
if errors.Is(err, io.EOF) {
|
||||
h.done = true
|
||||
}
|
||||
|
||||
return n, err
|
||||
}
|
||||
|
||||
@@ -41,6 +43,7 @@ func (h *hashVerifyReader) Close() error {
|
||||
firstHash := h.reader.Sum256()
|
||||
secondHasher := sha256.New()
|
||||
secondHasher.Write(firstHash)
|
||||
|
||||
actualHashHex := hex.EncodeToString(secondHasher.Sum(nil))
|
||||
if actualHashHex != h.blobHash {
|
||||
return fmt.Errorf("blob hash mismatch: expected %s, got %s", h.blobHash[:16], actualHashHex[:16])
|
||||
@@ -50,6 +53,7 @@ func (h *hashVerifyReader) Close() error {
|
||||
if readerErr != nil {
|
||||
return readerErr
|
||||
}
|
||||
|
||||
return fetcherErr
|
||||
}
|
||||
|
||||
@@ -66,6 +70,7 @@ func (v *Vaultik) FetchAndDecryptBlob(ctx context.Context, blobHash string, expe
|
||||
reader, err := blobgen.NewReader(rc, identity)
|
||||
if err != nil {
|
||||
_ = rc.Close()
|
||||
|
||||
return nil, fmt.Errorf("creating blob reader: %w", err)
|
||||
}
|
||||
|
||||
@@ -86,6 +91,7 @@ func (v *Vaultik) FetchBlob(ctx context.Context, blobHash string, expectedSize i
|
||||
t0 := time.Now()
|
||||
rc, err := v.Storage.Get(ctx, blobPath)
|
||||
getDur := time.Since(t0)
|
||||
|
||||
if err != nil {
|
||||
return nil, 0, fmt.Errorf("downloading blob %s: %w", blobHash[:16], err)
|
||||
}
|
||||
@@ -93,8 +99,10 @@ func (v *Vaultik) FetchBlob(ctx context.Context, blobHash string, expectedSize i
|
||||
t0 = time.Now()
|
||||
info, err := v.Storage.Stat(ctx, blobPath)
|
||||
statDur := time.Since(t0)
|
||||
|
||||
if err != nil {
|
||||
_ = rc.Close()
|
||||
|
||||
return nil, 0, fmt.Errorf("stat blob %s: %w", blobHash[:16], err)
|
||||
}
|
||||
|
||||
|
||||
@@ -24,17 +24,22 @@ func TestFetchAndDecryptBlobVerifiesHash(t *testing.T) {
|
||||
|
||||
// Create test data and encrypt it using blobgen.Writer
|
||||
plaintext := []byte("hello world test data for blob hash verification")
|
||||
|
||||
var encBuf bytes.Buffer
|
||||
|
||||
writer, err := blobgen.NewWriter(&encBuf, 1, []string{identity.Recipient().String()})
|
||||
if err != nil {
|
||||
t.Fatalf("creating blobgen writer: %v", err)
|
||||
}
|
||||
|
||||
if _, err := writer.Write(plaintext); err != nil {
|
||||
t.Fatalf("writing plaintext: %v", err)
|
||||
}
|
||||
|
||||
if err := writer.Close(); err != nil {
|
||||
t.Fatalf("closing writer: %v", err)
|
||||
}
|
||||
|
||||
encryptedData := encBuf.Bytes()
|
||||
|
||||
// Compute correct double-SHA-256 hash of the plaintext (matches blobgen.Writer.Sum256)
|
||||
@@ -51,6 +56,7 @@ func TestFetchAndDecryptBlobVerifiesHash(t *testing.T) {
|
||||
// Set up mock storage with the blob at the correct path
|
||||
mockStorage := NewMockStorer()
|
||||
blobPath := "blobs/" + correctHash[:2] + "/" + correctHash[2:4] + "/" + correctHash
|
||||
|
||||
mockStorage.mu.Lock()
|
||||
mockStorage.data[blobPath] = encryptedData
|
||||
mockStorage.mu.Unlock()
|
||||
@@ -63,13 +69,16 @@ func TestFetchAndDecryptBlobVerifiesHash(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("expected success, got error: %v", err)
|
||||
}
|
||||
|
||||
data, err := io.ReadAll(rc)
|
||||
if err != nil {
|
||||
t.Fatalf("reading stream: %v", err)
|
||||
}
|
||||
|
||||
if err := rc.Close(); err != nil {
|
||||
t.Fatalf("close (hash verification) failed: %v", err)
|
||||
}
|
||||
|
||||
if !bytes.Equal(data, plaintext) {
|
||||
t.Fatalf("decrypted data mismatch: got %q, want %q", data, plaintext)
|
||||
}
|
||||
@@ -79,6 +88,7 @@ func TestFetchAndDecryptBlobVerifiesHash(t *testing.T) {
|
||||
// Use a fake hash that doesn't match the actual plaintext
|
||||
fakeHash := strings.Repeat("ab", 32) // 64 hex chars
|
||||
fakePath := "blobs/" + fakeHash[:2] + "/" + fakeHash[2:4] + "/" + fakeHash
|
||||
|
||||
mockStorage.mu.Lock()
|
||||
mockStorage.data[fakePath] = encryptedData
|
||||
mockStorage.mu.Unlock()
|
||||
@@ -89,10 +99,12 @@ func TestFetchAndDecryptBlobVerifiesHash(t *testing.T) {
|
||||
}
|
||||
// Read all data — hash is verified on Close
|
||||
_, _ = io.ReadAll(rc)
|
||||
|
||||
err = rc.Close()
|
||||
if err == nil {
|
||||
t.Fatal("expected error for mismatched hash, got nil")
|
||||
}
|
||||
|
||||
if !strings.Contains(err.Error(), "hash mismatch") {
|
||||
t.Fatalf("expected hash mismatch error, got: %v", err)
|
||||
}
|
||||
|
||||
@@ -53,6 +53,7 @@ func newBlobDiskCache(maxBytes int64) (*blobDiskCache, error) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("creating blob cache dir: %w", err)
|
||||
}
|
||||
|
||||
return &blobDiskCache{
|
||||
dir: dir,
|
||||
maxBytes: maxBytes,
|
||||
@@ -70,21 +71,25 @@ func (c *blobDiskCache) unlink(e *blobDiskCacheEntry) {
|
||||
} else {
|
||||
c.head = e.next
|
||||
}
|
||||
|
||||
if e.next != nil {
|
||||
e.next.prev = e.prev
|
||||
} else {
|
||||
c.tail = e.prev
|
||||
}
|
||||
|
||||
e.prev = nil
|
||||
e.next = nil
|
||||
}
|
||||
|
||||
func (c *blobDiskCache) pushFront(e *blobDiskCacheEntry) {
|
||||
e.prev = nil
|
||||
|
||||
e.next = c.head
|
||||
if c.head != nil {
|
||||
c.head.prev = e
|
||||
}
|
||||
|
||||
c.head = e
|
||||
if c.tail == nil {
|
||||
c.tail = e
|
||||
@@ -95,6 +100,7 @@ func (c *blobDiskCache) evictLRU() {
|
||||
if c.tail == nil {
|
||||
return
|
||||
}
|
||||
|
||||
victim := c.tail
|
||||
c.unlink(victim)
|
||||
delete(c.items, victim.key)
|
||||
@@ -121,7 +127,8 @@ func (c *blobDiskCache) Put(key string, data []byte) error {
|
||||
delete(c.items, key)
|
||||
}
|
||||
|
||||
if err := os.WriteFile(c.path(key), data, 0600); err != nil {
|
||||
err := os.WriteFile(c.path(key), data, 0600)
|
||||
if err != nil {
|
||||
return fmt.Errorf("writing blob to cache: %w", err)
|
||||
}
|
||||
|
||||
@@ -163,14 +170,19 @@ func (c *blobDiskCache) PutFromReader(key string, r io.Reader) (int64, error) {
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("creating cache file: %w", err)
|
||||
}
|
||||
|
||||
written, copyErr := io.Copy(f, r)
|
||||
closeErr := f.Close()
|
||||
|
||||
if copyErr != nil {
|
||||
_ = os.Remove(c.path(key))
|
||||
|
||||
return written, fmt.Errorf("streaming to cache file: %w", copyErr)
|
||||
}
|
||||
|
||||
if closeErr != nil {
|
||||
_ = os.Remove(c.path(key))
|
||||
|
||||
return written, fmt.Errorf("closing cache file: %w", closeErr)
|
||||
}
|
||||
|
||||
@@ -182,6 +194,7 @@ func (c *blobDiskCache) PutFromReader(key string, r io.Reader) (int64, error) {
|
||||
// so this branch is effectively unreachable there.
|
||||
if written > c.maxBytes {
|
||||
_ = os.Remove(c.path(key))
|
||||
|
||||
return written, nil
|
||||
}
|
||||
|
||||
@@ -205,11 +218,14 @@ func (c *blobDiskCache) PutFromReader(key string, r io.Reader) (int64, error) {
|
||||
func (c *blobDiskCache) Get(key string) ([]byte, bool) {
|
||||
c.mu.Lock()
|
||||
c.getCalls++
|
||||
|
||||
e, ok := c.items[key]
|
||||
if !ok {
|
||||
c.mu.Unlock()
|
||||
|
||||
return nil, false
|
||||
}
|
||||
|
||||
c.unlink(e)
|
||||
c.pushFront(e)
|
||||
c.mu.Unlock()
|
||||
@@ -223,8 +239,10 @@ func (c *blobDiskCache) Get(key string) ([]byte, bool) {
|
||||
c.curBytes -= e.size
|
||||
}
|
||||
c.mu.Unlock()
|
||||
|
||||
return nil, false
|
||||
}
|
||||
|
||||
return data, true
|
||||
}
|
||||
|
||||
@@ -232,15 +250,20 @@ func (c *blobDiskCache) Get(key string) ([]byte, bool) {
|
||||
func (c *blobDiskCache) ReadAt(key string, offset, length int64) ([]byte, error) {
|
||||
c.mu.Lock()
|
||||
c.readAtCalls++
|
||||
|
||||
e, ok := c.items[key]
|
||||
if !ok {
|
||||
c.mu.Unlock()
|
||||
|
||||
return nil, fmt.Errorf("key %q not in cache", key)
|
||||
}
|
||||
|
||||
if offset+length > e.size {
|
||||
c.mu.Unlock()
|
||||
|
||||
return nil, fmt.Errorf("read beyond blob size: offset=%d length=%d size=%d", offset, length, e.size)
|
||||
}
|
||||
|
||||
c.unlink(e)
|
||||
c.pushFront(e)
|
||||
c.mu.Unlock()
|
||||
@@ -255,6 +278,7 @@ func (c *blobDiskCache) ReadAt(key string, offset, length int64) ([]byte, error)
|
||||
if _, err := f.ReadAt(buf, offset); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return buf, nil
|
||||
}
|
||||
|
||||
@@ -262,7 +286,9 @@ func (c *blobDiskCache) ReadAt(key string, offset, length int64) ([]byte, error)
|
||||
func (c *blobDiskCache) Has(key string) bool {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
_, ok := c.items[key]
|
||||
|
||||
return ok
|
||||
}
|
||||
|
||||
@@ -272,10 +298,12 @@ func (c *blobDiskCache) Has(key string) bool {
|
||||
func (c *blobDiskCache) Delete(key string) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
e, ok := c.items[key]
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
c.unlink(e)
|
||||
delete(c.items, key)
|
||||
c.curBytes -= e.size
|
||||
@@ -287,10 +315,12 @@ func (c *blobDiskCache) Delete(key string) {
|
||||
func (c *blobDiskCache) Keys() []string {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
keys := make([]string, 0, len(c.items))
|
||||
for k := range c.items {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
|
||||
return keys
|
||||
}
|
||||
|
||||
@@ -298,6 +328,7 @@ func (c *blobDiskCache) Keys() []string {
|
||||
func (c *blobDiskCache) Size() int64 {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
return c.curBytes
|
||||
}
|
||||
|
||||
@@ -305,6 +336,7 @@ func (c *blobDiskCache) Size() int64 {
|
||||
func (c *blobDiskCache) Len() int {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
return len(c.items)
|
||||
}
|
||||
|
||||
@@ -312,6 +344,7 @@ func (c *blobDiskCache) Len() int {
|
||||
func (c *blobDiskCache) GetCalls() int {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
return c.getCalls
|
||||
}
|
||||
|
||||
@@ -319,6 +352,7 @@ func (c *blobDiskCache) GetCalls() int {
|
||||
func (c *blobDiskCache) ReadAtCalls() int {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
return c.readAtCalls
|
||||
}
|
||||
|
||||
@@ -327,6 +361,7 @@ func (c *blobDiskCache) ReadAtCalls() int {
|
||||
func (c *blobDiskCache) PeakLen() int {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
return c.peakLen
|
||||
}
|
||||
|
||||
@@ -334,9 +369,11 @@ func (c *blobDiskCache) PeakLen() int {
|
||||
func (c *blobDiskCache) Close() error {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
c.items = nil
|
||||
c.head = nil
|
||||
c.tail = nil
|
||||
c.curBytes = 0
|
||||
|
||||
return os.RemoveAll(c.dir)
|
||||
}
|
||||
|
||||
@@ -23,6 +23,7 @@ func TestBlobDiskCache_BasicGetPut(t *testing.T) {
|
||||
if !ok {
|
||||
t.Fatal("expected cache hit")
|
||||
}
|
||||
|
||||
if !bytes.Equal(got, data) {
|
||||
t.Fatalf("got %q, want %q", got, data)
|
||||
}
|
||||
@@ -35,15 +36,19 @@ func TestBlobDiskCache_BasicGetPut(t *testing.T) {
|
||||
|
||||
func TestBlobDiskCache_EvictionUnderPressure(t *testing.T) {
|
||||
maxBytes := int64(1000)
|
||||
|
||||
cache, err := newBlobDiskCache(maxBytes)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
defer func() { _ = cache.Close() }()
|
||||
|
||||
for i := 0; i < 5; i++ {
|
||||
for i := range 5 {
|
||||
data := make([]byte, 300)
|
||||
if err := cache.Put(fmt.Sprintf("key%d", i), data); err != nil {
|
||||
|
||||
err := cache.Put(fmt.Sprintf("key%d", i), data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
@@ -55,6 +60,7 @@ func TestBlobDiskCache_EvictionUnderPressure(t *testing.T) {
|
||||
if !cache.Has("key4") {
|
||||
t.Fatal("expected key4 to be cached")
|
||||
}
|
||||
|
||||
if cache.Has("key0") {
|
||||
t.Fatal("expected key0 to be evicted")
|
||||
}
|
||||
@@ -87,6 +93,7 @@ func TestBlobDiskCache_UpdateInPlace(t *testing.T) {
|
||||
if err := cache.Put("key1", []byte("v1")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := cache.Put("key1", []byte("version2")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -95,12 +102,15 @@ func TestBlobDiskCache_UpdateInPlace(t *testing.T) {
|
||||
if !ok {
|
||||
t.Fatal("expected hit")
|
||||
}
|
||||
|
||||
if string(got) != "version2" {
|
||||
t.Fatalf("got %q, want %q", got, "version2")
|
||||
}
|
||||
|
||||
if cache.Len() != 1 {
|
||||
t.Fatalf("expected 1 entry, got %d", cache.Len())
|
||||
}
|
||||
|
||||
if cache.Size() != int64(len("version2")) {
|
||||
t.Fatalf("expected size %d, got %d", len("version2"), cache.Size())
|
||||
}
|
||||
@@ -117,6 +127,7 @@ func TestBlobDiskCache_ReadAt(t *testing.T) {
|
||||
if _, err := rand.Read(data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := cache.Put("blob1", data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -125,6 +136,7 @@ func TestBlobDiskCache_ReadAt(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if !bytes.Equal(chunk, data[100:300]) {
|
||||
t.Fatal("ReadAt returned wrong data")
|
||||
}
|
||||
@@ -149,6 +161,7 @@ func TestBlobDiskCache_Close(t *testing.T) {
|
||||
if err := cache.Put("key1", []byte("data")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := cache.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -165,6 +178,7 @@ func TestBlobDiskCache_LRUOrder(t *testing.T) {
|
||||
if err := cache.Put("a", d); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := cache.Put("b", d); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -180,9 +194,11 @@ func TestBlobDiskCache_LRUOrder(t *testing.T) {
|
||||
if !cache.Has("a") {
|
||||
t.Fatal("expected 'a' to survive")
|
||||
}
|
||||
|
||||
if !cache.Has("c") {
|
||||
t.Fatal("expected 'c' to be present")
|
||||
}
|
||||
|
||||
if cache.Has("b") {
|
||||
t.Fatal("expected 'b' to be evicted")
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package vaultik
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strconv"
|
||||
@@ -29,11 +30,13 @@ func formatBytes(bytes int64) string {
|
||||
if bytes < unit {
|
||||
return fmt.Sprintf("%d B", bytes)
|
||||
}
|
||||
|
||||
div, exp := int64(unit), 0
|
||||
for n := bytes / unit; n >= unit; n /= unit {
|
||||
div *= unit
|
||||
exp++
|
||||
}
|
||||
|
||||
return fmt.Sprintf("%.1f %cB", float64(bytes)/float64(div), "KMGTPE"[exp])
|
||||
}
|
||||
|
||||
@@ -42,11 +45,12 @@ func formatBytes(bytes int64) string {
|
||||
func parseSnapshotTimestamp(snapshotID string) (time.Time, error) {
|
||||
parts := strings.Split(snapshotID, "_")
|
||||
if len(parts) < 2 {
|
||||
return time.Time{}, fmt.Errorf("invalid snapshot ID format: expected hostname_snapshotname_timestamp")
|
||||
return time.Time{}, errors.New("invalid snapshot ID format: expected hostname_snapshotname_timestamp")
|
||||
}
|
||||
|
||||
// Last part is the RFC3339 timestamp
|
||||
timestampStr := parts[len(parts)-1]
|
||||
|
||||
timestamp, err := time.Parse(time.RFC3339, timestampStr)
|
||||
if err != nil {
|
||||
return time.Time{}, fmt.Errorf("invalid timestamp: %w", err)
|
||||
@@ -80,17 +84,20 @@ func parseDuration(s string) (time.Duration, error) {
|
||||
}
|
||||
|
||||
re := regexp.MustCompile(`(\d+)\s*([a-zA-Z]+)`)
|
||||
|
||||
matches := re.FindAllStringSubmatch(s, -1)
|
||||
if len(matches) == 0 {
|
||||
return 0, fmt.Errorf("invalid duration: %q", s)
|
||||
}
|
||||
|
||||
var total time.Duration
|
||||
|
||||
for _, match := range matches {
|
||||
n, err := strconv.Atoi(match[1])
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("invalid number %q: %w", match[1], err)
|
||||
}
|
||||
|
||||
unit := strings.ToLower(match[2])
|
||||
switch unit {
|
||||
case "d", "day", "days":
|
||||
@@ -105,5 +112,6 @@ func parseDuration(s string) (time.Duration, error) {
|
||||
return 0, fmt.Errorf("unknown time unit %q", unit)
|
||||
}
|
||||
}
|
||||
|
||||
return total, nil
|
||||
}
|
||||
|
||||
@@ -61,11 +61,14 @@ func TestParseDuration(t *testing.T) {
|
||||
if err == nil {
|
||||
t.Fatalf("expected error for %q, got %v", tt.input, got)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error for %q: %v", tt.input, err)
|
||||
}
|
||||
|
||||
if got != tt.want {
|
||||
t.Errorf("parseDuration(%q) = %v, want %v", tt.input, got, tt.want)
|
||||
}
|
||||
|
||||
@@ -15,102 +15,122 @@ import (
|
||||
// ShowInfo displays system and configuration information
|
||||
func (v *Vaultik) ShowInfo() error {
|
||||
// System Information
|
||||
v.printfStdout("=== System Information ===\n")
|
||||
v.printfStdout("OS/Architecture: %s/%s\n", runtime.GOOS, runtime.GOARCH)
|
||||
v.printfStdout("Version: %s\n", v.Globals.Version)
|
||||
v.printfStdout("Commit: %s\n", v.Globals.Commit)
|
||||
v.printfStdout("Go Version: %s\n", runtime.Version())
|
||||
v.stdoutf("=== System Information ===\n")
|
||||
v.stdoutf("OS/Architecture: %s/%s\n", runtime.GOOS, runtime.GOARCH)
|
||||
v.stdoutf("Version: %s\n", v.Globals.Version)
|
||||
v.stdoutf("Commit: %s\n", v.Globals.Commit)
|
||||
v.stdoutf("Go Version: %s\n", runtime.Version())
|
||||
v.printlnStdout()
|
||||
|
||||
// Storage Configuration. The backend is selected by storage_url
|
||||
// (s3://, file://, rclone://); the legacy s3.* fields are only
|
||||
// printed when they're actually populated, since the URL scheme
|
||||
// is the primary configuration.
|
||||
v.printfStdout("=== Storage Configuration ===\n")
|
||||
v.stdoutf("=== Storage Configuration ===\n")
|
||||
storageInfo := v.Storage.Info()
|
||||
v.printfStdout("Type: %s\n", storageInfo.Type)
|
||||
v.printfStdout("Location: %s\n", storageInfo.Location)
|
||||
v.stdoutf("Type: %s\n", storageInfo.Type)
|
||||
v.stdoutf("Location: %s\n", storageInfo.Location)
|
||||
|
||||
if v.Config.StorageURL != "" {
|
||||
v.printfStdout("Storage URL: %s\n", v.Config.StorageURL)
|
||||
v.stdoutf("Storage URL: %s\n", v.Config.StorageURL)
|
||||
}
|
||||
|
||||
if v.Config.S3.Bucket != "" {
|
||||
v.printfStdout("S3 Bucket: %s\n", v.Config.S3.Bucket)
|
||||
v.stdoutf("S3 Bucket: %s\n", v.Config.S3.Bucket)
|
||||
}
|
||||
|
||||
if v.Config.S3.Prefix != "" {
|
||||
v.printfStdout("S3 Prefix: %s\n", v.Config.S3.Prefix)
|
||||
v.stdoutf("S3 Prefix: %s\n", v.Config.S3.Prefix)
|
||||
}
|
||||
|
||||
if v.Config.S3.Endpoint != "" {
|
||||
v.printfStdout("S3 Endpoint: %s\n", v.Config.S3.Endpoint)
|
||||
v.stdoutf("S3 Endpoint: %s\n", v.Config.S3.Endpoint)
|
||||
}
|
||||
|
||||
if v.Config.S3.Region != "" {
|
||||
v.printfStdout("S3 Region: %s\n", v.Config.S3.Region)
|
||||
v.stdoutf("S3 Region: %s\n", v.Config.S3.Region)
|
||||
}
|
||||
|
||||
v.printlnStdout()
|
||||
|
||||
// Backup Settings
|
||||
v.printfStdout("=== Backup Settings ===\n")
|
||||
v.stdoutf("=== Backup Settings ===\n")
|
||||
|
||||
// Show configured snapshots
|
||||
v.printfStdout("Snapshots:\n")
|
||||
v.stdoutf("Snapshots:\n")
|
||||
|
||||
for _, name := range v.Config.SnapshotNames() {
|
||||
snap := v.Config.Snapshots[name]
|
||||
v.printfStdout(" %s:\n", name)
|
||||
v.stdoutf(" %s:\n", name)
|
||||
|
||||
for _, path := range snap.Paths {
|
||||
v.printfStdout(" - %s\n", path)
|
||||
v.stdoutf(" - %s\n", path)
|
||||
}
|
||||
|
||||
if len(snap.Exclude) > 0 {
|
||||
v.printfStdout(" exclude: %s\n", strings.Join(snap.Exclude, ", "))
|
||||
v.stdoutf(" exclude: %s\n", strings.Join(snap.Exclude, ", "))
|
||||
}
|
||||
}
|
||||
|
||||
// Global exclude patterns
|
||||
if len(v.Config.Exclude) > 0 {
|
||||
v.printfStdout("Global Exclude: %s\n", strings.Join(v.Config.Exclude, ", "))
|
||||
v.stdoutf("Global Exclude: %s\n", strings.Join(v.Config.Exclude, ", "))
|
||||
}
|
||||
|
||||
v.printfStdout("Compression: zstd level %d\n", v.Config.CompressionLevel)
|
||||
v.printfStdout("Chunk Size: %s\n", humanize.Bytes(uint64(v.Config.ChunkSize)))
|
||||
v.printfStdout("Blob Size Limit: %s\n", humanize.Bytes(uint64(v.Config.BlobSizeLimit)))
|
||||
v.stdoutf("Compression: zstd level %d\n", v.Config.CompressionLevel)
|
||||
v.stdoutf("Chunk Size: %s\n", humanize.Bytes(uint64(v.Config.ChunkSize)))
|
||||
v.stdoutf("Blob Size Limit: %s\n", humanize.Bytes(uint64(v.Config.BlobSizeLimit)))
|
||||
v.printlnStdout()
|
||||
|
||||
// Encryption Configuration
|
||||
v.printfStdout("=== Encryption Configuration ===\n")
|
||||
v.printfStdout("Recipients:\n")
|
||||
v.stdoutf("=== Encryption Configuration ===\n")
|
||||
v.stdoutf("Recipients:\n")
|
||||
|
||||
for _, recipient := range v.Config.AgeRecipients {
|
||||
v.printfStdout(" - %s\n", recipient)
|
||||
v.stdoutf(" - %s\n", recipient)
|
||||
}
|
||||
|
||||
v.printlnStdout()
|
||||
|
||||
// Local Database
|
||||
v.printfStdout("=== Local Database ===\n")
|
||||
v.printfStdout("Index Path: %s\n", v.Config.IndexPath)
|
||||
v.stdoutf("=== Local Database ===\n")
|
||||
v.stdoutf("Index Path: %s\n", v.Config.IndexPath)
|
||||
|
||||
// Check if index file exists and get its size
|
||||
if info, err := v.Fs.Stat(v.Config.IndexPath); err == nil {
|
||||
v.printfStdout("Index Size: %s\n", humanize.Bytes(uint64(info.Size())))
|
||||
v.stdoutf("Index Size: %s\n", humanize.Bytes(uint64(info.Size())))
|
||||
|
||||
// Get snapshot count from database
|
||||
query := `SELECT COUNT(*) FROM snapshots WHERE completed_at IS NOT NULL`
|
||||
|
||||
var snapshotCount int
|
||||
if err := v.DB.Conn().QueryRowContext(v.ctx, query).Scan(&snapshotCount); err == nil {
|
||||
v.printfStdout("Snapshots: %d\n", snapshotCount)
|
||||
|
||||
err := v.DB.Conn().QueryRowContext(v.ctx, query).Scan(&snapshotCount)
|
||||
if err == nil {
|
||||
v.stdoutf("Snapshots: %d\n", snapshotCount)
|
||||
}
|
||||
|
||||
// Get blob count from database
|
||||
query = `SELECT COUNT(*) FROM blobs`
|
||||
|
||||
var blobCount int
|
||||
if err := v.DB.Conn().QueryRowContext(v.ctx, query).Scan(&blobCount); err == nil {
|
||||
v.printfStdout("Blobs: %d\n", blobCount)
|
||||
|
||||
err = v.DB.Conn().QueryRowContext(v.ctx, query).Scan(&blobCount)
|
||||
if err == nil {
|
||||
v.stdoutf("Blobs: %d\n", blobCount)
|
||||
}
|
||||
|
||||
// Get file count from database
|
||||
query = `SELECT COUNT(*) FROM files`
|
||||
|
||||
var fileCount int
|
||||
if err := v.DB.Conn().QueryRowContext(v.ctx, query).Scan(&fileCount); err == nil {
|
||||
v.printfStdout("Files: %d\n", fileCount)
|
||||
|
||||
err = v.DB.Conn().QueryRowContext(v.ctx, query).Scan(&fileCount)
|
||||
if err == nil {
|
||||
v.stdoutf("Files: %d\n", fileCount)
|
||||
}
|
||||
} else {
|
||||
v.printfStdout("Index Size: (not created)\n")
|
||||
v.stdoutf("Index Size: (not created)\n")
|
||||
}
|
||||
|
||||
return nil
|
||||
@@ -153,6 +173,7 @@ type RemoteInfoResult struct {
|
||||
// RemoteInfo displays information about remote storage
|
||||
func (v *Vaultik) RemoteInfo(jsonOutput bool) error {
|
||||
log.Info("Starting remote storage info gathering")
|
||||
|
||||
result := &RemoteInfoResult{}
|
||||
|
||||
storageInfo := v.Storage.Info()
|
||||
@@ -160,11 +181,11 @@ func (v *Vaultik) RemoteInfo(jsonOutput bool) error {
|
||||
result.StorageLocation = storageInfo.Location
|
||||
|
||||
if !jsonOutput {
|
||||
v.printfStdout("=== Remote Storage ===\n")
|
||||
v.printfStdout("Type: %s\n", storageInfo.Type)
|
||||
v.printfStdout("Location: %s\n", storageInfo.Location)
|
||||
v.stdoutf("=== Remote Storage ===\n")
|
||||
v.stdoutf("Type: %s\n", storageInfo.Type)
|
||||
v.stdoutf("Location: %s\n", storageInfo.Location)
|
||||
v.printlnStdout()
|
||||
v.printfStdout("Scanning snapshot metadata...\n")
|
||||
v.stdoutf("Scanning snapshot metadata...\n")
|
||||
}
|
||||
|
||||
snapshotMetadata, snapshotIDs, err := v.collectSnapshotMetadata()
|
||||
@@ -173,7 +194,7 @@ func (v *Vaultik) RemoteInfo(jsonOutput bool) error {
|
||||
}
|
||||
|
||||
if !jsonOutput {
|
||||
v.printfStdout("Downloading %d manifest(s)...\n", len(snapshotIDs))
|
||||
v.stdoutf("Downloading %d manifest(s)...\n", len(snapshotIDs))
|
||||
}
|
||||
|
||||
referencedBlobs := v.collectReferencedBlobsFromManifests(snapshotIDs, snapshotMetadata)
|
||||
@@ -193,10 +214,12 @@ func (v *Vaultik) RemoteInfo(jsonOutput bool) error {
|
||||
if jsonOutput {
|
||||
enc := json.NewEncoder(v.Stdout)
|
||||
enc.SetIndent("", " ")
|
||||
|
||||
return enc.Encode(result)
|
||||
}
|
||||
|
||||
v.printRemoteInfoTable(result)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -214,6 +237,7 @@ func (v *Vaultik) collectSnapshotMetadata() (map[string]*SnapshotMetadataInfo, [
|
||||
if len(parts) < 3 {
|
||||
continue
|
||||
}
|
||||
|
||||
snapshotID := parts[1]
|
||||
|
||||
if _, exists := snapshotMetadata[snapshotID]; !exists {
|
||||
@@ -221,12 +245,14 @@ func (v *Vaultik) collectSnapshotMetadata() (map[string]*SnapshotMetadataInfo, [
|
||||
}
|
||||
|
||||
info := snapshotMetadata[snapshotID]
|
||||
|
||||
filename := parts[2]
|
||||
if strings.HasPrefix(filename, "manifest") {
|
||||
info.ManifestSize = obj.Size
|
||||
} else if strings.HasPrefix(filename, "db") {
|
||||
info.DatabaseSize = obj.Size
|
||||
}
|
||||
|
||||
info.TotalSize = info.ManifestSize + info.DatabaseSize
|
||||
}
|
||||
|
||||
@@ -234,6 +260,7 @@ func (v *Vaultik) collectSnapshotMetadata() (map[string]*SnapshotMetadataInfo, [
|
||||
for id := range snapshotMetadata {
|
||||
snapshotIDs = append(snapshotIDs, id)
|
||||
}
|
||||
|
||||
sort.Strings(snapshotIDs)
|
||||
|
||||
return snapshotMetadata, snapshotIDs, nil
|
||||
@@ -245,26 +272,33 @@ func (v *Vaultik) collectReferencedBlobsFromManifests(snapshotIDs []string, snap
|
||||
|
||||
for _, snapshotID := range snapshotIDs {
|
||||
manifestKey := fmt.Sprintf("metadata/%s/manifest.json.zst", snapshotID)
|
||||
|
||||
reader, err := v.Storage.Get(v.ctx, manifestKey)
|
||||
if err != nil {
|
||||
log.Warn("Failed to get manifest", "snapshot", snapshotID, "error", err)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
manifest, err := snapshot.DecodeManifest(reader)
|
||||
_ = reader.Close()
|
||||
|
||||
if err != nil {
|
||||
log.Warn("Failed to decode manifest", "snapshot", snapshotID, "error", err)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
info := snapshotMetadata[snapshotID]
|
||||
info.BlobCount = manifest.BlobCount
|
||||
|
||||
var blobsSize int64
|
||||
|
||||
for _, blob := range manifest.Blobs {
|
||||
referencedBlobs[blob.Hash] = blob.CompressedSize
|
||||
blobsSize += blob.CompressedSize
|
||||
}
|
||||
|
||||
info.BlobsSize = blobsSize
|
||||
}
|
||||
|
||||
@@ -274,11 +308,13 @@ func (v *Vaultik) collectReferencedBlobsFromManifests(snapshotIDs []string, snap
|
||||
// populateRemoteInfoResult fills in the result's snapshot and referenced blob stats
|
||||
func (v *Vaultik) populateRemoteInfoResult(result *RemoteInfoResult, snapshotMetadata map[string]*SnapshotMetadataInfo, snapshotIDs []string, referencedBlobs map[string]int64) {
|
||||
var totalMetadataSize int64
|
||||
|
||||
for _, id := range snapshotIDs {
|
||||
info := snapshotMetadata[id]
|
||||
result.Snapshots = append(result.Snapshots, *info)
|
||||
totalMetadataSize += info.TotalSize
|
||||
}
|
||||
|
||||
result.TotalMetadataSize = totalMetadataSize
|
||||
result.TotalMetadataCount = len(snapshotIDs)
|
||||
|
||||
@@ -291,7 +327,7 @@ func (v *Vaultik) populateRemoteInfoResult(result *RemoteInfoResult, snapshotMet
|
||||
// scanRemoteBlobStorage lists all blobs on remote and computes orphan stats
|
||||
func (v *Vaultik) scanRemoteBlobStorage(result *RemoteInfoResult, referencedBlobs map[string]int64, jsonOutput bool) error {
|
||||
if !jsonOutput {
|
||||
v.printfStdout("Scanning blobs...\n")
|
||||
v.stdoutf("Scanning blobs...\n")
|
||||
}
|
||||
|
||||
blobCh := v.Storage.ListStream(v.ctx, "blobs/")
|
||||
@@ -301,10 +337,12 @@ func (v *Vaultik) scanRemoteBlobStorage(result *RemoteInfoResult, referencedBlob
|
||||
if obj.Err != nil {
|
||||
return fmt.Errorf("listing blobs: %w", obj.Err)
|
||||
}
|
||||
|
||||
parts := strings.Split(obj.Key, "/")
|
||||
if len(parts) < 4 {
|
||||
continue
|
||||
}
|
||||
|
||||
hash := parts[3]
|
||||
allBlobs[hash] = obj.Size
|
||||
result.TotalBlobCount++
|
||||
@@ -323,14 +361,16 @@ func (v *Vaultik) scanRemoteBlobStorage(result *RemoteInfoResult, referencedBlob
|
||||
|
||||
// printRemoteInfoTable renders the human-readable remote info output
|
||||
func (v *Vaultik) printRemoteInfoTable(result *RemoteInfoResult) {
|
||||
v.printfStdout("\n=== Snapshot Metadata ===\n")
|
||||
v.stdoutf("\n=== Snapshot Metadata ===\n")
|
||||
|
||||
if len(result.Snapshots) == 0 {
|
||||
v.printfStdout("No snapshots found\n")
|
||||
v.stdoutf("No snapshots found\n")
|
||||
} else {
|
||||
v.printfStdout("%-45s %12s %12s %12s %10s %12s\n", "SNAPSHOT", "MANIFEST", "DATABASE", "TOTAL", "BLOBS", "BLOB SIZE")
|
||||
v.printfStdout("%-45s %12s %12s %12s %10s %12s\n", strings.Repeat("-", 45), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 10), strings.Repeat("-", 12))
|
||||
v.stdoutf("%-45s %12s %12s %12s %10s %12s\n", "SNAPSHOT", "MANIFEST", "DATABASE", "TOTAL", "BLOBS", "BLOB SIZE")
|
||||
v.stdoutf("%-45s %12s %12s %12s %10s %12s\n", strings.Repeat("-", 45), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 10), strings.Repeat("-", 12))
|
||||
|
||||
for _, info := range result.Snapshots {
|
||||
v.printfStdout("%-45s %12s %12s %12s %10s %12s\n",
|
||||
v.stdoutf("%-45s %12s %12s %12s %10s %12s\n",
|
||||
truncateString(info.SnapshotID, 45),
|
||||
humanize.Bytes(uint64(info.ManifestSize)),
|
||||
humanize.Bytes(uint64(info.DatabaseSize)),
|
||||
@@ -339,20 +379,21 @@ func (v *Vaultik) printRemoteInfoTable(result *RemoteInfoResult) {
|
||||
humanize.Bytes(uint64(info.BlobsSize)),
|
||||
)
|
||||
}
|
||||
v.printfStdout("%-45s %12s %12s %12s %10s %12s\n", strings.Repeat("-", 45), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 10), strings.Repeat("-", 12))
|
||||
v.printfStdout("%-45s %12s %12s %12s\n", fmt.Sprintf("Total (%d snapshots)", result.TotalMetadataCount), "", "", humanize.Bytes(uint64(result.TotalMetadataSize)))
|
||||
|
||||
v.stdoutf("%-45s %12s %12s %12s %10s %12s\n", strings.Repeat("-", 45), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 10), strings.Repeat("-", 12))
|
||||
v.stdoutf("%-45s %12s %12s %12s\n", fmt.Sprintf("Total (%d snapshots)", result.TotalMetadataCount), "", "", humanize.Bytes(uint64(result.TotalMetadataSize)))
|
||||
}
|
||||
|
||||
v.printfStdout("\n=== Blob Storage ===\n")
|
||||
v.printfStdout("Total blobs on remote: %s (%s)\n",
|
||||
v.stdoutf("\n=== Blob Storage ===\n")
|
||||
v.stdoutf("Total blobs on remote: %s (%s)\n",
|
||||
humanize.Comma(int64(result.TotalBlobCount)), humanize.Bytes(uint64(result.TotalBlobSize)))
|
||||
v.printfStdout("Referenced by snapshots: %s (%s)\n",
|
||||
v.stdoutf("Referenced by snapshots: %s (%s)\n",
|
||||
humanize.Comma(int64(result.ReferencedBlobCount)), humanize.Bytes(uint64(result.ReferencedBlobSize)))
|
||||
v.printfStdout("Orphaned (unreferenced): %s (%s)\n",
|
||||
v.stdoutf("Orphaned (unreferenced): %s (%s)\n",
|
||||
humanize.Comma(int64(result.OrphanedBlobCount)), humanize.Bytes(uint64(result.OrphanedBlobSize)))
|
||||
|
||||
if result.OrphanedBlobCount > 0 {
|
||||
v.printfStdout("\nRun 'vaultik prune' to remove orphaned blobs.\n")
|
||||
v.stdoutf("\nRun 'vaultik prune' to remove orphaned blobs.\n")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -361,8 +402,10 @@ func truncateString(s string, maxLen int) string {
|
||||
if len(s) <= maxLen {
|
||||
return s
|
||||
}
|
||||
|
||||
if maxLen <= 3 {
|
||||
return s[:maxLen]
|
||||
}
|
||||
|
||||
return s[:maxLen-3] + "..."
|
||||
}
|
||||
|
||||
@@ -43,11 +43,14 @@ func (m *MockStorer) Put(ctx context.Context, key string, reader io.Reader) erro
|
||||
defer m.mu.Unlock()
|
||||
|
||||
m.calls = append(m.calls, "Put:"+key)
|
||||
|
||||
data, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
m.data[key] = data
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -60,10 +63,12 @@ func (m *MockStorer) Get(ctx context.Context, key string) (io.ReadCloser, error)
|
||||
defer m.mu.Unlock()
|
||||
|
||||
m.calls = append(m.calls, "Get:"+key)
|
||||
|
||||
data, exists := m.data[key]
|
||||
if !exists {
|
||||
return nil, storage.ErrNotFound
|
||||
}
|
||||
|
||||
return io.NopCloser(bytes.NewReader(data)), nil
|
||||
}
|
||||
|
||||
@@ -72,10 +77,12 @@ func (m *MockStorer) Stat(ctx context.Context, key string) (*storage.ObjectInfo,
|
||||
defer m.mu.Unlock()
|
||||
|
||||
m.calls = append(m.calls, "Stat:"+key)
|
||||
|
||||
data, exists := m.data[key]
|
||||
if !exists {
|
||||
return nil, storage.ErrNotFound
|
||||
}
|
||||
|
||||
return &storage.ObjectInfo{
|
||||
Key: key,
|
||||
Size: int64(len(data)),
|
||||
@@ -88,6 +95,7 @@ func (m *MockStorer) Delete(ctx context.Context, key string) error {
|
||||
|
||||
m.calls = append(m.calls, "Delete:"+key)
|
||||
delete(m.data, key)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -96,12 +104,15 @@ func (m *MockStorer) List(ctx context.Context, prefix string) ([]string, error)
|
||||
defer m.mu.Unlock()
|
||||
|
||||
m.calls = append(m.calls, "List:"+prefix)
|
||||
|
||||
var keys []string
|
||||
|
||||
for key := range m.data {
|
||||
if len(prefix) == 0 || (len(key) >= len(prefix) && key[:len(prefix)] == prefix) {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
}
|
||||
|
||||
return keys, nil
|
||||
}
|
||||
|
||||
@@ -109,6 +120,7 @@ func (m *MockStorer) ListStream(ctx context.Context, prefix string) <-chan stora
|
||||
ch := make(chan storage.ObjectInfo)
|
||||
go func() {
|
||||
defer close(ch)
|
||||
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
@@ -121,6 +133,7 @@ func (m *MockStorer) ListStream(ctx context.Context, prefix string) <-chan stora
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
return ch
|
||||
}
|
||||
|
||||
@@ -138,6 +151,7 @@ func (m *MockStorer) GetCalls() []string {
|
||||
|
||||
calls := make([]string, len(m.calls))
|
||||
copy(calls, m.calls)
|
||||
|
||||
return calls
|
||||
}
|
||||
|
||||
@@ -172,14 +186,16 @@ func TestEndToEndBackup(t *testing.T) {
|
||||
"/home/user/code",
|
||||
}
|
||||
for _, dir := range dirs {
|
||||
if err := fs.MkdirAll(dir, 0755); err != nil {
|
||||
err := fs.MkdirAll(dir, 0755)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create directory %s: %v", dir, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Create test files
|
||||
for path, content := range testFiles {
|
||||
if err := afero.WriteFile(fs, path, []byte(content), 0644); err != nil {
|
||||
err := afero.WriteFile(fs, path, []byte(content), 0644)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create test file %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
@@ -216,9 +232,11 @@ func TestEndToEndBackup(t *testing.T) {
|
||||
|
||||
// Create in-memory database
|
||||
db, err := database.New(ctx, ":memory:")
|
||||
|
||||
require.NoError(t, err)
|
||||
defer func() {
|
||||
if err := db.Close(); err != nil {
|
||||
err := db.Close()
|
||||
if err != nil {
|
||||
t.Errorf("failed to close database: %v", err)
|
||||
}
|
||||
}()
|
||||
@@ -246,6 +264,7 @@ func TestEndToEndBackup(t *testing.T) {
|
||||
VaultikVersion: "test-version",
|
||||
StartedAt: time.Now(),
|
||||
}
|
||||
|
||||
return repos.Snapshots.Create(ctx, tx, snapshot)
|
||||
})
|
||||
require.NoError(t, err)
|
||||
@@ -258,9 +277,9 @@ func TestEndToEndBackup(t *testing.T) {
|
||||
// The scanner counts both files and directories, so we have:
|
||||
// 4 files + 4 directories (/home, /home/user, /home/user/documents, /home/user/pictures, /home/user/code)
|
||||
assert.GreaterOrEqual(t, result.FilesScanned, 4, "Should scan at least 4 files")
|
||||
assert.Greater(t, result.BytesScanned, int64(0), "Should scan some bytes")
|
||||
assert.Greater(t, result.ChunksCreated, 0, "Should create chunks")
|
||||
assert.Greater(t, result.BlobsCreated, 0, "Should create blobs")
|
||||
assert.Positive(t, result.BytesScanned, "Should scan some bytes")
|
||||
assert.Positive(t, result.ChunksCreated, "Should create chunks")
|
||||
assert.Positive(t, result.BlobsCreated, "Should create blobs")
|
||||
|
||||
// Verify storage operations
|
||||
calls := mockStorage.GetCalls()
|
||||
@@ -268,6 +287,7 @@ func TestEndToEndBackup(t *testing.T) {
|
||||
|
||||
// Should have uploaded at least one blob
|
||||
blobUploads := 0
|
||||
|
||||
for _, call := range calls {
|
||||
if len(call) > 4 && call[:4] == "Put:" {
|
||||
if len(call) > 10 && call[4:10] == "blobs/" {
|
||||
@@ -275,27 +295,30 @@ func TestEndToEndBackup(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
assert.Greater(t, blobUploads, 0, "Should upload at least one blob")
|
||||
|
||||
assert.Positive(t, blobUploads, "Should upload at least one blob")
|
||||
|
||||
// Verify files in database
|
||||
files, err := repos.Files.ListByPrefix(ctx, "/home/user")
|
||||
require.NoError(t, err)
|
||||
// Count only regular files (not directories)
|
||||
regularFiles := 0
|
||||
|
||||
for _, f := range files {
|
||||
if f.Mode&0x80000000 == 0 { // Check if regular file (not directory)
|
||||
regularFiles++
|
||||
}
|
||||
}
|
||||
|
||||
assert.Equal(t, 4, regularFiles, "Should have 4 regular files in database")
|
||||
|
||||
// Verify chunks were created by checking a specific file
|
||||
fileChunks, err := repos.FileChunks.GetByPath(ctx, "/home/user/documents/file1.txt")
|
||||
require.NoError(t, err)
|
||||
assert.Greater(t, len(fileChunks), 0, "Should have chunks for file1.txt")
|
||||
assert.NotEmpty(t, fileChunks, "Should have chunks for file1.txt")
|
||||
|
||||
// Verify blobs were uploaded to storage
|
||||
assert.Greater(t, mockStorage.GetStorageSize(), 0, "Should have blobs in storage")
|
||||
assert.Positive(t, mockStorage.GetStorageSize(), "Should have blobs in storage")
|
||||
|
||||
// Complete the snapshot - just verify we got results
|
||||
// In a real integration test, we'd update the snapshot record
|
||||
@@ -337,9 +360,11 @@ func TestBackupAndVerify(t *testing.T) {
|
||||
// Create test database
|
||||
ctx := context.Background()
|
||||
db, err := database.New(ctx, ":memory:")
|
||||
|
||||
require.NoError(t, err)
|
||||
defer func() {
|
||||
if err := db.Close(); err != nil {
|
||||
err := db.Close()
|
||||
if err != nil {
|
||||
t.Errorf("failed to close database: %v", err)
|
||||
}
|
||||
}()
|
||||
@@ -366,6 +391,7 @@ func TestBackupAndVerify(t *testing.T) {
|
||||
VaultikVersion: "test-version",
|
||||
StartedAt: time.Now(),
|
||||
}
|
||||
|
||||
return repos.Snapshots.Create(ctx, tx, snapshot)
|
||||
})
|
||||
require.NoError(t, err)
|
||||
@@ -375,7 +401,7 @@ func TestBackupAndVerify(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify backup created blobs
|
||||
assert.Greater(t, result.BlobsCreated, 0, "Should create at least one blob")
|
||||
assert.Positive(t, result.BlobsCreated, "Should create at least one blob")
|
||||
assert.Equal(t, mockStorage.GetStorageSize(), result.BlobsCreated, "Storage should have the blobs")
|
||||
|
||||
// Verify we can retrieve the blob from storage
|
||||
@@ -391,18 +417,19 @@ func TestBackupAndVerify(t *testing.T) {
|
||||
// Get blob info
|
||||
blobInfo, err := mockStorage.Stat(ctx, blobKey)
|
||||
require.NoError(t, err)
|
||||
assert.Greater(t, blobInfo.Size, int64(0), "Blob should have content")
|
||||
assert.Positive(t, blobInfo.Size, "Blob should have content")
|
||||
|
||||
// Get blob content
|
||||
reader, err := mockStorage.Get(ctx, blobKey)
|
||||
require.NoError(t, err)
|
||||
|
||||
defer func() { _ = reader.Close() }()
|
||||
|
||||
// Verify blob data is encrypted (should not contain plaintext)
|
||||
blobData, err := io.ReadAll(reader)
|
||||
require.NoError(t, err)
|
||||
assert.NotContains(t, string(blobData), testContent, "Blob should be encrypted")
|
||||
assert.Greater(t, len(blobData), 0, "Blob should have data")
|
||||
assert.NotEmpty(t, blobData, "Blob should have data")
|
||||
}
|
||||
|
||||
t.Logf("Backup and verify test completed successfully")
|
||||
@@ -418,6 +445,7 @@ func TestBackupAndRestore(t *testing.T) {
|
||||
// Create real temp directory for the database (SQLite needs real filesystem)
|
||||
realTempDir, err := os.MkdirTemp("", "vaultik-test-")
|
||||
require.NoError(t, err)
|
||||
|
||||
defer func() { _ = os.RemoveAll(realTempDir) }()
|
||||
|
||||
// Use real OS filesystem for this test
|
||||
@@ -434,10 +462,14 @@ func TestBackupAndRestore(t *testing.T) {
|
||||
// Create directories and files
|
||||
for path, content := range testFiles {
|
||||
dir := filepath.Dir(path)
|
||||
if err := fs.MkdirAll(dir, 0755); err != nil {
|
||||
|
||||
err := fs.MkdirAll(dir, 0755)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create directory %s: %v", dir, err)
|
||||
}
|
||||
if err := afero.WriteFile(fs, path, []byte(content), 0644); err != nil {
|
||||
|
||||
err = afero.WriteFile(fs, path, []byte(content), 0644)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create test file %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
@@ -455,6 +487,7 @@ func TestBackupAndRestore(t *testing.T) {
|
||||
dbPath := filepath.Join(realTempDir, "test.db")
|
||||
db, err := database.New(ctx, dbPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
defer func() { _ = db.Close() }()
|
||||
|
||||
repos := database.NewRepositories(db)
|
||||
@@ -558,6 +591,7 @@ func TestEndToEndFileStorage(t *testing.T) {
|
||||
fs := afero.NewOsFs()
|
||||
tempDir, err := os.MkdirTemp("", "vaultik-e2e-")
|
||||
require.NoError(t, err)
|
||||
|
||||
defer func() { _ = os.RemoveAll(tempDir) }()
|
||||
|
||||
dataDir := filepath.Join(tempDir, "source")
|
||||
@@ -618,6 +652,7 @@ func TestEndToEndFileStorage(t *testing.T) {
|
||||
|
||||
db, err := database.New(ctx, dbPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
defer func() { _ = db.Close() }()
|
||||
|
||||
repos := database.NewRepositories(db)
|
||||
@@ -644,8 +679,8 @@ func TestEndToEndFileStorage(t *testing.T) {
|
||||
|
||||
scanResult, err := scanner.Scan(ctx, dataDir, snapshotID)
|
||||
require.NoError(t, err)
|
||||
require.Greater(t, scanResult.FilesScanned, 0)
|
||||
require.Greater(t, scanResult.BlobsCreated, 0)
|
||||
require.Positive(t, scanResult.FilesScanned)
|
||||
require.Positive(t, scanResult.BlobsCreated)
|
||||
|
||||
require.NoError(t, sm.CompleteSnapshot(ctx, snapshotID))
|
||||
require.NoError(t, sm.ExportSnapshotMetadata(ctx, dbPath, snapshotID))
|
||||
@@ -656,6 +691,7 @@ func TestEndToEndFileStorage(t *testing.T) {
|
||||
blobInfo, err := os.Stat(filepath.Join(storeDir, "blobs"))
|
||||
require.NoError(t, err)
|
||||
require.True(t, blobInfo.IsDir())
|
||||
|
||||
metaInfo, err := os.Stat(filepath.Join(storeDir, "metadata", snapshot.RemoteSnapshotKey(snapshotID)))
|
||||
require.NoError(t, err)
|
||||
require.True(t, metaInfo.IsDir())
|
||||
@@ -721,6 +757,7 @@ func TestDedupOnlySnapshotRestores(t *testing.T) {
|
||||
fs := afero.NewOsFs()
|
||||
tempDir, err := os.MkdirTemp("", "vaultik-dedup-")
|
||||
require.NoError(t, err)
|
||||
|
||||
defer func() { _ = os.RemoveAll(tempDir) }()
|
||||
|
||||
dataDir := filepath.Join(tempDir, "source")
|
||||
@@ -756,7 +793,9 @@ func TestDedupOnlySnapshotRestores(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
db, err := database.New(ctx, dbPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
defer func() { _ = db.Close() }()
|
||||
|
||||
repos := database.NewRepositories(db)
|
||||
|
||||
makeScanner := func() *snapshot.Scanner {
|
||||
@@ -780,13 +819,14 @@ func TestDedupOnlySnapshotRestores(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
r1, err := makeScanner().Scan(ctx, dataDir, id1)
|
||||
require.NoError(t, err)
|
||||
require.Greater(t, r1.BlobsCreated, 0, "first snapshot should upload at least one blob")
|
||||
require.Positive(t, r1.BlobsCreated, "first snapshot should upload at least one blob")
|
||||
require.NoError(t, sm.CompleteSnapshot(ctx, id1))
|
||||
require.NoError(t, sm.ExportSnapshotMetadata(ctx, dbPath, id1))
|
||||
|
||||
// Second snapshot — same data, every chunk dedups. Sleep past the
|
||||
// second-precision timestamp so the snapshot IDs differ.
|
||||
time.Sleep(1100 * time.Millisecond)
|
||||
|
||||
id2, err := sm.CreateSnapshotWithName(ctx, cfg.Hostname, "dedup", "v", "g")
|
||||
require.NoError(t, err)
|
||||
r2, err := makeScanner().Scan(ctx, dataDir, id2)
|
||||
@@ -833,5 +873,6 @@ func bytesPattern(tag string, n int) []byte {
|
||||
for i := range out {
|
||||
out[i] = byte(tag[i%len(tag)] ^ byte(i&0xff))
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package vaultik
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
@@ -23,20 +24,24 @@ type PruneOptions struct {
|
||||
// confirming with the user.
|
||||
func (v *Vaultik) NukeRemote(force bool) error {
|
||||
if !force {
|
||||
return fmt.Errorf("nuke requires --force (this deletes ALL remote snapshots and blobs)")
|
||||
return errors.New("nuke requires --force (this deletes ALL remote snapshots and blobs)")
|
||||
}
|
||||
|
||||
v.UI.Begin("Removing all snapshot metadata from backup destination store.")
|
||||
v.UI.Beginf("Removing all snapshot metadata from backup destination store.")
|
||||
|
||||
if _, err := v.RemoveAllSnapshots(&RemoveOptions{Force: true}); err != nil {
|
||||
return fmt.Errorf("removing all snapshots: %w", err)
|
||||
}
|
||||
|
||||
v.UI.Begin("Removing any blobs still present in backup destination store.")
|
||||
if err := v.PruneBlobs(&PruneOptions{Force: true}); err != nil {
|
||||
v.UI.Beginf("Removing any blobs still present in backup destination store.")
|
||||
|
||||
err := v.PruneBlobs(&PruneOptions{Force: true})
|
||||
if err != nil {
|
||||
return fmt.Errorf("pruning blobs: %w", err)
|
||||
}
|
||||
|
||||
v.UI.Complete("Backup destination store is now empty.")
|
||||
v.UI.Completef("Backup destination store is now empty.")
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -55,7 +60,8 @@ type PruneBlobsResult struct {
|
||||
// prefer this method over PruneDatabase or PruneBlobs individually
|
||||
// unless it specifically wants one half.
|
||||
func (v *Vaultik) Prune(opts *PruneOptions) error {
|
||||
if err := v.EnsureStorageBinding(); err != nil {
|
||||
err := v.EnsureStorageBinding()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// First reconcile local snapshot records against remote metadata:
|
||||
@@ -63,12 +69,15 @@ func (v *Vaultik) Prune(opts *PruneOptions) error {
|
||||
// store is treated as gone. This used to be the separate 'snapshot
|
||||
// cleanup' command and is now folded in so a single 'vaultik prune'
|
||||
// gets the local index fully back in sync with the destination.
|
||||
if err := v.CleanupLocalSnapshots(); err != nil {
|
||||
err = v.CleanupLocalSnapshots()
|
||||
if err != nil {
|
||||
return fmt.Errorf("reconciling local snapshots with remote: %w", err)
|
||||
}
|
||||
|
||||
if _, err := v.PruneDatabase(); err != nil {
|
||||
return fmt.Errorf("pruning local database: %w", err)
|
||||
}
|
||||
|
||||
return v.PruneBlobs(opts)
|
||||
}
|
||||
|
||||
@@ -92,27 +101,35 @@ func (v *Vaultik) PruneBlobs(opts *PruneOptions) error {
|
||||
|
||||
if len(unreferencedBlobs) == 0 {
|
||||
log.Info("No unreferenced blobs found")
|
||||
|
||||
if opts.JSON {
|
||||
return v.outputPruneBlobsJSON(result)
|
||||
}
|
||||
|
||||
v.printlnStdout("No unreferenced blobs to remove.")
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
log.Info("Found unreferenced blobs", "count", len(unreferencedBlobs), "total_size", humanize.Bytes(uint64(totalSize)))
|
||||
|
||||
if !opts.JSON {
|
||||
v.printfStdout("Found %d unreferenced blob(s) totaling %s\n", len(unreferencedBlobs), humanize.Bytes(uint64(totalSize)))
|
||||
v.stdoutf("Found %d unreferenced blob(s) totaling %s\n", len(unreferencedBlobs), humanize.Bytes(uint64(totalSize)))
|
||||
}
|
||||
|
||||
if !opts.Force && !opts.JSON {
|
||||
v.printfStdout("\nDelete %d unreferenced blob(s)? [y/N] ", len(unreferencedBlobs))
|
||||
v.stdoutf("\nDelete %d unreferenced blob(s)? [y/N] ", len(unreferencedBlobs))
|
||||
|
||||
var confirm string
|
||||
if _, err := v.scanStdin(&confirm); err != nil {
|
||||
v.printlnStdout("Cancelled")
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
if strings.ToLower(confirm) != "y" {
|
||||
v.printlnStdout("Cancelled")
|
||||
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -123,9 +140,10 @@ func (v *Vaultik) PruneBlobs(opts *PruneOptions) error {
|
||||
return v.outputPruneBlobsJSON(result)
|
||||
}
|
||||
|
||||
v.printfStdout("\nDeleted %d blob(s) totaling %s\n", result.BlobsDeleted, humanize.Bytes(uint64(result.BytesFreed)))
|
||||
v.stdoutf("\nDeleted %d blob(s) totaling %s\n", result.BlobsDeleted, humanize.Bytes(uint64(result.BytesFreed)))
|
||||
|
||||
if result.BlobsFailed > 0 {
|
||||
v.printfStdout("Failed to delete %d blob(s)\n", result.BlobsFailed)
|
||||
v.stdoutf("Failed to delete %d blob(s)\n", result.BlobsFailed)
|
||||
}
|
||||
|
||||
return nil
|
||||
@@ -140,6 +158,7 @@ func (v *Vaultik) collectReferencedBlobs() (map[string]bool, error) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("listing snapshot keys: %w", err)
|
||||
}
|
||||
|
||||
log.Info("Found manifests in remote storage", "count", len(remoteKeys))
|
||||
|
||||
allBlobsReferenced := make(map[string]bool)
|
||||
@@ -147,18 +166,23 @@ func (v *Vaultik) collectReferencedBlobs() (map[string]bool, error) {
|
||||
|
||||
for _, remoteKey := range remoteKeys {
|
||||
log.Debug("Processing manifest", "remote_key", remoteKey)
|
||||
|
||||
manifest, err := v.downloadManifestByKey(remoteKey)
|
||||
if err != nil {
|
||||
log.Error("Failed to download manifest", "remote_key", remoteKey, "error", err)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
for _, blob := range manifest.Blobs {
|
||||
allBlobsReferenced[blob.Hash] = true
|
||||
}
|
||||
|
||||
manifestCount++
|
||||
}
|
||||
|
||||
log.Info("Processed manifests", "count", manifestCount, "unique_blobs_referenced", len(allBlobsReferenced))
|
||||
|
||||
return allBlobsReferenced, nil
|
||||
}
|
||||
|
||||
@@ -166,12 +190,14 @@ func (v *Vaultik) collectReferencedBlobs() (map[string]bool, error) {
|
||||
func (v *Vaultik) listUniqueSnapshotIDs() ([]string, error) {
|
||||
objectCh := v.Storage.ListStream(v.ctx, "metadata/")
|
||||
seen := make(map[string]bool)
|
||||
|
||||
var snapshotIDs []string
|
||||
|
||||
for object := range objectCh {
|
||||
if object.Err != nil {
|
||||
return nil, fmt.Errorf("listing metadata objects: %w", object.Err)
|
||||
}
|
||||
|
||||
parts := strings.Split(object.Key, "/")
|
||||
if len(parts) >= 2 && parts[0] == "metadata" && parts[1] != "" {
|
||||
if strings.HasSuffix(object.Key, "/") || strings.Contains(object.Key, "/manifest.json.zst") {
|
||||
@@ -183,12 +209,14 @@ func (v *Vaultik) listUniqueSnapshotIDs() ([]string, error) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return snapshotIDs, nil
|
||||
}
|
||||
|
||||
// listAllRemoteBlobs returns a map of all blob hashes to their sizes in remote storage
|
||||
func (v *Vaultik) listAllRemoteBlobs() (map[string]int64, error) {
|
||||
log.Info("Listing all blobs in storage")
|
||||
|
||||
allBlobs := make(map[string]int64)
|
||||
blobObjectCh := v.Storage.ListStream(v.ctx, "blobs/")
|
||||
|
||||
@@ -196,6 +224,7 @@ func (v *Vaultik) listAllRemoteBlobs() (map[string]int64, error) {
|
||||
if object.Err != nil {
|
||||
return nil, fmt.Errorf("listing blobs: %w", object.Err)
|
||||
}
|
||||
|
||||
parts := strings.Split(object.Key, "/")
|
||||
if len(parts) == 4 && parts[0] == "blobs" {
|
||||
allBlobs[parts[3]] = object.Size
|
||||
@@ -203,19 +232,24 @@ func (v *Vaultik) listAllRemoteBlobs() (map[string]int64, error) {
|
||||
}
|
||||
|
||||
log.Info("Found blobs in storage", "count", len(allBlobs))
|
||||
|
||||
return allBlobs, nil
|
||||
}
|
||||
|
||||
// findUnreferencedBlobs returns blob hashes not referenced by any manifest and their total size
|
||||
func (v *Vaultik) findUnreferencedBlobs(allBlobs map[string]int64, referenced map[string]bool) ([]string, int64) {
|
||||
var unreferenced []string
|
||||
var totalSize int64
|
||||
var (
|
||||
unreferenced []string
|
||||
totalSize int64
|
||||
)
|
||||
|
||||
for hash, size := range allBlobs {
|
||||
if !referenced[hash] {
|
||||
unreferenced = append(unreferenced, hash)
|
||||
totalSize += size
|
||||
}
|
||||
}
|
||||
|
||||
return unreferenced, totalSize
|
||||
}
|
||||
|
||||
@@ -226,8 +260,10 @@ func (v *Vaultik) deleteUnreferencedBlobs(unreferencedBlobs []string, allBlobs m
|
||||
for i, hash := range unreferencedBlobs {
|
||||
blobPath := fmt.Sprintf("blobs/%s/%s/%s", hash[:2], hash[2:4], hash)
|
||||
|
||||
if err := v.Storage.Delete(v.ctx, blobPath); err != nil {
|
||||
err := v.Storage.Delete(v.ctx, blobPath)
|
||||
if err != nil {
|
||||
log.Error("Failed to delete blob", "hash", hash, "error", err)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -256,5 +292,6 @@ func (v *Vaultik) deleteUnreferencedBlobs(unreferencedBlobs []string, allBlobs m
|
||||
func (v *Vaultik) outputPruneBlobsJSON(result *PruneBlobsResult) error {
|
||||
encoder := json.NewEncoder(v.Stdout)
|
||||
encoder.SetIndent("", " ")
|
||||
|
||||
return encoder.Encode(result)
|
||||
}
|
||||
|
||||
@@ -79,16 +79,19 @@ func setupPurgeTest(t *testing.T, snapshotIDs []string) *vaultik.Vaultik {
|
||||
// listRemainingSnapshots returns IDs of all completed snapshots in the database.
|
||||
func listRemainingSnapshots(t *testing.T, v *vaultik.Vaultik) []string {
|
||||
t.Helper()
|
||||
|
||||
ctx := context.Background()
|
||||
dbSnaps, err := v.Repositories.Snapshots.ListRecent(ctx, 10000)
|
||||
require.NoError(t, err)
|
||||
|
||||
var ids []string
|
||||
|
||||
for _, s := range dbSnaps {
|
||||
if s.CompletedAt != nil {
|
||||
ids = append(ids, s.ID.String())
|
||||
}
|
||||
}
|
||||
|
||||
return ids
|
||||
}
|
||||
|
||||
|
||||
@@ -37,7 +37,9 @@ func (s *testStorer) Put(ctx context.Context, key string, reader io.Reader) erro
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
s.data[key] = data
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -53,6 +55,7 @@ func (s *testStorer) Get(ctx context.Context, key string) (io.ReadCloser, error)
|
||||
if !exists {
|
||||
return nil, storage.ErrNotFound
|
||||
}
|
||||
|
||||
return io.NopCloser(bytes.NewReader(data)), nil
|
||||
}
|
||||
|
||||
@@ -64,6 +67,7 @@ func (s *testStorer) Stat(ctx context.Context, key string) (*storage.ObjectInfo,
|
||||
if !exists {
|
||||
return nil, storage.ErrNotFound
|
||||
}
|
||||
|
||||
return &storage.ObjectInfo{
|
||||
Key: key,
|
||||
Size: int64(len(data)),
|
||||
@@ -75,6 +79,7 @@ func (s *testStorer) Delete(ctx context.Context, key string) error {
|
||||
defer s.mu.Unlock()
|
||||
|
||||
delete(s.data, key)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -83,11 +88,13 @@ func (s *testStorer) List(ctx context.Context, prefix string) ([]string, error)
|
||||
defer s.mu.Unlock()
|
||||
|
||||
var keys []string
|
||||
|
||||
for key := range s.data {
|
||||
if prefix == "" || strings.HasPrefix(key, prefix) {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
}
|
||||
|
||||
return keys, nil
|
||||
}
|
||||
|
||||
@@ -96,6 +103,7 @@ func (s *testStorer) ListStream(ctx context.Context, prefix string) <-chan stora
|
||||
|
||||
go func() {
|
||||
defer close(ch)
|
||||
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
@@ -115,13 +123,16 @@ func (s *testStorer) ListStream(ctx context.Context, prefix string) <-chan stora
|
||||
func (s *testStorer) hasKey(key string) bool {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
_, exists := s.data[key]
|
||||
|
||||
return exists
|
||||
}
|
||||
|
||||
func (s *testStorer) keyCount() int {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
return len(s.data)
|
||||
}
|
||||
|
||||
@@ -175,6 +186,7 @@ func addBlob(t *testing.T, store *testStorer, hash string) {
|
||||
|
||||
// Create zstd compressed data
|
||||
var buf bytes.Buffer
|
||||
|
||||
writer, _ := zstd.NewWriter(&buf)
|
||||
_, _ = writer.Write([]byte("blob data"))
|
||||
_ = writer.Close()
|
||||
@@ -366,7 +378,7 @@ func TestRemoveAllSnapshots_NoSnapshots(t *testing.T) {
|
||||
result, err := tv.RemoveAllSnapshots(opts)
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, result.SnapshotsRemoved, 0)
|
||||
assert.Empty(t, result.SnapshotsRemoved)
|
||||
|
||||
// Verify output
|
||||
assert.Contains(t, tv.Stdout.String(), "No snapshots found")
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user