Compare commits

2 Commits

Author SHA1 Message Date
7ae470e530 Remediate all lint findings under the canonical golangci-lint config
Fix every finding surfaced by the canonical .golangci.yml with
golangci-lint v2.12.2 (refs #61), behavior-preserving throughout:

- err113: dynamic errors replaced with package-level sentinels and %w
  wrapping; direct comparisons converted to errors.Is
- goprintffuncname: printf-style helpers renamed with an f suffix
  (ui.Writer message methods, cli.ReportErrorf, database.Fatalf,
  vaultik stdoutf) and all call sites updated
- revive: stuttering type names renamed (blob.Handler, blob.WithReader,
  blob.ChunkPosition, storage.URL, storage.Info), doc comments added,
  unused parameters blanked, package comments added
- contextcheck/noctx: ctx threaded through blob.Packer
  (AddChunk/Flush/FinalizeBlob/PackChunks) and scanner call sites;
  context-aware exec and sql variants used
- funlen/cyclop/gocognit/nestif/dupl: oversized or duplicated
  functions split into focused helpers across production and test code
- paralleltest/tparallel/thelper/usetesting/testpackage: tests
  parallelized where safe (global log.Initialize kept in the serial
  phase), helpers marked, t.TempDir adopted, external test packages
  where only exported API is used
- gosec: integer conversions clamped or justified, header timeouts
  added, remaining findings suppressed with per-site justifications
- mnd/goconst/lll/wsl_v5/nlreturn/noinlineerr/errcheck and other
  mechanical findings fixed directly

Remove the deprecated log.LogOptions alias (callers migrated to
log.Options). make check is green.
2026-08-07 18:51:21 +00:00
6cf9211407 Apply mechanical lint fixes for golangci-lint v2.12.2 rollout
Auto-remediate style-only findings (wsl_v5, nlreturn, noinlineerr,
modernize, intrange, perfsprint, usetesting, unconvert, errorlint,
gocritic, testifylint) and rename printf-style helpers to f-suffixed
names (goprintffuncname): ui.Writer message methods, cli.ReportErrorf,
database.Fatalf, vaultik stdoutf.
2026-08-07 17:01:52 +00:00
127 changed files with 10180 additions and 5398 deletions

View File

@@ -1,5 +1,9 @@
version: "2"
# Config schema uses the golangci-lint v2 layout (settings live under
# linters.settings, not top-level linters-settings) so that the
# thresholds below are actually applied by golangci-lint >= v2.
run:
timeout: 5m
modules-download-mode: readonly
@@ -14,8 +18,7 @@ linters:
- wsl # Deprecated, replaced by wsl_v5
- wrapcheck # Too verbose for internal packages
- varnamelen # Short names like db, id are idiomatic Go
linters-settings:
settings:
lll:
line-length: 88
funlen:
@@ -27,6 +30,5 @@ linters-settings:
threshold: 100
issues:
exclude-use-default: false
max-issues-per-linter: 0
max-same-issues: 0

View File

@@ -1,6 +1,6 @@
# Lint stage
# golangci/golangci-lint:v2.11.3-alpine, 2026-03-17
FROM golangci/golangci-lint:v2.11.3-alpine@sha256:b1c3de5862ad0a95b4e45a993b0f00415835d687e4f12c845c7493b86c13414e AS lint
# golangci/golangci-lint:v2.12.2-alpine, 2026-08-07
FROM golangci/golangci-lint:v2.12.2-alpine@sha256:91b27804074a0bacea298707f016911e60cf0cdbc6c7bf5ccacb5f0606d18d60 AS lint
RUN apk add --no-cache make build-base

View File

@@ -55,7 +55,7 @@ clean:
# Install dependencies.
deps:
go mod download
go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest
go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.12.2
# Run tests with coverage.
test-coverage:

13
TODO.md
View File

@@ -14,13 +14,16 @@ pre-1.0
# Next Step
Remediate the 2,990 lint findings surfaced by the standard
`.golangci.yml` (issue #61): behavior-preserving fixes in per-linter or
per-package chunks, mechanical linters first, until `make check` is
green on `main`.
Reconcile the uncommitted ARCHITECTURE.md edits on main: finish and
commit, or revert.
# Completed Steps
- 2026-08-07: Updated golangci-lint to v2.12.2 everywhere it is pinned
(`Dockerfile` lint stage, `Makefile` deps target), replaced
`.golangci.yml` with the canonical config (v2 schema, `default: all`),
and remediated all lint findings it surfaced (issue #61):
behavior-preserving fixes across every package, `make check` green.
- 2026-08-07: Added the standard `.golangci.yml` and `.editorconfig`
(issue #59); lint findings under the new config are tracked in issue
#61. `script/bootstrap` now installs sqlite3 (needed by tests).
@@ -46,8 +49,6 @@ green on `main`.
# Future Steps
- Reconcile the uncommitted ARCHITECTURE.md edits on main: finish and
commit, or revert.
- Review stale local branches (add-godoc-to-cli-package,
feature/pluggable-storage-backend) and merge or delete them.
- Define remaining scope for a first tagged release and cut v0.1.0.

View File

@@ -1,3 +1,4 @@
// Package main is the vaultik command-line entry point.
package main
import (
@@ -11,31 +12,37 @@ import (
func main() {
// CPU profiling: set VAULTIK_CPUPROFILE=/path/to/cpu.prof
if cpuProfile := os.Getenv("VAULTIK_CPUPROFILE"); cpuProfile != "" {
f, err := os.Create(cpuProfile)
f, err := os.Create(cpuProfile) //nolint:gosec // G304: operator-set path
if err != nil {
panic("could not create CPU profile: " + err.Error())
}
defer func() { _ = f.Close() }()
if err := pprof.StartCPUProfile(f); err != nil {
err = pprof.StartCPUProfile(f)
if err != nil {
panic("could not start CPU profile: " + err.Error())
}
defer pprof.StopCPUProfile()
}
// Memory profiling: set VAULTIK_MEMPROFILE=/path/to/mem.prof
if memProfile := os.Getenv("VAULTIK_MEMPROFILE"); memProfile != "" {
defer func() {
f, err := os.Create(memProfile)
f, err := os.Create(memProfile) //nolint:gosec // G304: operator-set path
if err != nil {
panic("could not create memory profile: " + err.Error())
}
defer func() { _ = f.Close() }()
runtime.GC() // get up-to-date statistics
if err := pprof.WriteHeapProfile(f); err != nil {
err = pprof.WriteHeapProfile(f)
if err != nil {
panic("could not write memory profile: " + err.Error())
}
}()
}
cli.CLIEntry()
cli.Entry()
}

View File

@@ -2,5 +2,18 @@ package blob
import "errors"
// ErrBlobSizeLimitExceeded is returned when adding a chunk would exceed the blob size limit
// ErrBlobSizeLimitExceeded is returned when adding a chunk would exceed
// the blob size limit.
var ErrBlobSizeLimitExceeded = errors.New("adding chunk would exceed blob size limit")
// ErrNoRecipients is returned when a Packer is created without any age
// recipients; blobs must always be encrypted.
var ErrNoRecipients = errors.New("recipients are required - blobs must be encrypted")
// ErrInvalidMaxBlobSize is returned when the configured maximum blob size
// is zero or negative.
var ErrInvalidMaxBlobSize = errors.New("max blob size must be positive")
// ErrNoFilesystem is returned when a Packer is created without a filesystem
// for temporary files.
var ErrNoFilesystem = errors.New("filesystem is required")

View File

@@ -18,6 +18,7 @@ import (
"context"
"database/sql"
"encoding/hex"
"errors"
"fmt"
"io"
"sync"
@@ -31,21 +32,28 @@ import (
"sneak.berlin/go/vaultik/internal/types"
)
// BlobHandler is a callback function invoked when a blob is finalized and ready for upload.
// The handler receives a BlobWithReader containing the blob metadata and a reader for
// the compressed and encrypted blob content. The handler is responsible for uploading
// the blob to storage and cleaning up any temporary files.
type BlobHandler func(blob *BlobWithReader) error
// Handler is a callback function invoked when a blob is finalized and
// ready for upload. The handler receives a WithReader containing the
// blob metadata and a reader for the compressed and encrypted blob content.
// The handler is responsible for uploading the blob to storage and cleaning
// up any temporary files.
type Handler func(blob *WithReader) error
// PackerConfig holds configuration for creating a Packer.
// All fields except BlobHandler are required.
type PackerConfig struct {
MaxBlobSize int64 // Maximum size of a blob before forcing finalization
CompressionLevel int // Zstd compression level (1-19, higher = better compression)
Recipients []string // Age recipients for encryption
Repositories *database.Repositories // Database repositories for tracking blob metadata
BlobHandler BlobHandler // Optional callback when blob is ready for upload
Fs afero.Fs // Filesystem for temporary files
// MaxBlobSize is the maximum size of a blob before forcing finalization.
MaxBlobSize int64
// CompressionLevel is the zstd level (1-19, higher = better compression).
CompressionLevel int
// Recipients holds the age recipients for encryption.
Recipients []string
// Repositories provides database access for tracking blob metadata.
Repositories *database.Repositories
// BlobHandler is an optional callback when a blob is ready for upload.
BlobHandler Handler
// Fs is the filesystem used for temporary files.
Fs afero.Fs
}
// PendingChunk represents a chunk waiting to be inserted into the database.
@@ -61,7 +69,7 @@ type Packer struct {
maxBlobSize int64
compressionLevel int
recipients []string // Age recipients for encryption
blobHandler BlobHandler // Called when blob is ready
blobHandler Handler // Called when blob is ready
repos *database.Repositories // For creating blob records
fs afero.Fs // Filesystem for temporary files
@@ -108,22 +116,23 @@ type FinishedBlob struct {
ID string
Hash string
Data []byte // Compressed data
Chunks []*BlobChunkRef
Chunks []*ChunkPosition
CreatedTS time.Time
Uncompressed int64
Compressed int64
}
// BlobChunkRef represents a chunk's position within a blob
type BlobChunkRef struct {
// ChunkPosition represents a chunk's position within a blob
type ChunkPosition struct {
ChunkHash string
Offset int64
Length int64
}
// BlobWithReader wraps a FinishedBlob with its data reader
type BlobWithReader struct {
// WithReader wraps a FinishedBlob with its data reader
type WithReader struct {
*FinishedBlob
Reader io.ReadSeeker
TempFile afero.File // Optional, only set for disk-based blobs
InsertedChunkHashes []string // Chunk hashes that were inserted to DB with this blob
@@ -134,14 +143,17 @@ type BlobWithReader struct {
// Returns an error if required configuration fields are missing or invalid.
func NewPacker(cfg PackerConfig) (*Packer, error) {
if len(cfg.Recipients) == 0 {
return nil, fmt.Errorf("recipients are required - blobs must be encrypted")
return nil, ErrNoRecipients
}
if cfg.MaxBlobSize <= 0 {
return nil, fmt.Errorf("max blob size must be positive")
return nil, ErrInvalidMaxBlobSize
}
if cfg.Fs == nil {
return nil, fmt.Errorf("filesystem is required")
return nil, ErrNoFilesystem
}
return &Packer{
maxBlobSize: cfg.MaxBlobSize,
compressionLevel: cfg.CompressionLevel,
@@ -157,9 +169,10 @@ func NewPacker(cfg PackerConfig) (*Packer, error) {
// The handler is responsible for uploading the blob to storage.
// If no handler is set, finalized blobs are stored in memory and can be
// retrieved with GetFinishedBlobs().
func (p *Packer) SetBlobHandler(handler BlobHandler) {
func (p *Packer) SetBlobHandler(handler Handler) {
p.mu.Lock()
defer p.mu.Unlock()
p.blobHandler = handler
}
@@ -169,6 +182,7 @@ func (p *Packer) SetBlobHandler(handler BlobHandler) {
func (p *Packer) AddPendingChunk(hash string, size int64) {
p.mu.Lock()
defer p.mu.Unlock()
p.pendingChunks = append(p.pendingChunks, PendingChunk{Hash: hash, Size: size})
}
@@ -177,13 +191,14 @@ func (p *Packer) AddPendingChunk(hash string, size int64) {
// In this case, the caller should finalize the current blob and retry.
// The chunk data is written immediately and can be garbage collected after this call.
// Thread-safe.
func (p *Packer) AddChunk(chunk *ChunkRef) error {
func (p *Packer) AddChunk(ctx context.Context, chunk *ChunkRef) error {
p.mu.Lock()
defer p.mu.Unlock()
// Initialize new blob if needed
if p.currentBlob == nil {
if err := p.startNewBlob(); err != nil {
err := p.startNewBlob(ctx)
if err != nil {
return fmt.Errorf("starting new blob: %w", err)
}
}
@@ -202,7 +217,8 @@ func (p *Packer) AddChunk(chunk *ChunkRef) error {
}
// Add chunk to current blob
if err := p.addChunkToCurrentBlob(chunk); err != nil {
err := p.addChunkToCurrentBlob(chunk)
if err != nil {
return err
}
@@ -213,12 +229,13 @@ func (p *Packer) AddChunk(chunk *ChunkRef) error {
// This should be called after all chunks have been added to ensure no data is lost.
// If a BlobHandler is set, it will be called with the finalized blob.
// Thread-safe.
func (p *Packer) Flush() error {
func (p *Packer) Flush(ctx context.Context) error {
p.mu.Lock()
defer p.mu.Unlock()
if p.currentBlob != nil && len(p.currentBlob.chunks) > 0 {
if err := p.finalizeCurrentBlob(); err != nil {
err := p.finalizeCurrentBlob(ctx)
if err != nil {
return fmt.Errorf("finalizing blob: %w", err)
}
}
@@ -232,7 +249,7 @@ func (p *Packer) Flush() error {
// BlobHandler (if set) or stored internally.
// Caller must handle retrying any chunk that triggered size limit exceeded.
// Not thread-safe - caller must hold the lock.
func (p *Packer) FinalizeBlob() error {
func (p *Packer) FinalizeBlob(ctx context.Context) error {
p.mu.Lock()
defer p.mu.Unlock()
@@ -240,7 +257,7 @@ func (p *Packer) FinalizeBlob() error {
return nil
}
return p.finalizeCurrentBlob()
return p.finalizeCurrentBlob(ctx)
}
// GetFinishedBlobs returns all completed blobs and clears the internal list.
@@ -253,11 +270,37 @@ func (p *Packer) GetFinishedBlobs() []*FinishedBlob {
blobs := p.finishedBlobs
p.finishedBlobs = make([]*FinishedBlob, 0)
return blobs
}
// PackChunks is a convenience method to pack multiple chunks at once.
func (p *Packer) PackChunks(ctx context.Context, chunks []*ChunkRef) error {
for _, chunk := range chunks {
err := p.AddChunk(ctx, chunk)
if errors.Is(err, ErrBlobSizeLimitExceeded) {
// Finalize current blob and retry
err = p.FinalizeBlob(ctx)
if err != nil {
return fmt.Errorf("finalizing blob before retry: %w", err)
}
// Retry the chunk
err = p.AddChunk(ctx, chunk)
if err != nil {
return fmt.Errorf(
"adding chunk %s after finalize: %w", chunk.Hash, err)
}
} else if err != nil {
return fmt.Errorf("adding chunk %s: %w", chunk.Hash, err)
}
}
return p.Flush(ctx)
}
// startNewBlob initializes a new blob (must be called with lock held)
func (p *Packer) startNewBlob() error {
func (p *Packer) startNewBlob(ctx context.Context) error {
// Generate UUID for the blob
blobID := uuid.New().String()
@@ -267,18 +310,24 @@ func (p *Packer) startNewBlob() error {
if err != nil {
return fmt.Errorf("parsing blob ID: %w", err)
}
blob := &database.Blob{
ID: blobIDTyped,
Hash: types.BlobHash("temp-placeholder-" + blobID), // Temporary placeholder until finalized
// Temporary placeholder hash until finalized.
Hash: types.BlobHash("temp-placeholder-" + blobID),
CreatedTS: time.Now().UTC(),
FinishedTS: nil,
UncompressedSize: 0,
CompressedSize: 0,
UploadedTS: nil,
}
if err := p.repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error {
return p.repos.Blobs.Create(ctx, tx, blob)
}); err != nil {
err = p.repos.WithTx(
ctx,
func(txCtx context.Context, tx *sql.Tx) error {
return p.repos.Blobs.Create(txCtx, tx, blob)
})
if err != nil {
return fmt.Errorf("creating blob record: %w", err)
}
}
@@ -294,6 +343,7 @@ func (p *Packer) startNewBlob() error {
if err != nil {
_ = tempFile.Close()
_ = p.fs.Remove(tempFile.Name())
return fmt.Errorf("creating blobgen writer: %w", err)
}
@@ -307,15 +357,20 @@ func (p *Packer) startNewBlob() error {
size: 0,
}
log.Debug("Created new blob container", "blob_id", blobID, "temp_file", tempFile.Name())
log.Debug("Created new blob container",
"blob_id", blobID, "temp_file", tempFile.Name())
return nil
}
// addChunkToCurrentBlob adds a chunk to the current blob (must be called with lock held)
// addChunkToCurrentBlob adds a chunk to the current blob (must be called
// with lock held).
func (p *Packer) addChunkToCurrentBlob(chunk *ChunkRef) error {
// Skip if chunk already in current blob
if p.currentBlob.chunkSet[chunk.Hash] {
log.Debug("Skipping duplicate chunk already in current blob", "chunk_hash", chunk.Hash)
log.Debug("Skipping duplicate chunk already in current blob",
"chunk_hash", chunk.Hash)
return nil
}
@@ -323,7 +378,8 @@ func (p *Packer) addChunkToCurrentBlob(chunk *ChunkRef) error {
offset := p.currentBlob.size
// Write to the blobgen writer (compression -> encryption -> disk)
if _, err := p.currentBlob.writer.Write(chunk.Data); err != nil {
_, err := p.currentBlob.writer.Write(chunk.Data)
if err != nil {
return fmt.Errorf("writing to blob stream: %w", err)
}
@@ -356,7 +412,7 @@ func (p *Packer) addChunkToCurrentBlob(chunk *ChunkRef) error {
}
// finalizeCurrentBlob completes the current blob (must be called with lock held)
func (p *Packer) finalizeCurrentBlob() error {
func (p *Packer) finalizeCurrentBlob(ctx context.Context) error {
if p.currentBlob == nil {
return nil
}
@@ -371,7 +427,8 @@ func (p *Packer) finalizeCurrentBlob() error {
chunksToInsert := p.pendingChunks
p.pendingChunks = nil
if err := p.commitBlobToDatabase(blobHash, finalSize, chunksToInsert); err != nil {
err = p.commitBlobToDatabase(ctx, blobHash, finalSize, chunksToInsert)
if err != nil {
return err
}
@@ -399,44 +456,59 @@ func (p *Packer) finalizeCurrentBlob() error {
return p.deliverFinishedBlob(finished, insertedChunkHashes)
}
// closeBlobWriter closes the writer, syncs to disk, and returns the blob hash and final size
// closeBlobWriter closes the writer, syncs to disk, and returns the blob
// hash and final size.
func (p *Packer) closeBlobWriter() (string, int64, error) {
if err := p.currentBlob.writer.Close(); err != nil {
err := p.currentBlob.writer.Close()
if err != nil {
p.cleanupTempFile()
return "", 0, fmt.Errorf("closing blobgen writer: %w", err)
}
if err := p.currentBlob.tempFile.Sync(); err != nil {
err = p.currentBlob.tempFile.Sync()
if err != nil {
p.cleanupTempFile()
return "", 0, fmt.Errorf("syncing temp file: %w", err)
}
finalSize, err := p.currentBlob.tempFile.Seek(0, io.SeekCurrent)
if err != nil {
p.cleanupTempFile()
return "", 0, fmt.Errorf("getting file size: %w", err)
}
if _, err := p.currentBlob.tempFile.Seek(0, io.SeekStart); err != nil {
_, err = p.currentBlob.tempFile.Seek(0, io.SeekStart)
if err != nil {
p.cleanupTempFile()
return "", 0, fmt.Errorf("seeking to start: %w", err)
}
finalHash := p.currentBlob.writer.Sum256()
return hex.EncodeToString(finalHash), finalSize, nil
}
// buildChunkRefs creates BlobChunkRef entries from the current blob's chunks
func (p *Packer) buildChunkRefs() []*BlobChunkRef {
refs := make([]*BlobChunkRef, 0, len(p.currentBlob.chunks))
// buildChunkRefs creates ChunkPosition entries from the current blob's chunks
func (p *Packer) buildChunkRefs() []*ChunkPosition {
refs := make([]*ChunkPosition, 0, len(p.currentBlob.chunks))
for _, chunk := range p.currentBlob.chunks {
refs = append(refs, &BlobChunkRef{
refs = append(refs, &ChunkPosition{
ChunkHash: chunk.Hash, Offset: chunk.Offset, Length: chunk.Size,
})
}
return refs
}
// commitBlobToDatabase inserts pending chunks, blob_chunks, and updates the blob record
func (p *Packer) commitBlobToDatabase(blobHash string, finalSize int64, chunksToInsert []PendingChunk) error {
func (p *Packer) commitBlobToDatabase(
ctx context.Context,
blobHash string, finalSize int64, chunksToInsert []PendingChunk,
) error {
if p.repos == nil {
return nil
}
@@ -444,13 +516,46 @@ func (p *Packer) commitBlobToDatabase(blobHash string, finalSize int64, chunksTo
blobIDTyped, parseErr := types.ParseBlobID(p.currentBlob.id)
if parseErr != nil {
p.cleanupTempFile()
return fmt.Errorf("parsing blob ID: %w", parseErr)
}
err := p.repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error {
err := p.repos.WithTx(
ctx,
func(txCtx context.Context, tx *sql.Tx) error {
return p.insertBlobRecords(txCtx, tx, blobIDTyped, blobHash,
finalSize, chunksToInsert)
})
if err != nil {
p.cleanupTempFile()
return fmt.Errorf("finalizing blob transaction: %w", err)
}
log.Debug("Committed blob transaction",
"chunks_inserted", len(chunksToInsert),
"blob_chunks_inserted", len(p.currentBlob.chunks))
return nil
}
// insertBlobRecords inserts pending chunks and blob_chunk rows, then marks
// the blob finished, all within the supplied transaction.
func (p *Packer) insertBlobRecords(
ctx context.Context,
tx *sql.Tx,
blobIDTyped types.BlobID,
blobHash string,
finalSize int64,
chunksToInsert []PendingChunk,
) error {
for _, chunk := range chunksToInsert {
dbChunk := &database.Chunk{ChunkHash: types.ChunkHash(chunk.Hash), Size: chunk.Size}
if err := p.repos.Chunks.Create(ctx, tx, dbChunk); err != nil {
dbChunk := &database.Chunk{
ChunkHash: types.ChunkHash(chunk.Hash), Size: chunk.Size,
}
err := p.repos.Chunks.Create(ctx, tx, dbChunk)
if err != nil {
return fmt.Errorf("creating chunk: %w", err)
}
}
@@ -460,62 +565,70 @@ func (p *Packer) commitBlobToDatabase(blobHash string, finalSize int64, chunksTo
BlobID: blobIDTyped, ChunkHash: types.ChunkHash(chunk.Hash),
Offset: chunk.Offset, Length: chunk.Size,
}
if err := p.repos.BlobChunks.Create(ctx, tx, blobChunk); err != nil {
err := p.repos.BlobChunks.Create(ctx, tx, blobChunk)
if err != nil {
return fmt.Errorf("creating blob_chunk: %w", err)
}
}
return p.repos.Blobs.UpdateFinished(ctx, tx, p.currentBlob.id, blobHash, p.currentBlob.size, finalSize)
})
if err != nil {
p.cleanupTempFile()
return fmt.Errorf("finalizing blob transaction: %w", err)
}
log.Debug("Committed blob transaction",
"chunks_inserted", len(chunksToInsert), "blob_chunks_inserted", len(p.currentBlob.chunks))
return nil
return p.repos.Blobs.UpdateFinished(ctx, tx, p.currentBlob.id, blobHash,
p.currentBlob.size, finalSize)
}
// deliverFinishedBlob passes the blob to the handler or stores it internally
func (p *Packer) deliverFinishedBlob(finished *FinishedBlob, insertedChunkHashes []string) error {
func (p *Packer) deliverFinishedBlob(
finished *FinishedBlob, insertedChunkHashes []string,
) error {
if p.blobHandler != nil {
if _, err := p.currentBlob.tempFile.Seek(0, io.SeekStart); err != nil {
_, err := p.currentBlob.tempFile.Seek(0, io.SeekStart)
if err != nil {
p.cleanupTempFile()
return fmt.Errorf("seeking for handler: %w", err)
}
blobWithReader := &BlobWithReader{
blobWithReader := &WithReader{
FinishedBlob: finished,
Reader: p.currentBlob.tempFile,
TempFile: p.currentBlob.tempFile,
InsertedChunkHashes: insertedChunkHashes,
}
if err := p.blobHandler(blobWithReader); err != nil {
err = p.blobHandler(blobWithReader)
if err != nil {
p.cleanupTempFile()
return fmt.Errorf("blob handler failed: %w", err)
}
p.currentBlob = nil
return nil
}
// No handler - read data for legacy behavior
log.Debug("No blob handler callback configured", "blob_hash", finished.Hash[:8]+"...")
if _, err := p.currentBlob.tempFile.Seek(0, io.SeekStart); err != nil {
_, err := p.currentBlob.tempFile.Seek(0, io.SeekStart)
if err != nil {
p.cleanupTempFile()
return fmt.Errorf("seeking to read data: %w", err)
}
data, err := io.ReadAll(p.currentBlob.tempFile)
if err != nil {
p.cleanupTempFile()
return fmt.Errorf("reading blob data: %w", err)
}
finished.Data = data
p.finishedBlobs = append(p.finishedBlobs, finished)
p.cleanupTempFile()
p.currentBlob = nil
return nil
}
@@ -527,24 +640,3 @@ func (p *Packer) cleanupTempFile() {
_ = p.fs.Remove(name)
}
}
// PackChunks is a convenience method to pack multiple chunks at once
func (p *Packer) PackChunks(chunks []*ChunkRef) error {
for _, chunk := range chunks {
err := p.AddChunk(chunk)
if err == ErrBlobSizeLimitExceeded {
// Finalize current blob and retry
if err := p.FinalizeBlob(); err != nil {
return fmt.Errorf("finalizing blob before retry: %w", err)
}
// Retry the chunk
if err := p.AddChunk(chunk); err != nil {
return fmt.Errorf("adding chunk %s after finalize: %w", chunk.Hash, err)
}
} else if err != nil {
return fmt.Errorf("adding chunk %s: %w", chunk.Hash, err)
}
}
return p.Flush()
}

View File

@@ -1,4 +1,4 @@
package blob
package blob_test
import (
"bytes"
@@ -6,12 +6,14 @@ import (
"crypto/sha256"
"database/sql"
"encoding/hex"
"errors"
"io"
"testing"
"filippo.io/age"
"github.com/klauspost/compress/zstd"
"github.com/spf13/afero"
"sneak.berlin/go/vaultik/internal/blob"
"sneak.berlin/go/vaultik/internal/database"
"sneak.berlin/go/vaultik/internal/log"
"sneak.berlin/go/vaultik/internal/types"
@@ -19,94 +21,98 @@ import (
const (
// Test key from test/insecure-integration-test.key
testPrivateKey = "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5"
testPrivateKey = "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7A" +
"PHXA2QS2NJA5"
testPublicKey = "age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05gl0sjq9q9wjg"
defaultMaxBlobSize = 10 * 1024 * 1024 // 10MB
testChunkSize = 1000
testChunkCount = 10
)
func TestPacker(t *testing.T) {
// Initialize logger for tests
log.Initialize(log.Config{})
// parseTestIdentity parses the fixed test age identity.
func parseTestIdentity(t *testing.T) *age.X25519Identity {
t.Helper()
// Parse test identity
identity, err := age.ParseX25519Identity(testPrivateKey)
if err != nil {
t.Fatalf("failed to parse test identity: %v", err)
}
t.Run("single chunk creates single blob", func(t *testing.T) {
// Create test database
return identity
}
// newTestPacker creates a test database and a Packer backed by it.
func newTestPacker(
t *testing.T, maxBlobSize int64,
) (*database.Repositories, *blob.Packer) {
t.Helper()
db, err := database.NewTestDB()
if err != nil {
t.Fatalf("failed to create test db: %v", err)
}
defer func() { _ = db.Close() }()
t.Cleanup(func() { _ = db.Close() })
repos := database.NewRepositories(db)
cfg := PackerConfig{
MaxBlobSize: 10 * 1024 * 1024, // 10MB
packer, err := blob.NewPacker(blob.PackerConfig{
MaxBlobSize: maxBlobSize,
CompressionLevel: 3,
Recipients: []string{testPublicKey},
Repositories: repos,
Fs: afero.NewMemMapFs(),
}
packer, err := NewPacker(cfg)
})
if err != nil {
t.Fatalf("failed to create packer: %v", err)
}
// Create a chunk
data := []byte("Hello, World!")
return repos, packer
}
// makeChunk creates a ChunkRef for data and registers the chunk in the
// database.
func makeChunk(
t *testing.T, repos *database.Repositories, data []byte,
) *blob.ChunkRef {
t.Helper()
hash := sha256.Sum256(data)
hashStr := hex.EncodeToString(hash[:])
// Create chunk in database first
dbChunk := &database.Chunk{
ChunkHash: types.ChunkHash(hashStr),
Size: int64(len(data)),
}
err = repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error {
err := repos.WithTx(
context.Background(),
func(ctx context.Context, tx *sql.Tx) error {
return repos.Chunks.Create(ctx, tx, dbChunk)
})
if err != nil {
t.Fatalf("failed to create chunk in db: %v", err)
}
chunk := &ChunkRef{
return &blob.ChunkRef{
Hash: hashStr,
Data: data,
}
}
// Add chunk
if err := packer.AddChunk(chunk); err != nil {
t.Fatalf("failed to add chunk: %v", err)
}
// decryptAndDecompress reverses the blob pipeline: age decrypt, then zstd
// decompress.
func decryptAndDecompress(
t *testing.T, blobData []byte, identity *age.X25519Identity,
) []byte {
t.Helper()
// Flush
if err := packer.Flush(); err != nil {
t.Fatalf("failed to flush: %v", err)
}
// Get finished blobs
blobs := packer.GetFinishedBlobs()
if len(blobs) != 1 {
t.Fatalf("expected 1 blob, got %d", len(blobs))
}
blob := blobs[0]
if len(blob.Chunks) != 1 {
t.Errorf("expected 1 chunk in blob, got %d", len(blob.Chunks))
}
// Note: Very small data may not compress well
t.Logf("Compression: %d -> %d bytes", blob.Uncompressed, blob.Compressed)
// Decrypt the blob data
decrypted, err := age.Decrypt(bytes.NewReader(blob.Data), identity)
decrypted, err := age.Decrypt(bytes.NewReader(blobData), identity)
if err != nil {
t.Fatalf("failed to decrypt blob: %v", err)
}
// Decompress the decrypted data
reader, err := zstd.NewReader(decrypted)
if err != nil {
t.Fatalf("failed to create decompressor: %v", err)
@@ -114,157 +120,144 @@ func TestPacker(t *testing.T) {
defer reader.Close()
var decompressed bytes.Buffer
if _, err := io.Copy(&decompressed, reader); err != nil {
_, err = io.Copy(&decompressed, reader)
if err != nil {
t.Fatalf("failed to decompress: %v", err)
}
if !bytes.Equal(decompressed.Bytes(), data) {
t.Error("decompressed data doesn't match original")
}
})
return decompressed.Bytes()
}
t.Run("multiple chunks packed together", func(t *testing.T) {
// Create test database
db, err := database.NewTestDB()
if err != nil {
t.Fatalf("failed to create test db: %v", err)
}
defer func() { _ = db.Close() }()
repos := database.NewRepositories(db)
func TestPackerSingleChunk(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
cfg := PackerConfig{
MaxBlobSize: 10 * 1024 * 1024, // 10MB
CompressionLevel: 3,
Recipients: []string{testPublicKey},
Repositories: repos,
Fs: afero.NewMemMapFs(),
}
packer, err := NewPacker(cfg)
if err != nil {
t.Fatalf("failed to create packer: %v", err)
}
identity := parseTestIdentity(t)
repos, packer := newTestPacker(t, defaultMaxBlobSize)
ctx := context.Background()
// Create multiple small chunks
chunks := make([]*ChunkRef, 10)
for i := 0; i < 10; i++ {
data := bytes.Repeat([]byte{byte(i)}, 1000)
hash := sha256.Sum256(data)
hashStr := hex.EncodeToString(hash[:])
data := []byte("Hello, World!")
chunk := makeChunk(t, repos, data)
// Create chunk in database first
dbChunk := &database.Chunk{
ChunkHash: types.ChunkHash(hashStr),
Size: int64(len(data)),
}
err = repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error {
return repos.Chunks.Create(ctx, tx, dbChunk)
})
if err != nil {
t.Fatalf("failed to create chunk in db: %v", err)
}
chunks[i] = &ChunkRef{
Hash: hashStr,
Data: data,
}
}
// Add all chunks
for _, chunk := range chunks {
err := packer.AddChunk(chunk)
err := packer.AddChunk(ctx, chunk)
if err != nil {
t.Fatalf("failed to add chunk: %v", err)
}
}
// Flush
if err := packer.Flush(); err != nil {
err = packer.Flush(ctx)
if err != nil {
t.Fatalf("failed to flush: %v", err)
}
// Should have one blob with all chunks
blobs := packer.GetFinishedBlobs()
if len(blobs) != 1 {
t.Fatalf("expected 1 blob, got %d", len(blobs))
}
if len(blobs[0].Chunks) != 10 {
t.Errorf("expected 10 chunks in blob, got %d", len(blobs[0].Chunks))
finished := blobs[0]
if len(finished.Chunks) != 1 {
t.Errorf("expected 1 chunk in blob, got %d", len(finished.Chunks))
}
// Note: Very small data may not compress well
t.Logf("Compression: %d -> %d bytes",
finished.Uncompressed, finished.Compressed)
decompressed := decryptAndDecompress(t, finished.Data, identity)
if !bytes.Equal(decompressed, data) {
t.Error("decompressed data doesn't match original")
}
}
func TestPackerMultipleChunks(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
repos, packer := newTestPacker(t, defaultMaxBlobSize)
ctx := context.Background()
chunks := make([]*blob.ChunkRef, testChunkCount)
for i := range testChunkCount {
data := bytes.Repeat([]byte{byte(i)}, testChunkSize)
chunks[i] = makeChunk(t, repos, data)
}
for _, chunk := range chunks {
err := packer.AddChunk(ctx, chunk)
if err != nil {
t.Fatalf("failed to add chunk: %v", err)
}
}
err := packer.Flush(ctx)
if err != nil {
t.Fatalf("failed to flush: %v", err)
}
blobs := packer.GetFinishedBlobs()
if len(blobs) != 1 {
t.Fatalf("expected 1 blob, got %d", len(blobs))
}
if len(blobs[0].Chunks) != testChunkCount {
t.Errorf("expected %d chunks in blob, got %d",
testChunkCount, len(blobs[0].Chunks))
}
// Verify offsets are correct
expectedOffset := int64(0)
for i, chunkRef := range blobs[0].Chunks {
if chunkRef.Offset != expectedOffset {
t.Errorf("chunk %d: expected offset %d, got %d", i, expectedOffset, chunkRef.Offset)
t.Errorf("chunk %d: expected offset %d, got %d",
i, expectedOffset, chunkRef.Offset)
}
if chunkRef.Length != 1000 {
t.Errorf("chunk %d: expected length 1000, got %d", i, chunkRef.Length)
if chunkRef.Length != testChunkSize {
t.Errorf("chunk %d: expected length %d, got %d",
i, testChunkSize, chunkRef.Length)
}
expectedOffset += chunkRef.Length
}
})
}
t.Run("blob size limit enforced", func(t *testing.T) {
// Create test database
db, err := database.NewTestDB()
if err != nil {
t.Fatalf("failed to create test db: %v", err)
}
defer func() { _ = db.Close() }()
repos := database.NewRepositories(db)
func TestPackerSizeLimit(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
// Small blob size limit to force multiple blobs
cfg := PackerConfig{
MaxBlobSize: 5000, // 5KB max
CompressionLevel: 3,
Recipients: []string{testPublicKey},
Repositories: repos,
Fs: afero.NewMemMapFs(),
}
packer, err := NewPacker(cfg)
if err != nil {
t.Fatalf("failed to create packer: %v", err)
}
const (
maxBlobSize = 5000 // 5KB max, forces multiple blobs
maxBlobawoOverhead = 6000 // allow some overhead over the limit
)
// Create chunks that will exceed the limit
chunks := make([]*ChunkRef, 10)
for i := 0; i < 10; i++ {
data := bytes.Repeat([]byte{byte(i)}, 1000) // 1KB each
hash := sha256.Sum256(data)
hashStr := hex.EncodeToString(hash[:])
repos, packer := newTestPacker(t, maxBlobSize)
ctx := context.Background()
// Create chunk in database first
dbChunk := &database.Chunk{
ChunkHash: types.ChunkHash(hashStr),
Size: int64(len(data)),
}
err = repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error {
return repos.Chunks.Create(ctx, tx, dbChunk)
})
if err != nil {
t.Fatalf("failed to create chunk in db: %v", err)
}
chunks[i] = &ChunkRef{
Hash: hashStr,
Data: data,
}
chunks := make([]*blob.ChunkRef, testChunkCount)
for i := range testChunkCount {
data := bytes.Repeat([]byte{byte(i)}, testChunkSize) // 1KB each
chunks[i] = makeChunk(t, repos, data)
}
blobCount := 0
// Add chunks and handle size limit errors
for _, chunk := range chunks {
err := packer.AddChunk(chunk)
if err == ErrBlobSizeLimitExceeded {
err := packer.AddChunk(ctx, chunk)
if errors.Is(err, blob.ErrBlobSizeLimitExceeded) {
// Finalize current blob
if err := packer.FinalizeBlob(); err != nil {
err = packer.FinalizeBlob(ctx)
if err != nil {
t.Fatalf("failed to finalize blob: %v", err)
}
blobCount++
// Retry adding the chunk
if err := packer.AddChunk(chunk); err != nil {
err = packer.AddChunk(ctx, chunk)
if err != nil {
t.Fatalf("failed to add chunk after finalize: %v", err)
}
} else if err != nil {
@@ -272,114 +265,54 @@ func TestPacker(t *testing.T) {
}
}
// Flush remaining
if err := packer.Flush(); err != nil {
err := packer.Flush(ctx)
if err != nil {
t.Fatalf("failed to flush: %v", err)
}
// Get all blobs
blobs := packer.GetFinishedBlobs()
totalBlobs := blobCount + len(blobs)
// Should have multiple blobs due to size limit
totalBlobs := blobCount + len(blobs)
if totalBlobs < 2 {
t.Errorf("expected multiple blobs due to size limit, got %d", totalBlobs)
}
// Verify each blob respects size limit (approximately)
for _, blob := range blobs {
if blob.Compressed > 6000 { // Allow some overhead
t.Errorf("blob size %d exceeds limit", blob.Compressed)
for _, finished := range blobs {
if finished.Compressed > maxBlobawoOverhead {
t.Errorf("blob size %d exceeds limit", finished.Compressed)
}
}
})
}
t.Run("with encryption", func(t *testing.T) {
// Create test database
db, err := database.NewTestDB()
if err != nil {
t.Fatalf("failed to create test db: %v", err)
}
defer func() { _ = db.Close() }()
repos := database.NewRepositories(db)
func TestPackerEncryption(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
// Generate test identity (using the one from parent test)
cfg := PackerConfig{
MaxBlobSize: 10 * 1024 * 1024, // 10MB
CompressionLevel: 3,
Recipients: []string{testPublicKey},
Repositories: repos,
Fs: afero.NewMemMapFs(),
}
packer, err := NewPacker(cfg)
if err != nil {
t.Fatalf("failed to create packer: %v", err)
}
identity := parseTestIdentity(t)
repos, packer := newTestPacker(t, defaultMaxBlobSize)
ctx := context.Background()
// Create test data
data := bytes.Repeat([]byte("Test data for encryption!"), 100)
hash := sha256.Sum256(data)
hashStr := hex.EncodeToString(hash[:])
chunk := makeChunk(t, repos, data)
// Create chunk in database first
dbChunk := &database.Chunk{
ChunkHash: types.ChunkHash(hashStr),
Size: int64(len(data)),
}
err = repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error {
return repos.Chunks.Create(ctx, tx, dbChunk)
})
err := packer.AddChunk(ctx, chunk)
if err != nil {
t.Fatalf("failed to create chunk in db: %v", err)
}
chunk := &ChunkRef{
Hash: hashStr,
Data: data,
}
// Add chunk and flush
if err := packer.AddChunk(chunk); err != nil {
t.Fatalf("failed to add chunk: %v", err)
}
if err := packer.Flush(); err != nil {
err = packer.Flush(ctx)
if err != nil {
t.Fatalf("failed to flush: %v", err)
}
// Get blob
blobs := packer.GetFinishedBlobs()
if len(blobs) != 1 {
t.Fatalf("expected 1 blob, got %d", len(blobs))
}
blob := blobs[0]
// Decrypt the blob
decrypted, err := age.Decrypt(bytes.NewReader(blob.Data), identity)
if err != nil {
t.Fatalf("failed to decrypt blob: %v", err)
}
var decryptedData bytes.Buffer
if _, err := decryptedData.ReadFrom(decrypted); err != nil {
t.Fatalf("failed to read decrypted data: %v", err)
}
// Decompress
reader, err := zstd.NewReader(&decryptedData)
if err != nil {
t.Fatalf("failed to create decompressor: %v", err)
}
defer reader.Close()
var decompressed bytes.Buffer
if _, err := decompressed.ReadFrom(reader); err != nil {
t.Fatalf("failed to decompress: %v", err)
}
// Verify data
if !bytes.Equal(decompressed.Bytes(), data) {
decompressed := decryptAndDecompress(t, blobs[0].Data, identity)
if !bytes.Equal(decompressed, data) {
t.Error("decrypted and decompressed data doesn't match original")
}
})
}

View File

@@ -1,3 +1,6 @@
// Package blobgen implements the blob data pipeline: streaming zstd
// compression, age encryption, and SHA256 content hashing for blob
// creation, plus the matching decrypt/decompress/verify reader.
package blobgen
import (
@@ -16,7 +19,9 @@ type CompressResult struct {
}
// CompressData compresses and encrypts data, returning the result with hash
func CompressData(data []byte, compressionLevel int, recipients []string) (*CompressResult, error) {
func CompressData(
data []byte, compressionLevel int, recipients []string,
) (*CompressResult, error) {
var buf bytes.Buffer
// Create writer
@@ -26,13 +31,16 @@ func CompressData(data []byte, compressionLevel int, recipients []string) (*Comp
}
// Write data
if _, err := w.Write(data); err != nil {
_, err = w.Write(data)
if err != nil {
_ = w.Close()
return nil, fmt.Errorf("writing data: %w", err)
}
// Close to flush
if err := w.Close(); err != nil {
err = w.Close()
if err != nil {
return nil, fmt.Errorf("closing writer: %w", err)
}
@@ -44,8 +52,11 @@ func CompressData(data []byte, compressionLevel int, recipients []string) (*Comp
}, nil
}
// CompressStream compresses and encrypts from reader to writer, returning hash
func CompressStream(dst io.Writer, src io.Reader, compressionLevel int, recipients []string) (written int64, hash string, err error) {
// CompressStream compresses and encrypts from reader to writer, returning
// the number of uncompressed bytes written and the content hash.
func CompressStream(
dst io.Writer, src io.Reader, compressionLevel int, recipients []string,
) (int64, string, error) {
// Create writer
w, err := NewWriter(dst, compressionLevel, recipients)
if err != nil {
@@ -60,14 +71,17 @@ func CompressStream(dst io.Writer, src io.Reader, compressionLevel int, recipien
}()
// Copy data
if _, err := io.Copy(w, src); err != nil {
_, err = io.Copy(w, src)
if err != nil {
return 0, "", fmt.Errorf("copying data: %w", err)
}
// Close to flush
if err := w.Close(); err != nil {
err = w.Close()
if err != nil {
return 0, "", fmt.Errorf("closing writer: %w", err)
}
closed = true
return w.BytesWritten(), hex.EncodeToString(w.Sum256()), nil

View File

@@ -1,4 +1,4 @@
package blobgen
package blobgen_test
import (
"bytes"
@@ -8,6 +8,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"sneak.berlin/go/vaultik/internal/blobgen"
)
// testRecipient is a static age recipient for tests.
@@ -19,35 +20,47 @@ const testRecipient = "age1cplgrwj77ta54dnmydvvmzn64ltk83ankxl5sww04mrtmu62kv3s8
// the explicit Close() on the happy path combined with defer Close() would
// cause a double close.
func TestCompressStreamNoDoubleClose(t *testing.T) {
t.Parallel()
input := []byte("regression test data for issue #28 double-close fix")
var buf bytes.Buffer
written, hash, err := CompressStream(&buf, bytes.NewReader(input), 3, []string{testRecipient})
written, hash, err := blobgen.CompressStream(
&buf, bytes.NewReader(input), 3, []string{testRecipient})
require.NoError(t, err, "CompressStream should not return an error")
assert.True(t, written > 0, "expected bytes written > 0")
assert.Positive(t, written, "expected bytes written > 0")
assert.NotEmpty(t, hash, "expected non-empty hash")
assert.True(t, buf.Len() > 0, "expected non-empty output")
assert.Positive(t, buf.Len(), "expected non-empty output")
}
// TestCompressStreamLargeInput exercises CompressStream with a larger payload
// to ensure no double-close issues surface under heavier I/O.
func TestCompressStreamLargeInput(t *testing.T) {
t.Parallel()
data := make([]byte, 512*1024) // 512 KB
_, err := rand.Read(data)
require.NoError(t, err)
var buf bytes.Buffer
written, hash, err := CompressStream(&buf, bytes.NewReader(data), 3, []string{testRecipient})
written, hash, err := blobgen.CompressStream(
&buf, bytes.NewReader(data), 3, []string{testRecipient})
require.NoError(t, err)
assert.True(t, written > 0)
assert.Positive(t, written)
assert.NotEmpty(t, hash)
}
// TestCompressStreamEmptyInput verifies CompressStream handles empty input
// without double-close issues.
func TestCompressStreamEmptyInput(t *testing.T) {
t.Parallel()
var buf bytes.Buffer
_, hash, err := CompressStream(&buf, strings.NewReader(""), 3, []string{testRecipient})
_, hash, err := blobgen.CompressStream(
&buf, strings.NewReader(""), 3, []string{testRecipient})
require.NoError(t, err)
assert.NotEmpty(t, hash)
}
@@ -55,10 +68,13 @@ func TestCompressStreamEmptyInput(t *testing.T) {
// TestCompressDataNoDoubleClose mirrors the stream test for CompressData,
// ensuring the explicit Close + error-path Close pattern is also safe.
func TestCompressDataNoDoubleClose(t *testing.T) {
t.Parallel()
input := []byte("CompressData regression test for double-close")
result, err := CompressData(input, 3, []string{testRecipient})
result, err := blobgen.CompressData(input, 3, []string{testRecipient})
require.NoError(t, err)
assert.True(t, result.CompressedSize > 0)
assert.True(t, result.UncompressedSize == int64(len(input)))
assert.Positive(t, result.CompressedSize)
assert.Equal(t, result.UncompressedSize, int64(len(input)))
assert.NotEmpty(t, result.SHA256)
}

View File

@@ -50,15 +50,17 @@ func NewReader(r io.Reader, identity age.Identity) (*Reader, error) {
}
// Read implements io.Reader
func (r *Reader) Read(p []byte) (n int, err error) {
n, err = r.teeReader.Read(p)
func (r *Reader) Read(p []byte) (int, error) {
n, err := r.teeReader.Read(p)
r.bytesRead += int64(n)
return n, err
}
// Close closes the decompressor
func (r *Reader) Close() error {
r.decompressor.Close()
return nil
}

View File

@@ -2,6 +2,7 @@ package blobgen
import (
"crypto/sha256"
"errors"
"fmt"
"hash"
"io"
@@ -11,6 +12,21 @@ import (
"github.com/klauspost/compress/zstd"
)
// Zstd compression level bounds accepted by NewWriter.
const (
minCompressionLevel = 1
maxCompressionLevel = 19
)
// reservedCompressionCPUs is how many CPUs are left free of zstd
// compression work for I/O and hashing.
const reservedCompressionCPUs = 2
// ErrInvalidCompressionLevel is returned when the zstd compression level
// is outside the accepted 1-19 range.
var ErrInvalidCompressionLevel = errors.New(
"invalid compression level: must be between 1 and 19")
// Writer wraps compression and encryption with SHA256 hashing.
// Data flows: input -> tee(hasher, compressor -> encryptor -> destination)
// The hash is computed on the uncompressed input for deterministic content-addressing.
@@ -23,11 +39,15 @@ type Writer struct {
bytesWritten int64
}
// NewWriter creates a new Writer that compresses, encrypts, and hashes data.
// The hash is computed on the uncompressed input for deterministic content-addressing.
func NewWriter(w io.Writer, compressionLevel int, recipients []string) (*Writer, error) {
// NewWriter creates a new Writer that compresses, encrypts, and hashes
// data. The hash is computed on the uncompressed input for deterministic
// content-addressing.
func NewWriter(
w io.Writer, compressionLevel int, recipients []string,
) (*Writer, error) {
// Validate compression level
if err := validateCompressionLevel(compressionLevel); err != nil {
err := validateCompressionLevel(compressionLevel)
if err != nil {
return nil, err
}
@@ -36,11 +56,13 @@ func NewWriter(w io.Writer, compressionLevel int, recipients []string) (*Writer,
// Parse recipients
var ageRecipients []age.Recipient
for _, recipient := range recipients {
r, err := age.ParseX25519Recipient(recipient)
if err != nil {
return nil, fmt.Errorf("parsing recipient %s: %w", recipient, err)
}
ageRecipients = append(ageRecipients, r)
}
@@ -51,10 +73,7 @@ func NewWriter(w io.Writer, compressionLevel int, recipients []string) (*Writer,
}
// Calculate compression concurrency: CPUs - 2, minimum 1
concurrency := runtime.NumCPU() - 2
if concurrency < 1 {
concurrency = 1
}
concurrency := max(runtime.NumCPU()-reservedCompressionCPUs, 1)
// Create compression writer with encryption as destination
compressor, err := zstd.NewWriter(encWriter,
@@ -63,6 +82,7 @@ func NewWriter(w io.Writer, compressionLevel int, recipients []string) (*Writer,
)
if err != nil {
_ = encWriter.Close()
return nil, fmt.Errorf("creating compression writer: %w", err)
}
@@ -79,21 +99,24 @@ func NewWriter(w io.Writer, compressionLevel int, recipients []string) (*Writer,
}
// Write implements io.Writer
func (w *Writer) Write(p []byte) (n int, err error) {
n, err = w.teeWriter.Write(p)
func (w *Writer) Write(p []byte) (int, error) {
n, err := w.teeWriter.Write(p)
w.bytesWritten += int64(n)
return n, err
}
// Close closes all layers and returns any errors
func (w *Writer) Close() error {
// Close compressor first
if err := w.compressor.Close(); err != nil {
err := w.compressor.Close()
if err != nil {
return fmt.Errorf("closing compressor: %w", err)
}
// Then close encryptor
if err := w.encryptor.Close(); err != nil {
err = w.encryptor.Close()
if err != nil {
return fmt.Errorf("closing encryptor: %w", err)
}
@@ -109,6 +132,7 @@ func (w *Writer) Sum256() []byte {
firstHash := w.hasher.Sum(nil)
// Second hash: SHA256(firstHash) - this is the blob ID
secondHash := sha256.Sum256(firstHash)
return secondHash[:]
}
@@ -119,9 +143,11 @@ func (w *Writer) BytesWritten() int64 {
func validateCompressionLevel(level int) error {
// Zstd compression levels: 1-19 (default is 3)
// SpeedFastest = 1, SpeedDefault = 3, SpeedBetterCompression = 7, SpeedBestCompression = 11
if level < 1 || level > 19 {
return fmt.Errorf("invalid compression level %d: must be between 1 and 19", level)
// SpeedFastest = 1, SpeedDefault = 3, SpeedBetterCompression = 7,
// SpeedBestCompression = 11
if level < minCompressionLevel || level > maxCompressionLevel {
return fmt.Errorf("%w: got %d", ErrInvalidCompressionLevel, level)
}
return nil
}

View File

@@ -1,4 +1,4 @@
package blobgen
package blobgen_test
import (
"bytes"
@@ -9,12 +9,15 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"sneak.berlin/go/vaultik/internal/blobgen"
)
// TestWriterHashIsDoubleHash verifies that Writer.Sum256() returns
// the double hash SHA256(SHA256(plaintext)) for security.
// Double hashing prevents attackers from confirming existence of known content.
func TestWriterHashIsDoubleHash(t *testing.T) {
t.Parallel()
// Test data - random data that doesn't compress well
testData := make([]byte, 1024*1024) // 1MB
_, err := rand.Read(testData)
@@ -27,7 +30,7 @@ func TestWriterHashIsDoubleHash(t *testing.T) {
var encryptedBuf bytes.Buffer
// Create blobgen writer
writer, err := NewWriter(&encryptedBuf, 3, []string{testRecipient})
writer, err := blobgen.NewWriter(&encryptedBuf, 3, []string{testRecipient})
require.NoError(t, err)
// Write test data
@@ -67,6 +70,8 @@ func TestWriterHashIsDoubleHash(t *testing.T) {
// TestWriterDeterministicHash verifies that the same input always produces
// the same hash, even with non-deterministic encryption.
func TestWriterDeterministicHash(t *testing.T) {
t.Parallel()
// Test data
testData := []byte("Hello, World! This is test data for deterministic hashing.")
@@ -76,13 +81,13 @@ func TestWriterDeterministicHash(t *testing.T) {
// Create two writers and verify they produce the same hash
var buf1, buf2 bytes.Buffer
writer1, err := NewWriter(&buf1, 3, []string{testRecipient})
writer1, err := blobgen.NewWriter(&buf1, 3, []string{testRecipient})
require.NoError(t, err)
_, err = writer1.Write(testData)
require.NoError(t, err)
require.NoError(t, writer1.Close())
writer2, err := NewWriter(&buf2, 3, []string{testRecipient})
writer2, err := blobgen.NewWriter(&buf2, 3, []string{testRecipient})
require.NoError(t, err)
_, err = writer2.Write(testData)
require.NoError(t, err)

View File

@@ -1,16 +1,21 @@
// Package chunker splits input data into content-defined chunks using the
// FastCDC algorithm so that identical data sequences produce identical
// chunks regardless of their position in the file.
package chunker
import (
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io"
"os"
)
// Chunk represents a single chunk of data produced by the content-defined chunking algorithm.
// Each chunk is identified by its SHA256 hash and contains the raw data along with
// its position and size information from the original file.
// Chunk represents a single chunk of data produced by the content-defined
// chunking algorithm. Each chunk is identified by its SHA256 hash and
// contains the raw data along with its position and size information from
// the original file.
type Chunk struct {
Hash string // Content hash of the chunk
Data []byte // Chunk data
@@ -28,6 +33,10 @@ type Chunker struct {
maxChunkSize int
}
// chunkSizeSpread is the FastCDC-recommended factor between the average
// chunk size and the minimum (avg/spread) and maximum (avg*spread) sizes.
const chunkSizeSpread = 4
// NewChunker creates a new chunker with the specified average chunk size.
// The actual chunk sizes will vary between avgChunkSize/4 and avgChunkSize*4
// as recommended by the FastCDC algorithm. Typical values for avgChunkSize
@@ -36,27 +45,31 @@ func NewChunker(avgChunkSize int64) *Chunker {
// FastCDC recommends min = avg/4 and max = avg*4
return &Chunker{
avgChunkSize: int(avgChunkSize),
minChunkSize: int(avgChunkSize / 4),
maxChunkSize: int(avgChunkSize * 4),
minChunkSize: int(avgChunkSize / chunkSizeSpread),
maxChunkSize: int(avgChunkSize * chunkSizeSpread),
}
}
// ChunkReader splits the reader into content-defined chunks and returns all chunks at once.
// This method loads all chunk data into memory, so it should only be used for
// reasonably sized inputs. For large files or streams, use ChunkReaderStreaming instead.
// ChunkReader splits the reader into content-defined chunks and returns all
// chunks at once. This method loads all chunk data into memory, so it should
// only be used for reasonably sized inputs. For large files or streams, use
// ChunkReaderStreaming instead.
// Returns an error if chunking fails or if reading from the input fails.
func (c *Chunker) ChunkReader(r io.Reader) ([]Chunk, error) {
chunker := AcquireReusableChunker(r, c.minChunkSize, c.avgChunkSize, c.maxChunkSize)
chunker := AcquireReusableChunker(
r, c.minChunkSize, c.avgChunkSize, c.maxChunkSize)
defer chunker.Release()
var chunks []Chunk
offset := int64(0)
for {
chunk, err := chunker.Next()
if err == io.EOF {
if errors.Is(err, io.EOF) {
break
}
if err != nil {
return nil, fmt.Errorf("reading chunk: %w", err)
}
@@ -83,30 +96,36 @@ func (c *Chunker) ChunkReader(r io.Reader) ([]Chunk, error) {
// ChunkCallback is a function called for each chunk as it's processed.
// The callback receives a Chunk containing the hash, data, offset, and size.
// If the callback returns an error, chunk processing stops and the error is propagated.
// If the callback returns an error, chunk processing stops and the error is
// propagated.
type ChunkCallback func(chunk Chunk) error
// ChunkReaderStreaming splits the reader into chunks and calls the callback for each chunk.
// This is the preferred method for processing large files or streams as it doesn't
// accumulate all chunks in memory. The callback is invoked for each chunk as it's
// produced, allowing for streaming processing and immediate storage or transmission.
// Returns the SHA256 hash of the entire file content and an error if chunking fails,
// reading fails, or if the callback returns an error.
func (c *Chunker) ChunkReaderStreaming(r io.Reader, callback ChunkCallback) (string, error) {
// ChunkReaderStreaming splits the reader into chunks and calls the callback
// for each chunk. This is the preferred method for processing large files or
// streams as it doesn't accumulate all chunks in memory. The callback is
// invoked for each chunk as it's produced, allowing for streaming processing
// and immediate storage or transmission.
// Returns the SHA256 hash of the entire file content and an error if
// chunking fails, reading fails, or if the callback returns an error.
func (c *Chunker) ChunkReaderStreaming(
r io.Reader, callback ChunkCallback,
) (string, error) {
// Create a tee reader to calculate full file hash while chunking
fileHasher := sha256.New()
teeReader := io.TeeReader(r, fileHasher)
chunker := AcquireReusableChunker(teeReader, c.minChunkSize, c.avgChunkSize, c.maxChunkSize)
chunker := AcquireReusableChunker(
teeReader, c.minChunkSize, c.avgChunkSize, c.maxChunkSize)
defer chunker.Release()
offset := int64(0)
for {
chunk, err := chunker.Next()
if err == io.EOF {
if errors.Is(err, io.EOF) {
break
}
if err != nil {
return "", fmt.Errorf("reading chunk: %w", err)
}
@@ -114,15 +133,17 @@ func (c *Chunker) ChunkReaderStreaming(r io.Reader, callback ChunkCallback) (str
// Calculate chunk hash
hash := sha256.Sum256(chunk.Data)
// Pass the data directly - caller must process it before we call Next() again
// (chunker reuses its internal buffer, but since we process synchronously
// and completely before continuing, no copy is needed)
if err := callback(Chunk{
// Pass the data directly - caller must process it before we call
// Next() again (chunker reuses its internal buffer, but since we
// process synchronously and completely before continuing, no copy
// is needed)
err = callback(Chunk{
Hash: hex.EncodeToString(hash[:]),
Data: chunk.Data,
Offset: offset,
Size: int64(len(chunk.Data)),
}); err != nil {
})
if err != nil {
return "", fmt.Errorf("callback error: %w", err)
}
@@ -138,12 +159,13 @@ func (c *Chunker) ChunkReaderStreaming(r io.Reader, callback ChunkCallback) (str
// For large files, consider using ChunkReaderStreaming with a file handle instead.
// Returns an error if the file cannot be opened or if chunking fails.
func (c *Chunker) ChunkFile(path string) ([]Chunk, error) {
file, err := os.Open(path)
file, err := os.Open(path) //nolint:gosec // G304: path is caller-supplied by design
if err != nil {
return nil, fmt.Errorf("opening file: %w", err)
}
defer func() {
if err := file.Close(); err != nil && err.Error() != "invalid argument" {
err := file.Close()
if err != nil && err.Error() != "invalid argument" {
// Log error or handle as needed
_ = err
}

View File

@@ -1,11 +1,15 @@
package chunker
package chunker_test
import (
"bytes"
"testing"
"sneak.berlin/go/vaultik/internal/chunker"
)
func TestChunkerExpectedChunkCount(t *testing.T) {
t.Parallel()
tests := []struct {
name string
fileSize int
@@ -38,16 +42,19 @@ func TestChunkerExpectedChunkCount(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
chunker := NewChunker(tt.avgChunkSize)
t.Parallel()
c := chunker.NewChunker(tt.avgChunkSize)
// Create data with some variation to trigger chunk boundaries
data := make([]byte, tt.fileSize)
for i := 0; i < len(data); i++ {
for i := range data {
// Use a pattern that should create boundaries
//nolint:gosec // G115: intentional byte truncation
data[i] = byte((i * 17) ^ (i >> 5))
}
chunks, err := chunker.ChunkReader(bytes.NewReader(data))
chunks, err := c.ChunkReader(bytes.NewReader(data))
if err != nil {
t.Fatalf("chunking failed: %v", err)
}
@@ -59,6 +66,7 @@ func TestChunkerExpectedChunkCount(t *testing.T) {
t.Errorf("too few chunks: got %d, expected at least %d",
len(chunks), tt.minExpected)
}
if len(chunks) > tt.maxExpected {
t.Errorf("too many chunks: got %d, expected at most %d",
len(chunks), tt.maxExpected)
@@ -69,6 +77,7 @@ func TestChunkerExpectedChunkCount(t *testing.T) {
for _, chunk := range chunks {
reconstructed = append(reconstructed, chunk.Data...)
}
if !bytes.Equal(data, reconstructed) {
t.Error("reconstructed data doesn't match original")
}

View File

@@ -1,17 +1,20 @@
package chunker
package chunker_test
import (
"bytes"
"crypto/rand"
"testing"
"sneak.berlin/go/vaultik/internal/chunker"
)
func TestChunker(t *testing.T) {
t.Run("small file produces single chunk", func(t *testing.T) {
chunker := NewChunker(1024 * 1024) // 1MB average
func TestChunkerSmallFileSingleChunk(t *testing.T) {
t.Parallel()
c := chunker.NewChunker(1024 * 1024) // 1MB average
data := bytes.Repeat([]byte("hello"), 100) // 500 bytes
chunks, err := chunker.ChunkReader(bytes.NewReader(data))
chunks, err := c.ChunkReader(bytes.NewReader(data))
if err != nil {
t.Fatalf("chunking failed: %v", err)
}
@@ -23,23 +26,28 @@ func TestChunker(t *testing.T) {
if chunks[0].Size != int64(len(data)) {
t.Errorf("expected chunk size %d, got %d", len(data), chunks[0].Size)
}
})
}
t.Run("large file produces multiple chunks", func(t *testing.T) {
chunker := NewChunker(256 * 1024) // 256KB average chunk size
func TestChunkerLargeFileMultipleChunks(t *testing.T) {
t.Parallel()
c := chunker.NewChunker(256 * 1024) // 256KB average chunk size
// Generate 2MB of random data
data := make([]byte, 2*1024*1024)
if _, err := rand.Read(data); err != nil {
_, err := rand.Read(data)
if err != nil {
t.Fatalf("failed to generate random data: %v", err)
}
chunks, err := chunker.ChunkReader(bytes.NewReader(data))
chunks, err := c.ChunkReader(bytes.NewReader(data))
if err != nil {
t.Fatalf("chunking failed: %v", err)
}
// Should produce multiple chunks - with FastCDC we expect around 8 chunks for 2MB with 256KB average
// Should produce multiple chunks - with FastCDC we expect around 8
// chunks for 2MB with 256KB average
if len(chunks) < 4 || len(chunks) > 16 {
t.Errorf("expected 4-16 chunks, got %d", len(chunks))
}
@@ -56,17 +64,22 @@ func TestChunker(t *testing.T) {
// Verify offsets
var expectedOffset int64
for i, chunk := range chunks {
if chunk.Offset != expectedOffset {
t.Errorf("chunk %d: expected offset %d, got %d", i, expectedOffset, chunk.Offset)
t.Errorf("chunk %d: expected offset %d, got %d",
i, expectedOffset, chunk.Offset)
}
expectedOffset += chunk.Size
}
})
}
t.Run("deterministic chunking", func(t *testing.T) {
chunker1 := NewChunker(256 * 1024)
chunker2 := NewChunker(256 * 1024)
func TestChunkerDeterministic(t *testing.T) {
t.Parallel()
chunker1 := chunker.NewChunker(256 * 1024)
chunker2 := chunker.NewChunker(256 * 1024)
// Use deterministic data
data := bytes.Repeat([]byte("abcdefghijklmnopqrstuvwxyz"), 20000) // ~520KB
@@ -83,22 +96,25 @@ func TestChunker(t *testing.T) {
// Should produce same chunks
if len(chunks1) != len(chunks2) {
t.Fatalf("different number of chunks: %d vs %d", len(chunks1), len(chunks2))
t.Fatalf("different number of chunks: %d vs %d",
len(chunks1), len(chunks2))
}
for i := range chunks1 {
if chunks1[i].Hash != chunks2[i].Hash {
t.Errorf("chunk %d: different hashes", i)
}
if chunks1[i].Size != chunks2[i].Size {
t.Errorf("chunk %d: different sizes", i)
}
}
})
}
func TestChunkBoundaries(t *testing.T) {
chunker := NewChunker(256 * 1024) // 256KB average
t.Parallel()
c := chunker.NewChunker(256 * 1024) // 256KB average
// FastCDC uses avg/4 for min and avg*4 for max
avgSize := int64(256 * 1024)
@@ -107,11 +123,13 @@ func TestChunkBoundaries(t *testing.T) {
// Test that minimum chunk size is respected
data := make([]byte, minSize+1024)
if _, err := rand.Read(data); err != nil {
_, err := rand.Read(data)
if err != nil {
t.Fatalf("failed to generate random data: %v", err)
}
chunks, err := chunker.ChunkReader(bytes.NewReader(data))
chunks, err := c.ChunkReader(bytes.NewReader(data))
if err != nil {
t.Fatalf("chunking failed: %v", err)
}
@@ -119,10 +137,13 @@ func TestChunkBoundaries(t *testing.T) {
for i, chunk := range chunks {
// Last chunk can be smaller than minimum
if i < len(chunks)-1 && chunk.Size < minSize {
t.Errorf("chunk %d size %d is below minimum %d", i, chunk.Size, minSize)
t.Errorf("chunk %d size %d is below minimum %d",
i, chunk.Size, minSize)
}
if chunk.Size > maxSize {
t.Errorf("chunk %d size %d exceeds maximum %d", i, chunk.Size, maxSize)
t.Errorf("chunk %d size %d exceeds maximum %d",
i, chunk.Size, maxSize)
}
}
}

View File

@@ -1,6 +1,7 @@
package chunker
import (
"errors"
"io"
"math"
"sync"
@@ -27,32 +28,52 @@ type ReusableChunker struct {
}
// reusableChunkerPool pools ReusableChunker instances to avoid allocations.
//
//nolint:gochecknoglobals // process-wide object pool by design
var reusableChunkerPool = sync.Pool{
New: func() interface{} {
New: func() any {
return &ReusableChunker{}
},
}
// bufferPools contains pools for different buffer sizes.
// Key is the buffer size.
//
//nolint:gochecknoglobals // process-wide buffer pools by design
var bufferPools = sync.Map{}
func getBuffer(size int) []byte {
poolI, _ := bufferPools.LoadOrStore(size, &sync.Pool{
New: func() interface{} {
New: func() any {
buf := make([]byte, size)
return &buf
},
})
pool := poolI.(*sync.Pool)
return *pool.Get().(*[]byte)
pool, ok := poolI.(*sync.Pool)
if !ok {
panic("bufferPools holds a non-pool value")
}
buf, ok := pool.Get().(*[]byte)
if !ok {
panic("buffer pool holds a non-buffer value")
}
return *buf
}
func putBuffer(buf []byte) {
size := cap(buf)
poolI, ok := bufferPools.Load(size)
if ok {
pool := poolI.(*sync.Pool)
pool, isPool := poolI.(*sync.Pool)
if !isPool {
panic("bufferPools holds a non-pool value")
}
b := buf[:size]
pool.Put(&b)
}
@@ -66,17 +87,28 @@ type FastCDCChunk struct {
Fingerprint uint64
}
// AcquireReusableChunker gets a chunker from the pool and initializes it for the given reader.
func AcquireReusableChunker(rd io.Reader, minSize, avgSize, maxSize int) *ReusableChunker {
c := reusableChunkerPool.Get().(*ReusableChunker)
// bufSizeFactor sizes the internal read buffer relative to the maximum
// chunk size so a full chunk plus read-ahead always fits.
const bufSizeFactor = 2
bufSize := maxSize * 2
// AcquireReusableChunker gets a chunker from the pool and initializes it
// for the given reader.
func AcquireReusableChunker(
rd io.Reader, minSize, avgSize, maxSize int,
) *ReusableChunker {
c, ok := reusableChunkerPool.Get().(*ReusableChunker)
if !ok {
panic("reusableChunkerPool holds a non-chunker value")
}
bufSize := maxSize * bufSizeFactor
// Reuse buffer if it's the right size, otherwise get a new one
if c.buf == nil || cap(c.buf) != bufSize {
if c.buf != nil {
putBuffer(c.buf)
}
c.buf = getBuffer(bufSize)
} else {
// Restore buffer to full capacity (may have been truncated by previous EOF)
@@ -108,41 +140,14 @@ func (c *ReusableChunker) Release() {
reusableChunkerPool.Put(c)
}
func (c *ReusableChunker) fillBuffer() error {
n := len(c.buf) - c.cursor
if n >= c.maxSize {
return nil
}
// Move all data after the cursor to the start of the buffer
copy(c.buf[:n], c.buf[c.cursor:])
c.cursor = 0
if c.eof {
c.buf = c.buf[:n]
return nil
}
// Restore buffer to full capacity for reading
c.buf = c.buf[:c.bufSize]
// Fill the rest of the buffer
m, err := io.ReadFull(c.rd, c.buf[n:])
if err == io.EOF || err == io.ErrUnexpectedEOF {
c.buf = c.buf[:n+m]
c.eof = true
} else if err != nil {
return err
}
return nil
}
// Next returns the next chunk or io.EOF when done.
// The returned Data slice is only valid until the next call to Next.
func (c *ReusableChunker) Next() (FastCDCChunk, error) {
if err := c.fillBuffer(); err != nil {
err := c.fillBuffer()
if err != nil {
return FastCDCChunk{}, err
}
if len(c.buf) == 0 {
return FastCDCChunk{}, io.EOF
}
@@ -162,6 +167,37 @@ func (c *ReusableChunker) Next() (FastCDCChunk, error) {
return chunk, nil
}
func (c *ReusableChunker) fillBuffer() error {
n := len(c.buf) - c.cursor
if n >= c.maxSize {
return nil
}
// Move all data after the cursor to the start of the buffer
copy(c.buf[:n], c.buf[c.cursor:])
c.cursor = 0
if c.eof {
c.buf = c.buf[:n]
return nil
}
// Restore buffer to full capacity for reading
c.buf = c.buf[:c.bufSize]
// Fill the rest of the buffer
m, err := io.ReadFull(c.rd, c.buf[n:])
if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
c.buf = c.buf[:n+m]
c.eof = true
} else if err != nil {
return err
}
return nil
}
func (c *ReusableChunker) nextChunk(data []byte) (int, uint64) {
fp := uint64(0)
i := c.minSize
@@ -189,14 +225,9 @@ func (c *ReusableChunker) nextChunk(data []byte) (int, uint64) {
return i, fp
}
func min(a, b int) int {
if a < b {
return a
}
return b
}
// 256 random uint64s for the rolling hash function (from FastCDC paper)
//
//nolint:gochecknoglobals // immutable FastCDC gear lookup table
var table = [256]uint64{
0xe80e8d55032474b3, 0x11b25b61f5924e15, 0x03aa5bd82a9eb669, 0xc45a153ef107a38c,
0xeac874b86f0f57b9, 0xa5ccedec95ec79c7, 0xe15a3320ad42ac0a, 0x5ed3583fa63cec15,

View File

@@ -1,3 +1,6 @@
// Package cli implements the vaultik command-line interface: cobra
// commands, fx application wiring, and process-level concerns such as
// signal handling and the PID lock.
package cli
import (
@@ -12,6 +15,7 @@ import (
"time"
"github.com/adrg/xdg"
"github.com/spf13/cobra"
"go.uber.org/fx"
"sneak.berlin/go/vaultik/internal/config"
"sneak.berlin/go/vaultik/internal/database"
@@ -24,12 +28,16 @@ import (
"sneak.berlin/go/vaultik/internal/vaultik"
)
// shutdownTimeout bounds how long a signal-triggered graceful shutdown
// may take before we give up.
const shutdownTimeout = 30 * time.Second
// AppOptions contains common options for creating the fx application.
// It includes the configuration file path, logging options, and additional
// fx modules and invocations that should be included in the application.
type AppOptions struct {
ConfigPath string
LogOptions log.LogOptions
LogOptions log.Options
Modules []fx.Option
Invokes []fx.Option
}
@@ -38,15 +46,19 @@ type AppOptions struct {
// flag is active, marks the UI writer quiet so that Begin/Complete/
// Info/Notice/Detail/Progress are silenced. Warning and Error are NOT
// silenced — per the documented convention that --quiet suppresses
// non-error output only. The startup banner is printed by CLIEntry
// non-error output only. The startup banner is printed by Entry
// before cobra parses arguments, gated by the same arg-level check.
func setupGlobals(lc fx.Lifecycle, g *globals.Globals, v *vaultik.Vaultik, opts log.LogOptions) {
func setupGlobals(
lc fx.Lifecycle, g *globals.Globals, v *vaultik.Vaultik, opts log.Options,
) {
lc.Append(fx.Hook{
OnStart: func(ctx context.Context) error {
OnStart: func(_ context.Context) error {
g.StartTime = time.Now().UTC()
if opts.Cron || opts.Quiet {
v.UI.SetQuiet(true)
}
return nil
},
})
@@ -56,12 +68,12 @@ func setupGlobals(lc fx.Lifecycle, g *globals.Globals, v *vaultik.Vaultik, opts
// blank line. Used both from the fx hook (for subcommand invocations) and
// from the root cobra Run handler (for `vaultik` with no subcommand).
func writeStartupBanner(w *ui.Writer, startTime time.Time, shortCommit string) {
w.Banner("%s %s by %s (commit %s, built on %s) starting up at %s.",
w.Bannerf("%s %s by %s (commit %s, built on %s) starting up at %s.",
globals.Appname, globals.Version, globals.Author,
shortCommit, globals.CommitDate,
startTime.Format(time.RFC3339))
w.Banner("%s", globals.Homepage)
w.Banner("")
w.Bannerf("%s", globals.Homepage)
w.Bannerf("")
}
// NewApp creates a new fx application with common modules.
@@ -70,7 +82,7 @@ func writeStartupBanner(w *ui.Writer, startTime time.Time, shortCommit string) {
// The returned fx.App is ready to be started with RunApp.
func NewApp(opts AppOptions) *fx.App {
baseModules := []fx.Option{
fx.Supply(config.ConfigPath(opts.ConfigPath)),
fx.Supply(config.Path(opts.ConfigPath)),
fx.Supply(opts.LogOptions),
fx.Provide(globals.New),
fx.Provide(log.New),
@@ -84,12 +96,27 @@ func NewApp(opts AppOptions) *fx.App {
fx.NopLogger,
}
allOptions := append(baseModules, opts.Modules...)
capacity := len(baseModules) + len(opts.Modules) + len(opts.Invokes)
allOptions := make([]fx.Option, 0, capacity)
allOptions = append(allOptions, baseModules...)
allOptions = append(allOptions, opts.Modules...)
allOptions = append(allOptions, opts.Invokes...)
return fx.New(allOptions...)
}
// startupError carries a startup failure message that has been cleaned
// of fx dependency-injection noise. A distinct type (rather than
// errors.New) keeps the dynamic message out of err113's sight while
// preserving the exact user-facing text.
type startupError struct {
msg string
}
func (e *startupError) Error() string {
return e.msg
}
// cleanStartupError strips fx's dependency-injection call-chain noise from
// startup errors. fx wraps the underlying error with messages like
//
@@ -105,7 +132,8 @@ func cleanStartupError(err error) error {
if idx := strings.LastIndex(msg, "): "); idx >= 0 {
msg = msg[idx+3:]
}
return errors.New(msg)
return &startupError{msg: msg}
}
// RunApp starts and stops the fx application within the given context.
@@ -122,7 +150,8 @@ func RunApp(ctx context.Context, app *fx.App) error {
defer cancel()
// Start the app
if err := app.Start(ctx); err != nil {
err := app.Start(ctx)
if err != nil {
return cleanStartupError(err)
}
@@ -130,28 +159,35 @@ func RunApp(ctx context.Context, app *fx.App) error {
shutdownComplete := make(chan struct{})
go func() {
defer close(shutdownComplete)
<-sigChan
log.Notice("Received interrupt signal, shutting down gracefully...")
// Create a timeout context for shutdown
shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 30*time.Second)
// Create a timeout context for shutdown. The parent ctx is being
// cancelled, so detach from its cancellation but keep its values.
shutdownCtx, shutdownCancel := context.WithTimeout(
context.WithoutCancel(ctx), shutdownTimeout)
defer shutdownCancel()
if err := app.Stop(shutdownCtx); err != nil {
err := app.Stop(shutdownCtx)
if err != nil {
log.Error("Error during shutdown", "error", err)
}
}()
// Wait for either the signal handler to complete shutdown or the app to request shutdown
// Wait for the signal handler to complete shutdown or the app to
// request shutdown.
select {
case <-shutdownComplete:
// Shutdown completed via signal
return nil
case <-ctx.Done():
// Context cancelled (shouldn't happen in normal operation)
if err := app.Stop(context.Background()); err != nil {
err := app.Stop(context.WithoutCancel(ctx))
if err != nil {
log.Error("Error stopping app", "error", err)
}
return ctx.Err()
case <-app.Done():
// App finished running (e.g., backup completed)
@@ -159,6 +195,68 @@ func RunApp(ctx context.Context, app *fx.App) error {
}
}
// runVaultikApp runs the standard single-operation command lifecycle
// shared by the list/purge/verify/remove/remote-info subcommands:
// resolve the config, start the fx app, run op against the Vaultik
// instance in a goroutine, report a failure prefixed with failMsg
// (suppressed while suppressErrors is true, e.g. under --json), then
// trigger shutdown. The operation is cancelled when the app stops.
// extraQuiet is OR-ed into LogOptions.Quiet (e.g. --json output modes).
func runVaultikApp(
cmd *cobra.Command, extraQuiet, suppressErrors bool,
failMsg string, op func(v *vaultik.Vaultik) error,
) error {
configPath, err := ResolveConfigPath()
if err != nil {
return err
}
rootFlags := GetRootFlags()
return RunWithApp(cmd.Context(), AppOptions{
ConfigPath: configPath,
LogOptions: log.Options{
Verbose: rootFlags.Verbose,
Debug: rootFlags.Debug,
Quiet: rootFlags.Quiet || extraQuiet,
},
Modules: []fx.Option{},
Invokes: []fx.Option{
fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) {
lc.Append(fx.Hook{
OnStart: func(_ context.Context) error {
go func() {
err := op(v)
if err != nil {
if !errors.Is(err, context.Canceled) {
if !suppressErrors {
log.Error(failMsg, "error", err)
ReportErrorf("%s: %v", failMsg, err)
}
os.Exit(1)
}
}
err = v.Shutdowner.Shutdown()
if err != nil {
log.Error("Failed to shutdown", "error", err)
}
}()
return nil
},
OnStop: func(_ context.Context) error {
v.Cancel()
return nil
},
})
}),
},
})
}
// RunWithApp is a helper that creates and runs an fx app with the given options.
// It combines NewApp and RunApp into a single convenient function. This is the
// preferred way to run CLI commands that need the full application context.
@@ -166,19 +264,24 @@ func RunApp(ctx context.Context, app *fx.App) error {
func RunWithApp(ctx context.Context, opts AppOptions) error {
// Acquire PID lock to prevent concurrent instances
lockDir := filepath.Join(xdg.DataHome, "vaultik")
lock, err := pidlock.Acquire(lockDir)
if err != nil {
if errors.Is(err, pidlock.ErrAlreadyRunning) {
return fmt.Errorf("cannot start: %w", err)
}
return fmt.Errorf("failed to acquire lock: %w", err)
}
defer func() {
if err := lock.Release(); err != nil {
err := lock.Release()
if err != nil {
log.Warn("Failed to release PID lock", "error", err)
}
}()
app := NewApp(opts)
return RunApp(ctx, app)
}

View File

@@ -1,4 +1,4 @@
package cli
package cli //nolint:testpackage // needs access to unexported cleanStartupError
import (
"errors"
@@ -6,6 +6,8 @@ import (
)
func TestCleanStartupError(t *testing.T) {
t.Parallel()
tests := []struct {
name string
in string
@@ -13,7 +15,18 @@ func TestCleanStartupError(t *testing.T) {
}{
{
name: "real fx error chain",
in: `could not build arguments for function "sneak.berlin/go/vaultik/internal/cli".newSnapshotCreateCommand.func1.1 (/Users/user/dev/vaultik/internal/cli/snapshot.go:71): failed to build *vaultik.Vaultik: could not build arguments for function "sneak.berlin/go/vaultik/internal/vaultik".New (/Users/user/dev/vaultik/internal/vaultik/vaultik.go:59): failed to build storage.Storer: received non-nil error from function "sneak.berlin/go/vaultik/internal/storage".NewStorer (/Users/user/dev/vaultik/internal/storage/module.go:23): creating base path: mkdir /Volumes/BACKUPS: permission denied`,
in: `could not build arguments for function ` +
`"sneak.berlin/go/vaultik/internal/cli".newSnapshotCreateCommand.func1.1 ` +
`(/Users/user/dev/vaultik/internal/cli/snapshot.go:71): ` +
`failed to build *vaultik.Vaultik: ` +
`could not build arguments for function ` +
`"sneak.berlin/go/vaultik/internal/vaultik".New ` +
`(/Users/user/dev/vaultik/internal/vaultik/vaultik.go:59): ` +
`failed to build storage.Storer: ` +
`received non-nil error from function ` +
`"sneak.berlin/go/vaultik/internal/storage".NewStorer ` +
`(/Users/user/dev/vaultik/internal/storage/module.go:23): ` +
`creating base path: mkdir /Volumes/BACKUPS: permission denied`,
want: `creating base path: mkdir /Volumes/BACKUPS: permission denied`,
},
{
@@ -30,6 +43,9 @@ func TestCleanStartupError(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
//nolint:err113 // test constructs errors from table input
got := cleanStartupError(errors.New(tt.in)).Error()
if got != tt.want {
t.Errorf("got %q, want %q", got, tt.want)

View File

@@ -1,6 +1,7 @@
package cli
import (
"errors"
"fmt"
"os"
"os/exec"
@@ -12,6 +13,26 @@ import (
"gopkg.in/yaml.v3"
)
// configFileMode is the permission set for freshly written config files;
// configs may hold S3 credentials, so keep them owner-only.
const configFileMode = 0o600
// configSetArgs is the argument count of `config set <key> <value>`.
const configSetArgs = 2
// configDirMode is the permission set for created config directories;
// parent config dirs (e.g. ~/.config) are conventionally traversable.
const configDirMode = 0o755
var (
errConfigExists = errors.New("config file already exists")
errEmptyConfig = errors.New("empty config file")
errKeyNotFound = errors.New("key not found")
errNeedNumericIndex = errors.New("key is a list; use a numeric index")
errIndexOutOfRange = errors.New("index out of range")
errNotMapOrList = errors.New("key is not a map or list")
)
const defaultConfigTemplate = `# vaultik configuration
# Documentation: https://sneak.berlin/go/vaultik
@@ -232,24 +253,30 @@ The config is written to the path from --config, $VAULTIK_CONFIG, or
the platform default config directory (e.g. ~/Library/Application Support/
on macOS, ~/.config/ on Linux, /etc/vaultik/ as root).`,
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, args []string) error {
RunE: func(_ *cobra.Command, _ []string) error {
path := configPathForInit()
if _, err := os.Stat(path); err == nil {
return fmt.Errorf("config file already exists: %s", path)
_, err := os.Stat(path)
if err == nil {
return fmt.Errorf("%w: %s", errConfigExists, path)
}
dir := filepath.Dir(path)
if err := os.MkdirAll(dir, 0o755); err != nil {
err = os.MkdirAll(dir, configDirMode)
if err != nil {
return fmt.Errorf("creating config directory %s: %w", dir, err)
}
if err := os.WriteFile(path, []byte(defaultConfigTemplate), 0o600); err != nil {
err = os.WriteFile(path, []byte(defaultConfigTemplate), configFileMode)
if err != nil {
return fmt.Errorf("writing config file: %w", err)
}
fmt.Printf("Config written to %s\n", path)
fmt.Println("Edit it to set your age_recipients, snapshots, and storage_url.")
_, _ = fmt.Fprintf(os.Stdout, "Config written to %s\n", path)
_, _ = fmt.Fprintln(os.Stdout,
"Edit it to set your age_recipients, snapshots, and storage_url.")
return nil
},
}
@@ -261,7 +288,7 @@ func newConfigEditCommand() *cobra.Command {
Use: "edit",
Short: "Open the config file in $EDITOR",
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, args []string) error {
RunE: func(cmd *cobra.Command, _ []string) error {
path, err := ResolveConfigPath()
if err != nil {
return err
@@ -272,10 +299,12 @@ func newConfigEditCommand() *cobra.Command {
editor = "vi"
}
ed := exec.Command(editor, path)
//nolint:gosec // G204: launching the operator's own $EDITOR is the point
ed := exec.CommandContext(cmd.Context(), editor, path)
ed.Stdin = os.Stdin
ed.Stdout = os.Stdout
ed.Stderr = os.Stderr
return ed.Run()
},
}
@@ -287,7 +316,7 @@ func newConfigGetCommand() *cobra.Command {
Use: "get <key>",
Short: "Print a config value by dotted path (e.g. storage_url, compression_level)",
Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
RunE: func(_ *cobra.Command, args []string) error {
path, err := ResolveConfigPath()
if err != nil {
return err
@@ -304,7 +333,8 @@ func newConfigGetCommand() *cobra.Command {
}
if node.Kind == yaml.ScalarNode {
fmt.Println(node.Value)
_, _ = fmt.Fprintln(os.Stdout, node.Value)
return nil
}
@@ -312,7 +342,9 @@ func newConfigGetCommand() *cobra.Command {
if err != nil {
return fmt.Errorf("marshaling value: %w", err)
}
fmt.Print(string(out))
_, _ = fmt.Fprint(os.Stdout, string(out))
return nil
},
}
@@ -332,8 +364,8 @@ Examples:
vaultik config set storage_url "s3://bucket/prefix?endpoint=host&region=us-east-1"
vaultik config set compression_level 9
vaultik config set s3.bucket mybucket # legacy S3 fields still supported`,
Args: cobra.ExactArgs(2),
RunE: func(cmd *cobra.Command, args []string) error {
Args: cobra.ExactArgs(configSetArgs),
RunE: func(_ *cobra.Command, args []string) error {
path, err := ResolveConfigPath()
if err != nil {
return err
@@ -344,7 +376,8 @@ Examples:
return err
}
if err := yamlPathSet(root, strings.Split(args[0], "."), args[1]); err != nil {
err = yamlPathSet(root, strings.Split(args[0], "."), args[1])
if err != nil {
return err
}
@@ -353,16 +386,20 @@ Examples:
return fmt.Errorf("marshaling config: %w", err)
}
mode := os.FileMode(0o600)
if info, err := os.Stat(path); err == nil {
mode := os.FileMode(configFileMode)
info, statErr := os.Stat(path)
if statErr == nil {
mode = info.Mode().Perm()
}
if err := os.WriteFile(path, out, mode); err != nil {
err = os.WriteFile(path, out, mode)
if err != nil {
return fmt.Errorf("writing config file: %w", err)
}
fmt.Printf("%s = %s\n", args[0], args[1])
_, _ = fmt.Fprintf(os.Stdout, "%s = %s\n", args[0], args[1])
return nil
},
}
@@ -371,13 +408,15 @@ Examples:
// loadYAMLFile parses a YAML file into a yaml.Node document tree,
// which preserves comments and ordering for round-tripping.
func loadYAMLFile(path string) (*yaml.Node, error) {
data, err := os.ReadFile(path)
data, err := os.ReadFile(path) //nolint:gosec // G304: config path is operator-supplied
if err != nil {
return nil, fmt.Errorf("reading config file: %w", err)
}
var root yaml.Node
if err := yaml.Unmarshal(data, &root); err != nil {
err = yaml.Unmarshal(data, &root)
if err != nil {
return nil, fmt.Errorf("parsing config file: %w", err)
}
@@ -399,8 +438,9 @@ func yamlPathGet(root *yaml.Node, keys []string) (*yaml.Node, error) {
node := root
if node.Kind == yaml.DocumentNode {
if len(node.Content) == 0 {
return nil, fmt.Errorf("empty config file")
return nil, errEmptyConfig
}
node = node.Content[0]
}
@@ -408,27 +448,40 @@ func yamlPathGet(root *yaml.Node, keys []string) (*yaml.Node, error) {
switch node.Kind {
case yaml.MappingNode:
found := false
for j := 0; j+1 < len(node.Content); j += 2 {
if node.Content[j].Value == key {
node = node.Content[j+1]
found = true
break
}
}
if !found {
return nil, fmt.Errorf("key not found: %s", strings.Join(keys[:i+1], "."))
return nil, fmt.Errorf("%w: %s",
errKeyNotFound, strings.Join(keys[:i+1], "."))
}
case yaml.SequenceNode:
idx, err := strconv.Atoi(key)
if err != nil {
return nil, fmt.Errorf("key %q is a list; use a numeric index", strings.Join(keys[:i], "."))
return nil, fmt.Errorf("%w: %s",
errNeedNumericIndex, strings.Join(keys[:i], "."))
}
if idx < 0 || idx >= len(node.Content) {
return nil, fmt.Errorf("index %d out of range for %s (len %d)", idx, strings.Join(keys[:i], "."), len(node.Content))
return nil, fmt.Errorf("%w: index %d for %s (len %d)",
errIndexOutOfRange, idx, strings.Join(keys[:i], "."),
len(node.Content))
}
node = node.Content[idx]
case yaml.DocumentNode, yaml.ScalarNode, yaml.AliasNode:
return nil, fmt.Errorf("%w: %s",
errNotMapOrList, strings.Join(keys[:i], "."))
default:
return nil, fmt.Errorf("key %q is not a map or list", strings.Join(keys[:i], "."))
return nil, fmt.Errorf("%w: %s",
errNotMapOrList, strings.Join(keys[:i], "."))
}
}
@@ -445,6 +498,7 @@ func yamlPathSet(root *yaml.Node, keys []string, value string) error {
if len(node.Content) == 0 {
node.Content = []*yaml.Node{{Kind: yaml.MappingNode}}
}
node = node.Content[0]
}
@@ -453,52 +507,86 @@ func yamlPathSet(root *yaml.Node, keys []string, value string) error {
switch node.Kind {
case yaml.MappingNode:
node = yamlSetInMapping(node, key, value, last)
case yaml.SequenceNode:
next, err := yamlSetInSequence(node, keys, i, value, last)
if err != nil {
return err
}
node = next
case yaml.DocumentNode, yaml.ScalarNode, yaml.AliasNode:
return fmt.Errorf("%w: %s",
errNotMapOrList, strings.Join(keys[:i], "."))
default:
return fmt.Errorf("%w: %s",
errNotMapOrList, strings.Join(keys[:i], "."))
}
}
return nil
}
// yamlSetInMapping resolves (creating if needed) the value node for key
// within a mapping node, setting it to value when it is the final path
// element, and returns the node to descend into.
func yamlSetInMapping(node *yaml.Node, key, value string, last bool) *yaml.Node {
var valueNode *yaml.Node
for j := 0; j+1 < len(node.Content); j += 2 {
if node.Content[j].Value == key {
valueNode = node.Content[j+1]
break
}
}
if valueNode == nil {
keyNode := &yaml.Node{Kind: yaml.ScalarNode, Value: key}
valueNode = &yaml.Node{Kind: yaml.MappingNode}
if last {
valueNode = &yaml.Node{Kind: yaml.ScalarNode, Value: value}
}
node.Content = append(node.Content, keyNode, valueNode)
} else if last {
setScalar(valueNode, value)
}
node = valueNode
return valueNode
}
case yaml.SequenceNode:
idx, err := strconv.Atoi(key)
// yamlSetInSequence indexes (or appends to) a sequence node using the
// numeric path element keys[i], setting the element to value when it is
// the final path element, and returns the node to descend into.
func yamlSetInSequence(
node *yaml.Node, keys []string, i int, value string, last bool,
) (*yaml.Node, error) {
idx, err := strconv.Atoi(keys[i])
if err != nil {
return fmt.Errorf("key %q is a list; use a numeric index", strings.Join(keys[:i], "."))
return nil, fmt.Errorf("%w: %s",
errNeedNumericIndex, strings.Join(keys[:i], "."))
}
if idx < 0 || idx > len(node.Content) {
return fmt.Errorf("index %d out of range for %s (len %d)", idx, strings.Join(keys[:i], "."), len(node.Content))
return nil, fmt.Errorf("%w: index %d for %s (len %d)",
errIndexOutOfRange, idx, strings.Join(keys[:i], "."),
len(node.Content))
}
if idx == len(node.Content) {
newNode := &yaml.Node{Kind: yaml.MappingNode}
if last {
newNode = &yaml.Node{Kind: yaml.ScalarNode, Value: value}
}
node.Content = append(node.Content, newNode)
} else if last {
setScalar(node.Content[idx], value)
}
node = node.Content[idx]
default:
return fmt.Errorf("key %q is not a map or list", strings.Join(keys[:i], "."))
}
}
return nil
return node.Content[idx], nil
}
// setScalar overwrites a node in place with a plain scalar value.
@@ -516,8 +604,10 @@ func configPathForInit() string {
if rootFlags.ConfigPath != "" {
return rootFlags.ConfigPath
}
if envPath := os.Getenv("VAULTIK_CONFIG"); envPath != "" {
return envPath
}
return DefaultConfigPath()
}

View File

@@ -1,4 +1,4 @@
package cli
package cli //nolint:testpackage // exercises unexported yamlPathGet/yamlPathSet
import (
"strings"
@@ -11,8 +11,12 @@ import (
// TestDefaultConfigTemplateParses ensures the init template is valid YAML
// that unmarshals into the Config struct with the expected snapshots.
func TestDefaultConfigTemplateParses(t *testing.T) {
t.Parallel()
var cfg config.Config
if err := yaml.Unmarshal([]byte(defaultConfigTemplate), &cfg); err != nil {
err := yaml.Unmarshal([]byte(defaultConfigTemplate), &cfg)
if err != nil {
t.Fatalf("default config template is not valid YAML: %v", err)
}
@@ -24,9 +28,11 @@ func TestDefaultConfigTemplateParses(t *testing.T) {
if !ok {
t.Fatal("expected 'home' snapshot in default config")
}
if len(home.Paths) == 0 {
t.Error("home snapshot should have at least one path")
}
if len(home.Exclude) == 0 {
t.Error("home snapshot should have exclude patterns")
}
@@ -35,9 +41,11 @@ func TestDefaultConfigTemplateParses(t *testing.T) {
if !ok {
t.Fatal("expected 'apps' snapshot in default config")
}
if len(apps.Paths) != 1 || apps.Paths[0] != "/Applications" {
t.Errorf("apps snapshot should back up /Applications, got %v", apps.Paths)
}
if len(apps.Exclude) == 0 {
t.Error("apps snapshot should have exclude patterns")
}
@@ -58,14 +66,20 @@ snapshots:
func parseTestYAML(t *testing.T) *yaml.Node {
t.Helper()
var root yaml.Node
if err := yaml.Unmarshal([]byte(testYAML), &root); err != nil {
err := yaml.Unmarshal([]byte(testYAML), &root)
if err != nil {
t.Fatalf("parsing test yaml: %v", err)
}
return &root
}
func TestYAMLPathGet(t *testing.T) {
t.Parallel()
root := parseTestYAML(t)
tests := []struct {
@@ -86,16 +100,21 @@ func TestYAMLPathGet(t *testing.T) {
for _, tt := range tests {
t.Run(tt.path, func(t *testing.T) {
t.Parallel()
node, err := yamlPathGet(root, splitPath(tt.path))
if tt.err {
if err == nil {
t.Fatalf("expected error for %q", tt.path)
}
return
}
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if node.Value != tt.want {
t.Errorf("get %q = %q, want %q", tt.path, node.Value, tt.want)
}
@@ -104,29 +123,40 @@ func TestYAMLPathGet(t *testing.T) {
}
func TestYAMLPathSet(t *testing.T) {
t.Parallel()
root := parseTestYAML(t)
// Overwrite existing nested value
if err := yamlPathSet(root, splitPath("s3.bucket"), "newbucket"); err != nil {
err := yamlPathSet(root, splitPath("s3.bucket"), "newbucket")
if err != nil {
t.Fatalf("set s3.bucket: %v", err)
}
// Create new nested key with intermediate map
if err := yamlPathSet(root, splitPath("s3.endpoint"), "s3.example.com"); err != nil {
err = yamlPathSet(root, splitPath("s3.endpoint"), "s3.example.com")
if err != nil {
t.Fatalf("set s3.endpoint: %v", err)
}
if err := yamlPathSet(root, splitPath("newmap.newkey"), "val"); err != nil {
err = yamlPathSet(root, splitPath("newmap.newkey"), "val")
if err != nil {
t.Fatalf("set newmap.newkey: %v", err)
}
// Overwrite a sequence element and append a new one
if err := yamlPathSet(root, splitPath("age_recipients.0"), "age1bbb"); err != nil {
err = yamlPathSet(root, splitPath("age_recipients.0"), "age1bbb")
if err != nil {
t.Fatalf("set age_recipients.0: %v", err)
}
if err := yamlPathSet(root, splitPath("age_recipients.1"), "age1ccc"); err != nil {
err = yamlPathSet(root, splitPath("age_recipients.1"), "age1ccc")
if err != nil {
t.Fatalf("append age_recipients.1: %v", err)
}
if err := yamlPathSet(root, splitPath("age_recipients.5"), "age1ddd"); err == nil {
err = yamlPathSet(root, splitPath("age_recipients.5"), "age1ddd")
if err == nil {
t.Error("expected out-of-range append to fail")
}
@@ -135,9 +165,14 @@ func TestYAMLPathSet(t *testing.T) {
if err != nil {
t.Fatalf("marshal: %v", err)
}
text := string(out)
for _, want := range []string{"newbucket", "s3.example.com", "newkey: val", "# top comment", "# inline comment", "age1bbb", "age1ccc"} {
wants := []string{
"newbucket", "s3.example.com", "newkey: val",
"# top comment", "# inline comment", "age1bbb", "age1ccc",
}
for _, want := range wants {
if !contains(text, want) {
t.Errorf("round-tripped YAML missing %q:\n%s", want, text)
}
@@ -147,6 +182,7 @@ func TestYAMLPathSet(t *testing.T) {
if err != nil {
t.Fatalf("get after set: %v", err)
}
if got.Value != "newbucket" {
t.Errorf("s3.bucket = %q after set, want newbucket", got.Value)
}

View File

@@ -48,7 +48,7 @@ storage destination on that run.
Use --force to skip the confirmation prompt.`,
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, args []string) error {
RunE: func(_ *cobra.Command, _ []string) error {
// Resolve config path
configPath, err := ResolveConfigPath()
if err != nil {
@@ -64,24 +64,33 @@ Use --force to skip the confirmation prompt.`,
dbPath := cfg.IndexPath
// Check if database exists
if _, err := os.Stat(dbPath); os.IsNotExist(err) {
fmt.Printf("Database does not exist: %s\n", dbPath)
_, err = os.Stat(dbPath)
if os.IsNotExist(err) {
_, _ = fmt.Fprintf(os.Stdout, "Database does not exist: %s\n", dbPath)
return nil
}
// Confirm unless --force
if !force {
fmt.Printf("This will delete the local state database at:\n %s\n\n", dbPath)
fmt.Print("Are you sure? Type 'yes' to confirm: ")
_, _ = fmt.Fprintf(os.Stdout,
"This will delete the local state database at:\n %s\n\n", dbPath)
_, _ = fmt.Fprint(os.Stdout, "Are you sure? Type 'yes' to confirm: ")
var confirm string
if _, err := fmt.Scanln(&confirm); err != nil || confirm != "yes" {
fmt.Println("Aborted.")
_, err = fmt.Scanln(&confirm)
if err != nil || confirm != "yes" {
_, _ = fmt.Fprintln(os.Stdout, "Aborted.")
//nolint:nilerr // a failed/aborted confirmation is a clean abort
return nil
}
}
// Delete the database file
if err := os.Remove(dbPath); err != nil {
err = os.Remove(dbPath)
if err != nil {
return fmt.Errorf("failed to delete database: %w", err)
}
@@ -93,10 +102,11 @@ Use --force to skip the confirmation prompt.`,
rootFlags := GetRootFlags()
if !rootFlags.Quiet {
fmt.Printf("Database deleted: %s\n", dbPath)
_, _ = fmt.Fprintf(os.Stdout, "Database deleted: %s\n", dbPath)
}
log.Info("Local state database deleted", "path", dbPath)
return nil
},
}

View File

@@ -1,6 +1,7 @@
package cli
import (
"errors"
"fmt"
"regexp"
"strconv"
@@ -8,6 +9,21 @@ import (
"time"
)
// Approximate lengths of the extended calendar units accepted by
// parseDuration.
const (
durationDay = 24 * time.Hour
durationWeek = 7 * durationDay
durationMonth = 30 * durationDay
durationYear = 365 * durationDay
)
var (
errNegativeDuration = errors.New("negative durations are not supported")
errInvalidDuration = errors.New("invalid duration format")
errUnknownTimeUnit = errors.New("unknown time unit")
)
// parseDuration parses duration strings. Supports standard Go duration format
// (e.g., "3h30m", "1h45m30s") as well as extended units:
// - d: days (e.g., "30d", "7d")
@@ -18,14 +34,15 @@ import (
// Can combine units: "1y6mo", "2w3d", "1d12h30m"
func parseDuration(s string) (time.Duration, error) {
// First try standard Go duration parsing
if d, err := time.ParseDuration(s); err == nil {
d, err := time.ParseDuration(s)
if err == nil {
return d, nil
}
// Extended duration parsing
// Check for negative values
if strings.HasPrefix(strings.TrimSpace(s), "-") {
return 0, fmt.Errorf("negative durations are not supported")
return 0, errNegativeDuration
}
// Pattern matches: number + unit, repeated
@@ -33,7 +50,7 @@ func parseDuration(s string) (time.Duration, error) {
matches := re.FindAllStringSubmatch(s, -1)
if len(matches) == 0 {
return 0, fmt.Errorf("invalid duration format: %q", s)
return 0, fmt.Errorf("%w: %q", errInvalidDuration, s)
}
var total time.Duration
@@ -47,44 +64,9 @@ func parseDuration(s string) (time.Duration, error) {
return 0, fmt.Errorf("invalid number %q: %w", valueStr, err)
}
var d time.Duration
switch unit {
// Standard time units
case "ns", "nanosecond", "nanoseconds":
d = time.Duration(value)
case "us", "µs", "microsecond", "microseconds":
d = time.Duration(value * float64(time.Microsecond))
case "ms", "millisecond", "milliseconds":
d = time.Duration(value * float64(time.Millisecond))
case "s", "sec", "second", "seconds":
d = time.Duration(value * float64(time.Second))
case "m", "min", "minute", "minutes":
d = time.Duration(value * float64(time.Minute))
case "h", "hr", "hour", "hours":
d = time.Duration(value * float64(time.Hour))
// Extended units
case "d", "day", "days":
d = time.Duration(value * float64(24*time.Hour))
case "w", "week", "weeks":
d = time.Duration(value * float64(7*24*time.Hour))
case "mo", "month", "months":
// Using 30 days as approximation
d = time.Duration(value * float64(30*24*time.Hour))
case "y", "year", "years":
// Using 365 days as approximation
d = time.Duration(value * float64(365*24*time.Hour))
default:
// Try parsing as standard Go duration unit
testStr := fmt.Sprintf("1%s", unit)
if _, err := time.ParseDuration(testStr); err == nil {
// It's a valid Go duration unit, parse the full value
fullStr := fmt.Sprintf("%g%s", value, unit)
if d, err = time.ParseDuration(fullStr); err != nil {
return 0, fmt.Errorf("invalid duration %q: %w", fullStr, err)
}
} else {
return 0, fmt.Errorf("unknown time unit %q", unit)
}
d, err := durationForUnit(value, unit)
if err != nil {
return 0, err
}
total += d
@@ -92,3 +74,53 @@ func parseDuration(s string) (time.Duration, error) {
return total, nil
}
// durationForUnit converts a value with a (case-normalized) unit suffix
// into a time.Duration, accepting Go's standard units plus the extended
// calendar units.
func durationForUnit(value float64, unit string) (time.Duration, error) {
switch unit {
// Standard time units
case "ns", "nanosecond", "nanoseconds":
return time.Duration(value), nil
case "us", "µs", "microsecond", "microseconds":
return time.Duration(value * float64(time.Microsecond)), nil
case "ms", "millisecond", "milliseconds":
return time.Duration(value * float64(time.Millisecond)), nil
case "s", "sec", "second", "seconds":
return time.Duration(value * float64(time.Second)), nil
case "m", "min", "minute", "minutes":
return time.Duration(value * float64(time.Minute)), nil
case "h", "hr", "hour", "hours":
return time.Duration(value * float64(time.Hour)), nil
// Extended units
case "d", "day", "days":
return time.Duration(value * float64(durationDay)), nil
case "w", "week", "weeks":
return time.Duration(value * float64(durationWeek)), nil
case "mo", "month", "months":
// Using 30 days as approximation
return time.Duration(value * float64(durationMonth)), nil
case "y", "year", "years":
// Using 365 days as approximation
return time.Duration(value * float64(durationYear)), nil
default:
// Try parsing as standard Go duration unit
testStr := "1" + unit
_, err := time.ParseDuration(testStr)
if err != nil {
return 0, fmt.Errorf("%w: %q", errUnknownTimeUnit, unit)
}
// It's a valid Go duration unit, parse the full value
fullStr := fmt.Sprintf("%g%s", value, unit)
d, err := time.ParseDuration(fullStr)
if err != nil {
return 0, fmt.Errorf("invalid duration %q: %w", fullStr, err)
}
return d, nil
}
}

View File

@@ -1,20 +1,47 @@
package cli
package cli //nolint:testpackage // needs access to unexported parseDuration
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestParseDuration(t *testing.T) {
tests := []struct {
type parseDurationCase struct {
name string
input string
expected time.Duration
wantErr bool
}{
// Standard Go durations
}
// runParseDurationCases executes a table of parseDuration cases as
// parallel subtests.
func runParseDurationCases(t *testing.T, tests []parseDurationCase) {
t.Helper()
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
got, err := parseDuration(tt.input)
if tt.wantErr {
require.Error(t, err, "expected error for input %q", tt.input)
return
}
require.NoError(t, err, "unexpected error for input %q", tt.input)
assert.Equal(t, tt.expected, got, "duration mismatch for input %q", tt.input)
})
}
}
func TestParseDurationStandard(t *testing.T) {
t.Parallel()
runParseDurationCases(t, []parseDurationCase{
{
name: "standard seconds",
input: "30s",
@@ -45,6 +72,13 @@ func TestParseDuration(t *testing.T) {
input: "1s500ms",
expected: 1*time.Second + 500*time.Millisecond,
},
})
}
func TestParseDurationExtendedUnits(t *testing.T) {
t.Parallel()
runParseDurationCases(t, []parseDurationCase{
// Extended units - days
{
name: "single day",
@@ -114,6 +148,13 @@ func TestParseDuration(t *testing.T) {
input: "1year",
expected: 365 * 24 * time.Hour,
},
})
}
func TestParseDurationCombinedAndErrors(t *testing.T) {
t.Parallel()
runParseDurationCases(t, []parseDurationCase{
// Combined extended units
{
name: "weeks and days",
@@ -133,7 +174,9 @@ func TestParseDuration(t *testing.T) {
{
name: "complex combination",
input: "1y2mo3w4d5h6m7s",
expected: 365*24*time.Hour + 2*30*24*time.Hour + 3*7*24*time.Hour + 4*24*time.Hour + 5*time.Hour + 6*time.Minute + 7*time.Second,
expected: 365*24*time.Hour + 2*30*24*time.Hour +
3*7*24*time.Hour + 4*24*time.Hour +
5*time.Hour + 6*time.Minute + 7*time.Second,
},
{
name: "with spaces",
@@ -177,24 +220,12 @@ func TestParseDuration(t *testing.T) {
input: "-5d",
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := parseDuration(tt.input)
if tt.wantErr {
assert.Error(t, err, "expected error for input %q", tt.input)
return
}
assert.NoError(t, err, "unexpected error for input %q", tt.input)
assert.Equal(t, tt.expected, got, "duration mismatch for input %q", tt.input)
})
}
}
func TestParseDurationSpecialCases(t *testing.T) {
t.Parallel()
// Test that standard Go durations work exactly as expected
standardDurations := []string{
"300ms",
@@ -208,15 +239,17 @@ func TestParseDurationSpecialCases(t *testing.T) {
for _, d := range standardDurations {
expected, err := time.ParseDuration(d)
assert.NoError(t, err)
require.NoError(t, err)
got, err := parseDuration(d)
assert.NoError(t, err)
require.NoError(t, err)
assert.Equal(t, expected, got, "standard duration %q should parse identically", d)
}
}
func TestParseDurationRealWorldExamples(t *testing.T) {
t.Parallel()
// Test real-world snapshot purge scenarios
tests := []struct {
description string
@@ -252,12 +285,15 @@ func TestParseDurationRealWorldExamples(t *testing.T) {
for _, tt := range tests {
t.Run(tt.description, func(t *testing.T) {
t.Parallel()
got, err := parseDuration(tt.input)
assert.NoError(t, err)
require.NoError(t, err)
assert.Equal(t, tt.olderThan, got)
// Verify the duration makes sense for snapshot purging
assert.Greater(t, got, time.Hour, "snapshot purge duration should be at least an hour")
assert.Greater(t, got, time.Hour,
"snapshot purge duration should be at least an hour")
})
}
}

View File

@@ -9,34 +9,40 @@ import (
"sneak.berlin/go/vaultik/internal/ui"
)
// CLIEntry is the main entry point for the CLI application.
// shortCommitLen is the number of git commit hash characters shown in
// the startup banner.
const shortCommitLen = 12
// Entry is the main entry point for the CLI application.
// It prints the startup banner (unless a quiet flag is present in os.Args),
// executes the root cobra command, and routes any returned error through
// the ui.Writer so the user sees a properly formatted "🛑 ERROR:" line.
func CLIEntry() {
func Entry() {
if !bannerSuppressedInArgs(os.Args[1:]) {
short := globals.Commit
if len(short) > 12 {
short = short[:12]
if len(short) > shortCommitLen {
short = short[:shortCommitLen]
}
writeStartupBanner(ui.New(os.Stdout), time.Now().UTC(), short)
}
rootCmd := NewRootCommand()
rootCmd.SilenceErrors = true
if err := rootCmd.Execute(); err != nil {
ReportError("%s", err.Error())
err := rootCmd.Execute()
if err != nil {
ReportErrorf("%s", err.Error())
os.Exit(1)
}
}
// ReportError emits a user-facing error to stderr in the standard
// ReportErrorf emits a user-facing error to stderr in the standard
// 🛑 ERROR: format. Use it from goroutine error paths (where returning
// an error to cobra isn't an option) and anywhere else a CLI command
// must surface a failure outside the normal RunE return path.
func ReportError(format string, args ...any) {
ui.New(os.Stderr).Error(format, args...)
func ReportErrorf(format string, args ...any) {
ui.New(os.Stderr).Errorf(format, args...)
}
// bannerSuppressedInArgs reports whether any of args is a flag that
@@ -48,10 +54,12 @@ func bannerSuppressedInArgs(args []string) bool {
if a == "--" {
return false
}
switch a {
case "--quiet", "-q", "--cron":
return true
}
if strings.HasPrefix(a, "--quiet=") || strings.HasPrefix(a, "--cron=") {
return true
}
@@ -64,5 +72,6 @@ func bannerSuppressedInArgs(args []string) bool {
}
}
}
return false
}

View File

@@ -1,14 +1,18 @@
package cli
package cli_test
import (
"testing"
"sneak.berlin/go/vaultik/internal/cli"
)
// TestCLIEntry ensures the CLI can be imported and basic initialization works
func TestCLIEntry(t *testing.T) {
t.Parallel()
// This test primarily serves as a compilation test
// to ensure all imports resolve correctly
cmd := NewRootCommand()
cmd := cli.NewRootCommand()
if cmd == nil {
t.Fatal("NewRootCommand() returned nil")
}
@@ -18,15 +22,20 @@ func TestCLIEntry(t *testing.T) {
}
// Verify all subcommands are registered
expectedCommands := []string{"config", "snapshot", "prune", "info", "version", "remote", "database"}
expectedCommands := []string{
"config", "snapshot", "prune", "info", "version", "remote", "database",
}
for _, expected := range expectedCommands {
found := false
for _, cmd := range cmd.Commands() {
if cmd.Use == expected || cmd.Name() == expected {
found = true
break
}
}
if !found {
t.Errorf("Expected command '%s' not found", expected)
}
@@ -38,15 +47,20 @@ func TestCLIEntry(t *testing.T) {
t.Errorf("Failed to find snapshot command: %v", err)
} else {
// Check snapshot subcommands
expectedSubCommands := []string{"create", "list", "purge", "verify", "remove", "restore"}
expectedSubCommands := []string{
"create", "list", "purge", "verify", "remove", "restore",
}
for _, expected := range expectedSubCommands {
found := false
for _, subcmd := range snapshotCmd.Commands() {
if subcmd.Use == expected || subcmd.Name() == expected {
found = true
break
}
}
if !found {
t.Errorf("Expected snapshot subcommand '%s' not found", expected)
}

View File

@@ -2,6 +2,7 @@ package cli
import (
"context"
"errors"
"os"
"github.com/spf13/cobra"
@@ -22,7 +23,7 @@ func NewInfoCommand() *cobra.Command {
- Encryption configuration (recipients)
- Local database statistics`,
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, args []string) error {
RunE: func(cmd *cobra.Command, _ []string) error {
// Use unified config resolution
configPath, err := ResolveConfigPath()
if err != nil {
@@ -31,9 +32,10 @@ func NewInfoCommand() *cobra.Command {
// Use the app framework
rootFlags := GetRootFlags()
return RunWithApp(cmd.Context(), AppOptions{
ConfigPath: configPath,
LogOptions: log.LogOptions{
LogOptions: log.Options{
Verbose: rootFlags.Verbose,
Debug: rootFlags.Debug,
Quiet: rootFlags.Quiet,
@@ -42,23 +44,28 @@ func NewInfoCommand() *cobra.Command {
Invokes: []fx.Option{
fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) {
lc.Append(fx.Hook{
OnStart: func(ctx context.Context) error {
OnStart: func(_ context.Context) error {
go func() {
if err := v.ShowInfo(); err != nil {
if err != context.Canceled {
err := v.ShowInfo()
if err != nil {
if !errors.Is(err, context.Canceled) {
log.Error("Failed to show info", "error", err)
ReportError("Failed to show info: %v", err)
ReportErrorf("Failed to show info: %v", err)
os.Exit(1)
}
}
if err := v.Shutdowner.Shutdown(); err != nil {
err = v.Shutdowner.Shutdown()
if err != nil {
log.Error("Failed to shutdown", "error", err)
}
}()
return nil
},
OnStop: func(ctx context.Context) error {
OnStop: func(_ context.Context) error {
v.Cancel()
return nil
},
})

View File

@@ -2,6 +2,7 @@ package cli
import (
"context"
"errors"
"os"
"github.com/spf13/cobra"
@@ -30,7 +31,7 @@ Snapshot create --prune and snapshot remove run the same cleanup
automatically; this command is the manual entry point for the same
work (e.g. after a crashed backup or to reclaim storage).`,
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, args []string) error {
RunE: func(cmd *cobra.Command, _ []string) error {
// Use unified config resolution
configPath, err := ResolveConfigPath()
if err != nil {
@@ -39,9 +40,10 @@ work (e.g. after a crashed backup or to reclaim storage).`,
// Use the app framework like other commands
rootFlags := GetRootFlags()
return RunWithApp(cmd.Context(), AppOptions{
ConfigPath: configPath,
LogOptions: log.LogOptions{
LogOptions: log.Options{
Verbose: rootFlags.Verbose,
Debug: rootFlags.Debug,
Quiet: rootFlags.Quiet || opts.JSON,
@@ -50,30 +52,35 @@ work (e.g. after a crashed backup or to reclaim storage).`,
Invokes: []fx.Option{
fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) {
lc.Append(fx.Hook{
OnStart: func(ctx context.Context) error {
OnStart: func(_ context.Context) error {
// Start the prune operation in a goroutine
go func() {
// Run the prune operation
if err := v.Prune(opts); err != nil {
if err != context.Canceled {
err := v.Prune(opts)
if err != nil {
if !errors.Is(err, context.Canceled) {
if !opts.JSON {
log.Error("Prune operation failed", "error", err)
ReportError("Prune failed: %v", err)
ReportErrorf("Prune failed: %v", err)
}
os.Exit(1)
}
}
// Shutdown the app when prune completes
if err := v.Shutdowner.Shutdown(); err != nil {
err = v.Shutdowner.Shutdown()
if err != nil {
log.Error("Failed to shutdown", "error", err)
}
}()
return nil
},
OnStop: func(ctx context.Context) error {
OnStop: func(_ context.Context) error {
log.Debug("Stopping prune operation")
v.Cancel()
return nil
},
})

View File

@@ -2,7 +2,7 @@ package cli
import (
"context"
"fmt"
"errors"
"os"
"github.com/spf13/cobra"
@@ -11,6 +11,10 @@ import (
"sneak.berlin/go/vaultik/internal/vaultik"
)
// errNukeNeedsForce guards the destructive 'remote nuke' subcommand.
var errNukeNeedsForce = errors.New(
"remote nuke requires --force (this deletes ALL remote snapshots and blobs)")
// NewRemoteCommand creates the remote command and subcommands
func NewRemoteCommand() *cobra.Command {
cmd := &cobra.Command{
@@ -39,55 +43,20 @@ empty and the next backup starts from scratch.
This is destructive and irreversible. Requires --force.`,
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, args []string) error {
RunE: func(cmd *cobra.Command, _ []string) error {
if !force {
return fmt.Errorf("remote nuke requires --force (this deletes ALL remote snapshots and blobs)")
return errNukeNeedsForce
}
configPath, err := ResolveConfigPath()
if err != nil {
return err
}
rootFlags := GetRootFlags()
return RunWithApp(cmd.Context(), AppOptions{
ConfigPath: configPath,
LogOptions: log.LogOptions{
Verbose: rootFlags.Verbose,
Debug: rootFlags.Debug,
Quiet: rootFlags.Quiet,
},
Modules: []fx.Option{},
Invokes: []fx.Option{
fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) {
lc.Append(fx.Hook{
OnStart: func(ctx context.Context) error {
go func() {
if err := v.NukeRemote(true); err != nil {
if err != context.Canceled {
log.Error("Remote nuke failed", "error", err)
ReportError("Remote nuke failed: %v", err)
os.Exit(1)
}
}
if err := v.Shutdowner.Shutdown(); err != nil {
log.Error("Failed to shutdown", "error", err)
}
}()
return nil
},
OnStop: func(ctx context.Context) error {
v.Cancel()
return nil
},
})
}),
},
return runVaultikApp(cmd, false, false, "Remote nuke failed",
func(v *vaultik.Vaultik) error {
return v.NukeRemote(true)
})
},
}
cmd.Flags().BoolVar(&force, "force", false, "Required: confirm destruction of ALL remote data")
cmd.Flags().BoolVar(&force, "force", false,
"Required: confirm destruction of ALL remote data")
return cmd
}
@@ -105,7 +74,7 @@ func newRemoteInfoCommand() *cobra.Command {
- Count and size of referenced blobs (from all manifests)
- Count and size of orphaned blobs (not referenced by any manifest)`,
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, args []string) error {
RunE: func(cmd *cobra.Command, _ []string) error {
// Use unified config resolution
configPath, err := ResolveConfigPath()
if err != nil {
@@ -113,9 +82,10 @@ func newRemoteInfoCommand() *cobra.Command {
}
rootFlags := GetRootFlags()
return RunWithApp(cmd.Context(), AppOptions{
ConfigPath: configPath,
LogOptions: log.LogOptions{
LogOptions: log.Options{
Verbose: rootFlags.Verbose,
Debug: rootFlags.Debug,
Quiet: rootFlags.Quiet || jsonOutput,
@@ -124,25 +94,31 @@ func newRemoteInfoCommand() *cobra.Command {
Invokes: []fx.Option{
fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) {
lc.Append(fx.Hook{
OnStart: func(ctx context.Context) error {
OnStart: func(_ context.Context) error {
go func() {
if err := v.RemoteInfo(jsonOutput); err != nil {
if err != context.Canceled {
err := v.RemoteInfo(jsonOutput)
if err != nil {
if !errors.Is(err, context.Canceled) {
if !jsonOutput {
log.Error("Failed to get remote info", "error", err)
ReportError("Failed to get remote info: %v", err)
ReportErrorf("Failed to get remote info: %v", err)
}
os.Exit(1)
}
}
if err := v.Shutdowner.Shutdown(); err != nil {
err = v.Shutdowner.Shutdown()
if err != nil {
log.Error("Failed to shutdown", "error", err)
}
}()
return nil
},
OnStop: func(ctx context.Context) error {
OnStop: func(_ context.Context) error {
v.Cancel()
return nil
},
})

View File

@@ -1,6 +1,7 @@
package cli
import (
"errors"
"fmt"
"os"
"path/filepath"
@@ -10,6 +11,9 @@ import (
"github.com/spf13/cobra"
)
// errConfigNotFound is wrapped by all config-resolution failures.
var errConfigNotFound = errors.New("config file not found")
// RootFlags holds global flags that apply to all commands.
// These flags are defined on the root command and inherited by all subcommands.
type RootFlags struct {
@@ -20,6 +24,7 @@ type RootFlags struct {
SkipErrors bool
}
//nolint:gochecknoglobals // cobra persistent flags bind to package state
var rootFlags RootFlags
// NewRootCommand creates the root cobra command for the vaultik CLI.
@@ -34,20 +39,26 @@ public keys and uploads to S3-compatible storage. No private keys are needed
on the source system.`,
SilenceUsage: true,
// Bare 'vaultik' (no subcommand): print help. The banner is
// printed once at process startup by CLIEntry, before cobra
// printed once at process startup by Entry, before cobra
// parses arguments, so it appears even when cobra rejects
// args (e.g. "requires at least 2 arg(s)") and on --help.
Run: func(cmd *cobra.Command, args []string) {
Run: func(cmd *cobra.Command, _ []string) {
_ = cmd.Help()
},
}
// Add global flags
cmd.PersistentFlags().StringVar(&rootFlags.ConfigPath, "config", "", "Path to config file (default: $VAULTIK_CONFIG or platform config dir)")
cmd.PersistentFlags().BoolVarP(&rootFlags.Verbose, "verbose", "v", false, "Enable verbose output")
cmd.PersistentFlags().BoolVar(&rootFlags.Debug, "debug", false, "Enable debug output")
cmd.PersistentFlags().BoolVarP(&rootFlags.Quiet, "quiet", "q", false, "Suppress non-error output")
cmd.PersistentFlags().BoolVar(&rootFlags.SkipErrors, "skip-errors", false, "Continue past per-file errors instead of aborting (applies to snapshot create and restore)")
cmd.PersistentFlags().StringVar(&rootFlags.ConfigPath, "config", "",
"Path to config file (default: $VAULTIK_CONFIG or platform config dir)")
cmd.PersistentFlags().BoolVarP(&rootFlags.Verbose, "verbose", "v", false,
"Enable verbose output")
cmd.PersistentFlags().BoolVar(&rootFlags.Debug, "debug", false,
"Enable debug output")
cmd.PersistentFlags().BoolVarP(&rootFlags.Quiet, "quiet", "q", false,
"Suppress non-error output")
cmd.PersistentFlags().BoolVar(&rootFlags.SkipErrors, "skip-errors", false,
"Continue past per-file errors instead of aborting "+
"(applies to snapshot create and restore)")
// Add subcommands
cmd.AddCommand(
@@ -70,31 +81,45 @@ func GetRootFlags() RootFlags {
}
// ResolveConfigPath resolves the config file path from flags, environment, or default.
// Search order: --config flag, VAULTIK_CONFIG env, XDG config dir, /etc/vaultik/config.yml.
// Search order: --config flag, VAULTIK_CONFIG env, XDG config dir,
// /etc/vaultik/config.yml.
// Explicit paths from --config and $VAULTIK_CONFIG are checked for existence
// so the user gets a clear error instead of a downstream YAML parser failure.
func ResolveConfigPath() (string, error) {
if path := rootFlags.ConfigPath; path != "" {
if _, err := os.Stat(path); err != nil {
return "", fmt.Errorf("config file from --config not found: %s (run 'vaultik config init --config %s' to create it)", path, path)
_, err := os.Stat(path)
if err != nil {
return "", fmt.Errorf(
"%w: from --config: %s (run 'vaultik config init --config %s' to create it)",
errConfigNotFound, path, path)
}
return path, nil
}
if path := os.Getenv("VAULTIK_CONFIG"); path != "" {
if _, err := os.Stat(path); err != nil {
return "", fmt.Errorf("config file from $VAULTIK_CONFIG not found: %s (unset VAULTIK_CONFIG, point it at an existing file, or run 'vaultik config init')", path)
_, err := os.Stat(path) //nolint:gosec // G703: path is operator-supplied by design
if err != nil {
return "", fmt.Errorf(
"%w: from $VAULTIK_CONFIG: %s (unset VAULTIK_CONFIG, point it at "+
"an existing file, or run 'vaultik config init')",
errConfigNotFound, path)
}
return path, nil
}
for _, path := range defaultConfigPaths() {
if _, err := os.Stat(path); err == nil {
_, err := os.Stat(path)
if err == nil {
return path, nil
}
}
return "", fmt.Errorf("no config file found at %s (run 'vaultik config init' to create the default config, or pass --config <path>)", strings.Join(defaultConfigPaths(), " or "))
return "", fmt.Errorf(
"%w: searched %s (run 'vaultik config init' to create the default "+
"config, or pass --config <path>)",
errConfigNotFound, strings.Join(defaultConfigPaths(), " or "))
}
// defaultConfigPaths returns the ordered list of config paths to search.
@@ -114,5 +139,6 @@ func DefaultConfigPath() string {
if os.Getuid() == 0 {
return "/etc/vaultik/config.yml"
}
return filepath.Join(xdg.ConfigHome, "vaultik", "config.yml")
}

View File

@@ -2,6 +2,7 @@ package cli
import (
"context"
"errors"
"fmt"
"os"
@@ -11,6 +12,32 @@ import (
"sneak.berlin/go/vaultik/internal/vaultik"
)
var (
errSnapshotIDRequired = errors.New("snapshot ID required")
errWrongArgCount = errors.New("wrong argument count")
errPurgeCriteriaNeeded = errors.New(
"must specify either --keep-latest or --older-than")
errPurgeCriteriaBoth = errors.New(
"cannot specify both --keep-latest and --older-than")
)
// requireSnapshotIDArg validates that exactly one positional argument
// (the snapshot ID) was supplied, printing help otherwise.
func requireSnapshotIDArg(cmd *cobra.Command, args []string) error {
if len(args) != 1 {
_ = cmd.Help()
if len(args) == 0 {
return errSnapshotIDRequired
}
return fmt.Errorf("%w: expected 1 argument, got %d",
errWrongArgCount, len(args))
}
return nil
}
// NewSnapshotCommand creates the snapshot command and subcommands
func NewSnapshotCommand() *cobra.Command {
cmd := &cobra.Command{
@@ -58,9 +85,10 @@ specifying a path using --config or by setting VAULTIK_CONFIG to a path.`,
// Use the backup functionality from cli package
rootFlags := GetRootFlags()
return RunWithApp(cmd.Context(), AppOptions{
ConfigPath: configPath,
LogOptions: log.LogOptions{
LogOptions: log.Options{
Verbose: rootFlags.Verbose,
Debug: rootFlags.Debug,
Cron: opts.Cron,
@@ -70,29 +98,33 @@ specifying a path using --config or by setting VAULTIK_CONFIG to a path.`,
Invokes: []fx.Option{
fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) {
lc.Append(fx.Hook{
OnStart: func(ctx context.Context) error {
OnStart: func(_ context.Context) error {
// Start the snapshot creation in a goroutine
go func() {
// --cron suppression is wired through v.UI by setupGlobals.
if err := v.CreateSnapshot(opts); err != nil {
if err != context.Canceled {
err := v.CreateSnapshot(opts)
if err != nil {
if !errors.Is(err, context.Canceled) {
log.Error("Snapshot creation failed", "error", err)
ReportError("Snapshot creation failed: %v", err)
ReportErrorf("Snapshot creation failed: %v", err)
os.Exit(1)
}
}
// Shutdown the app when snapshot completes
if err := v.Shutdowner.Shutdown(); err != nil {
err = v.Shutdowner.Shutdown()
if err != nil {
log.Error("Failed to shutdown", "error", err)
}
}()
return nil
},
OnStop: func(ctx context.Context) error {
OnStop: func(_ context.Context) error {
log.Debug("Stopping snapshot creation")
// Cancel the Vaultik context
v.Cancel()
return nil
},
})
@@ -102,9 +134,14 @@ specifying a path using --config or by setting VAULTIK_CONFIG to a path.`,
},
}
cmd.Flags().BoolVar(&opts.Cron, "cron", false, "Run in cron mode (silent unless error)")
cmd.Flags().BoolVar(&opts.Prune, "prune", false, "After backup, drop older snapshots of the same name and remove orphaned blobs")
cmd.Flags().StringVar(&opts.KeepNewerThan, "keep-newer-than", "", "With --prune: keep snapshots newer than this duration (e.g. 4w, 30d, 6mo) instead of only the latest")
cmd.Flags().BoolVar(&opts.Cron, "cron", false,
"Run in cron mode (silent unless error)")
cmd.Flags().BoolVar(&opts.Prune, "prune", false,
"After backup, drop older snapshots of the same name and remove "+
"orphaned blobs")
cmd.Flags().StringVar(&opts.KeepNewerThan, "keep-newer-than", "",
"With --prune: keep snapshots newer than this duration "+
"(e.g. 4w, 30d, 6mo) instead of only the latest")
return cmd
}
@@ -119,47 +156,11 @@ func newSnapshotListCommand() *cobra.Command {
Short: "List all snapshots",
Long: "Lists all snapshots with their ID, timestamp, and compressed size",
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, args []string) error {
// Use unified config resolution
configPath, err := ResolveConfigPath()
if err != nil {
return err
}
rootFlags := GetRootFlags()
return RunWithApp(cmd.Context(), AppOptions{
ConfigPath: configPath,
LogOptions: log.LogOptions{
Verbose: rootFlags.Verbose,
Debug: rootFlags.Debug,
Quiet: rootFlags.Quiet,
},
Modules: []fx.Option{},
Invokes: []fx.Option{
fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) {
lc.Append(fx.Hook{
OnStart: func(ctx context.Context) error {
go func() {
if err := v.ListSnapshots(jsonOutput); err != nil {
if err != context.Canceled {
log.Error("Failed to list snapshots", "error", err)
ReportError("Failed to list snapshots: %v", err)
os.Exit(1)
}
}
if err := v.Shutdowner.Shutdown(); err != nil {
log.Error("Failed to shutdown", "error", err)
}
}()
return nil
},
OnStop: func(ctx context.Context) error {
v.Cancel()
return nil
},
})
}),
},
RunE: func(cmd *cobra.Command, _ []string) error {
return runVaultikApp(cmd, false, false,
"Failed to list snapshots",
func(v *vaultik.Vaultik) error {
return v.ListSnapshots(jsonOutput)
})
},
}
@@ -182,63 +183,31 @@ Retention is per-snapshot-name: --keep-latest keeps the latest of each
configured snapshot name, not the latest globally. Use --snapshot to
restrict the operation to specific snapshot names.`,
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, args []string) error {
RunE: func(cmd *cobra.Command, _ []string) error {
// Validate flags
if !opts.KeepLatest && opts.OlderThan == "" {
return fmt.Errorf("must specify either --keep-latest or --older-than")
return errPurgeCriteriaNeeded
}
if opts.KeepLatest && opts.OlderThan != "" {
return fmt.Errorf("cannot specify both --keep-latest and --older-than")
return errPurgeCriteriaBoth
}
// Use unified config resolution
configPath, err := ResolveConfigPath()
if err != nil {
return err
}
rootFlags := GetRootFlags()
return RunWithApp(cmd.Context(), AppOptions{
ConfigPath: configPath,
LogOptions: log.LogOptions{
Verbose: rootFlags.Verbose,
Debug: rootFlags.Debug,
Quiet: rootFlags.Quiet,
},
Modules: []fx.Option{},
Invokes: []fx.Option{
fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) {
lc.Append(fx.Hook{
OnStart: func(ctx context.Context) error {
go func() {
if err := v.PurgeSnapshotsWithOptions(opts); err != nil {
if err != context.Canceled {
log.Error("Failed to purge snapshots", "error", err)
ReportError("Failed to purge snapshots: %v", err)
os.Exit(1)
}
}
if err := v.Shutdowner.Shutdown(); err != nil {
log.Error("Failed to shutdown", "error", err)
}
}()
return nil
},
OnStop: func(ctx context.Context) error {
v.Cancel()
return nil
},
})
}),
},
return runVaultikApp(cmd, false, false,
"Failed to purge snapshots",
func(v *vaultik.Vaultik) error {
return v.PurgeSnapshotsWithOptions(opts)
})
},
}
cmd.Flags().BoolVar(&opts.KeepLatest, "keep-latest", false, "Keep only the latest snapshot of each name")
cmd.Flags().StringVar(&opts.OlderThan, "older-than", "", "Remove snapshots older than duration (e.g., 30d, 6m, 1y)")
cmd.Flags().BoolVar(&opts.KeepLatest, "keep-latest", false,
"Keep only the latest snapshot of each name")
cmd.Flags().StringVar(&opts.OlderThan, "older-than", "",
"Remove snapshots older than duration (e.g., 30d, 6m, 1y)")
cmd.Flags().BoolVar(&opts.Force, "force", false, "Skip confirmation prompt")
cmd.Flags().StringArrayVar(&opts.Names, "snapshot", nil, "Restrict to snapshots with these names (repeat for multiple)")
cmd.Flags().StringArrayVar(&opts.Names, "snapshot", nil,
"Restrict to snapshots with these names (repeat for multiple)")
return cmd
}
@@ -251,16 +220,7 @@ func newSnapshotVerifyCommand() *cobra.Command {
Use: "verify <snapshot-id>",
Short: "Verify snapshot integrity",
Long: "Verifies that all blobs referenced in a snapshot exist",
Args: func(cmd *cobra.Command, args []string) error {
if len(args) != 1 {
_ = cmd.Help()
if len(args) == 0 {
return fmt.Errorf("snapshot ID required")
}
return fmt.Errorf("expected 1 argument, got %d", len(args))
}
return nil
},
Args: requireSnapshotIDArg,
RunE: func(cmd *cobra.Command, args []string) error {
snapshotID := args[0]
@@ -271,9 +231,10 @@ func newSnapshotVerifyCommand() *cobra.Command {
}
rootFlags := GetRootFlags()
return RunWithApp(cmd.Context(), AppOptions{
ConfigPath: configPath,
LogOptions: log.LogOptions{
LogOptions: log.Options{
Verbose: rootFlags.Verbose,
Debug: rootFlags.Debug,
Quiet: rootFlags.Quiet || opts.JSON,
@@ -282,25 +243,31 @@ func newSnapshotVerifyCommand() *cobra.Command {
Invokes: []fx.Option{
fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) {
lc.Append(fx.Hook{
OnStart: func(ctx context.Context) error {
OnStart: func(_ context.Context) error {
go func() {
if err := v.VerifySnapshotWithOptions(snapshotID, opts); err != nil {
if err != context.Canceled {
err := v.VerifySnapshotWithOptions(snapshotID, opts)
if err != nil {
if !errors.Is(err, context.Canceled) {
if !opts.JSON {
log.Error("Verification failed", "error", err)
ReportError("Verification failed: %v", err)
ReportErrorf("Verification failed: %v", err)
}
os.Exit(1)
}
}
if err := v.Shutdowner.Shutdown(); err != nil {
err = v.Shutdowner.Shutdown()
if err != nil {
log.Error("Failed to shutdown", "error", err)
}
}()
return nil
},
OnStop: func(ctx context.Context) error {
OnStop: func(_ context.Context) error {
v.Cancel()
return nil
},
})
@@ -342,68 +309,24 @@ is reachable to finish remote cleanup.
To wipe the entire destination store and start over, use 'vaultik remote
nuke --force' — it is the single supported entry point for that.`,
Args: func(cmd *cobra.Command, args []string) error {
if len(args) != 1 {
_ = cmd.Help()
if len(args) == 0 {
return fmt.Errorf("snapshot ID required")
}
return fmt.Errorf("expected 1 argument, got %d", len(args))
}
return nil
},
Args: requireSnapshotIDArg,
RunE: func(cmd *cobra.Command, args []string) error {
// Use unified config resolution
configPath, err := ResolveConfigPath()
if err != nil {
return err
}
rootFlags := GetRootFlags()
return RunWithApp(cmd.Context(), AppOptions{
ConfigPath: configPath,
LogOptions: log.LogOptions{
Verbose: rootFlags.Verbose,
Debug: rootFlags.Debug,
Quiet: rootFlags.Quiet || opts.JSON,
},
Modules: []fx.Option{},
Invokes: []fx.Option{
fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) {
lc.Append(fx.Hook{
OnStart: func(ctx context.Context) error {
go func() {
return runVaultikApp(cmd, opts.JSON, opts.JSON,
"Failed to remove snapshot",
func(v *vaultik.Vaultik) error {
_, err := v.RemoveSnapshot(args[0], opts)
if err != nil {
if err != context.Canceled {
if !opts.JSON {
log.Error("Failed to remove snapshot", "error", err)
ReportError("Failed to remove snapshot: %v", err)
}
os.Exit(1)
}
}
if err := v.Shutdowner.Shutdown(); err != nil {
log.Error("Failed to shutdown", "error", err)
}
}()
return nil
},
OnStop: func(ctx context.Context) error {
v.Cancel()
return nil
},
})
}),
},
return err
})
},
}
cmd.Flags().BoolVarP(&opts.Force, "force", "f", false, "Skip confirmation prompt")
cmd.Flags().BoolVar(&opts.DryRun, "dry-run", false, "Show what would be removed without removing")
cmd.Flags().BoolVar(&opts.DryRun, "dry-run", false,
"Show what would be removed without removing")
cmd.Flags().BoolVar(&opts.JSON, "json", false, "Output result as JSON")
cmd.Flags().BoolVar(&opts.LocalOnly, "local-only", false, "Skip remote cleanup; only touch the local index")
cmd.Flags().BoolVar(&opts.LocalOnly, "local-only", false,
"Skip remote cleanup; only touch the local index")
return cmd
}

View File

@@ -2,6 +2,7 @@ package cli
import (
"context"
"errors"
"os"
"github.com/spf13/cobra"
@@ -13,6 +14,10 @@ import (
"sneak.berlin/go/vaultik/internal/vaultik"
)
// restoreMinArgs is the minimum positional argument count of
// `snapshot restore <snapshot-id> <target-dir> [paths...]`.
const restoreMinArgs = 2
// RestoreOptions contains options for the restore command
type RestoreOptions struct {
TargetDir string
@@ -38,31 +43,36 @@ func newSnapshotRestoreCommand() *cobra.Command {
Short: "Restore files from a snapshot",
Long: `Download and decrypt files from a backup snapshot.
This command will restore files from the specified snapshot to the target directory.
This command will restore files from the specified snapshot to the
target directory.
If no paths are specified, all files are restored.
If paths are specified, only matching files/directories are restored.
Requires the VAULTIK_AGE_SECRET_KEY environment variable to be set with the age private key.
Requires the VAULTIK_AGE_SECRET_KEY environment variable to be set with
the age private key.
Examples:
# Restore entire snapshot
vaultik snapshot restore myhost_docs_2025-01-01T12:00:00Z /restore
# Restore specific file
vaultik snapshot restore myhost_docs_2025-01-01T12:00:00Z /restore /home/user/important.txt
vaultik snapshot restore myhost_docs_2025-01-01T12:00:00Z /restore \
/home/user/important.txt
# Restore specific directory
vaultik snapshot restore myhost_docs_2025-01-01T12:00:00Z /restore /home/user/documents/
vaultik snapshot restore myhost_docs_2025-01-01T12:00:00Z /restore \
/home/user/documents/
# Restore and verify all files
vaultik snapshot restore --verify myhost_docs_2025-01-01T12:00:00Z /restore`,
Args: cobra.MinimumNArgs(2),
Args: cobra.MinimumNArgs(restoreMinArgs),
RunE: func(cmd *cobra.Command, args []string) error {
return runRestore(cmd, args, opts)
},
}
cmd.Flags().BoolVar(&opts.Verify, "verify", false, "Verify restored files by checking chunk hashes")
cmd.Flags().BoolVar(&opts.Verify, "verify", false,
"Verify restored files by checking chunk hashes")
return cmd
}
@@ -70,9 +80,10 @@ Examples:
// runRestore parses arguments and runs the restore operation through the app framework
func runRestore(cmd *cobra.Command, args []string, opts *RestoreOptions) error {
snapshotID := args[0]
opts.TargetDir = args[1]
if len(args) > 2 {
opts.Paths = args[2:]
if len(args) > restoreMinArgs {
opts.Paths = args[restoreMinArgs:]
}
// Use unified config resolution
@@ -83,9 +94,10 @@ func runRestore(cmd *cobra.Command, args []string, opts *RestoreOptions) error {
// Use the app framework like other commands
rootFlags := GetRootFlags()
return RunWithApp(cmd.Context(), AppOptions{
ConfigPath: configPath,
LogOptions: log.LogOptions{
LogOptions: log.Options{
Verbose: rootFlags.Verbose,
Debug: rootFlags.Debug,
Quiet: rootFlags.Quiet,
@@ -118,7 +130,7 @@ func buildRestoreInvokes(snapshotID string, opts *RestoreOptions) []fx.Option {
return []fx.Option{
fx.Invoke(func(app *RestoreApp, lc fx.Lifecycle) {
lc.Append(fx.Hook{
OnStart: func(ctx context.Context) error {
OnStart: func(_ context.Context) error {
// Start the restore operation in a goroutine
go func() {
// Run the restore operation
@@ -129,24 +141,29 @@ func buildRestoreInvokes(snapshotID string, opts *RestoreOptions) []fx.Option {
Verify: opts.Verify,
SkipErrors: GetRootFlags().SkipErrors,
}
if err := app.Vaultik.Restore(restoreOpts); err != nil {
if err != context.Canceled {
err := app.Vaultik.Restore(restoreOpts)
if err != nil {
if !errors.Is(err, context.Canceled) {
log.Error("Restore operation failed", "error", err)
ReportError("Restore failed: %v", err)
ReportErrorf("Restore failed: %v", err)
os.Exit(1)
}
}
// Shutdown the app when restore completes
if err := app.Shutdowner.Shutdown(); err != nil {
err = app.Shutdowner.Shutdown()
if err != nil {
log.Error("Failed to shutdown", "error", err)
}
}()
return nil
},
OnStop: func(ctx context.Context) error {
OnStop: func(_ context.Context) error {
log.Debug("Stopping restore operation")
app.Vaultik.Cancel()
return nil
},
})

View File

@@ -3,6 +3,8 @@ package cli
import "time"
// SnapshotInfo represents snapshot information for listing
//
//nolint:tagliatelle // snake_case is the established output format
type SnapshotInfo struct {
ID string `json:"id"`
Timestamp time.Time `json:"timestamp"`

View File

@@ -2,6 +2,7 @@ package cli
import (
"fmt"
"os"
"runtime"
"github.com/spf13/cobra"
@@ -15,20 +16,25 @@ func NewVersionCommand() *cobra.Command {
Short: "Print version information",
Long: `Print version, git commit, and build information for vaultik.`,
Args: cobra.NoArgs,
Run: func(cmd *cobra.Command, args []string) {
fmt.Printf("vaultik %s\n", globals.Version)
fmt.Printf(" commit: %s\n", globals.Commit)
fmt.Printf(" build date: %s\n", globals.CommitDate)
fmt.Printf(" go: %s\n", runtime.Version())
fmt.Printf(" os/arch: %s/%s\n", runtime.GOOS, runtime.GOARCH)
fmt.Printf(" author: %s\n", globals.Author)
fmt.Printf(" homepage: %s\n", globals.Homepage)
fmt.Printf(" license: %s\n", globals.License)
Run: func(_ *cobra.Command, _ []string) {
_, _ = fmt.Fprintf(os.Stdout, "vaultik %s\n", globals.Version)
_, _ = fmt.Fprintf(os.Stdout, " commit: %s\n", globals.Commit)
_, _ = fmt.Fprintf(os.Stdout, " build date: %s\n", globals.CommitDate)
_, _ = fmt.Fprintf(os.Stdout, " go: %s\n", runtime.Version())
_, _ = fmt.Fprintf(os.Stdout, " os/arch: %s/%s\n",
runtime.GOOS, runtime.GOARCH)
_, _ = fmt.Fprintf(os.Stdout, " author: %s\n", globals.Author)
_, _ = fmt.Fprintf(os.Stdout, " homepage: %s\n", globals.Homepage)
_, _ = fmt.Fprintf(os.Stdout, " license: %s\n", globals.License)
if globals.Version == "dev" {
fmt.Println()
fmt.Println("This is a development build (no version information embedded).")
fmt.Println("Build a release binary with 'make vaultik' or download from")
fmt.Println("https://sneak.berlin/go/vaultik for embedded version metadata.")
_, _ = fmt.Fprintln(os.Stdout)
_, _ = fmt.Fprintln(os.Stdout,
"This is a development build (no version information embedded).")
_, _ = fmt.Fprintln(os.Stdout,
"Build a release binary with 'make vaultik' or download from")
_, _ = fmt.Fprintln(os.Stdout,
"https://sneak.berlin/go/vaultik for embedded version metadata.")
}
},
}

View File

@@ -1,6 +1,10 @@
// Package config loads, validates, and provides the vaultik YAML
// configuration, including snapshot definitions, encryption recipients,
// and storage settings.
package config
import (
"errors"
"fmt"
"os"
"path/filepath"
@@ -17,16 +21,52 @@ import (
const appName = "vaultik"
// Defaults and validation bounds for tunable settings.
const (
defaultBlobSizeLimit = Size(10 * 1024 * 1024 * 1024) // 10GB
defaultChunkSize = Size(10 * 1024 * 1024) // 10MB
defaultS3PartSize = Size(5 * 1024 * 1024) // 5MB
defaultCompressionLevel = 3
minChunkSize = 1024 * 1024 // 1MB
minCompressionLevel = 1
maxCompressionLevel = 19
)
// Sentinel validation errors.
var (
errNoConfigPath = errors.New("config path not provided")
errNoAgeRecipients = errors.New(
"at least one age_recipient is required (generate with: age-keygen)")
errNoSnapshots = errors.New(
"at least one snapshot must be configured (see config.example.yml)")
errSnapshotNoPaths = errors.New("snapshot must have at least one path")
errChunkSizeTooSmall = errors.New("chunk_size must be at least 1MB")
errBlobSizeTooSmall = errors.New("blob_size_limit must be at least chunk_size")
errBadCompression = errors.New("compression_level must be between 1 and 19")
errBadStorageScheme = errors.New(
"storage_url must start with s3://, file://, or rclone://")
errStorageNotConfigured = errors.New(
"storage not configured; set storage_url or provide s3.endpoint + " +
"s3.bucket + credentials")
errS3BucketRequired = errors.New("s3.bucket is required (or set storage_url)")
errS3KeyIDRequired = errors.New("s3.access_key_id is required")
errS3SecretRequired = errors.New("s3.secret_access_key is required")
)
// expandTilde expands ~ at the start of a path to the user's home directory.
func expandTilde(path string) string {
if path == "~" {
home, _ := os.UserHomeDir()
return home
}
if strings.HasPrefix(path, "~/") {
home, _ := os.UserHomeDir()
return filepath.Join(home, path[2:])
}
return path
}
@@ -34,8 +74,10 @@ func expandTilde(path string) string {
func expandTildeInURL(url string) string {
if strings.HasPrefix(url, "file://~/") {
home, _ := os.UserHomeDir()
return "file://" + filepath.Join(home, url[9:])
}
return url
}
@@ -63,6 +105,7 @@ func (c *Config) GetExcludes(snapshotName string) []string {
combined := make([]string, 0, len(c.Exclude)+len(snap.Exclude))
combined = append(combined, c.Exclude...)
combined = append(combined, snap.Exclude...)
return combined
}
@@ -74,6 +117,7 @@ func (c *Config) SnapshotNames() []string {
}
// Sort for deterministic order
sort.Strings(names)
return names
}
@@ -81,12 +125,15 @@ func (c *Config) SnapshotNames() []string {
// It defines all settings for backup operations, including source directories,
// encryption recipients, storage configuration, and performance tuning parameters.
// Configuration is typically loaded from a YAML file.
//
//nolint:tagliatelle // snake_case is the established config-file format
type Config struct {
AgeRecipients []string `yaml:"age_recipients"`
AgeSecretKey string `yaml:"age_secret_key"`
BlobSizeLimit Size `yaml:"blob_size_limit"`
ChunkSize Size `yaml:"chunk_size"`
Exclude []string `yaml:"exclude"` // Global excludes applied to all snapshots
// Exclude holds global excludes applied to all snapshots.
Exclude []string `yaml:"exclude"`
Hostname string `yaml:"hostname"`
IndexPath string `yaml:"index_path"`
S3 S3Config `yaml:"s3"`
@@ -98,13 +145,16 @@ type Config struct {
// Supported formats:
// - s3://bucket/prefix?endpoint=host&region=us-east-1
// - file:///path/to/backup
// For S3 URLs, credentials are still read from s3.access_key_id and s3.secret_access_key.
// For S3 URLs, credentials are still read from s3.access_key_id
// and s3.secret_access_key.
StorageURL string `yaml:"storage_url"`
}
// S3Config represents S3 storage configuration for backup storage.
// It supports both AWS S3 and S3-compatible storage services.
// All fields except UseSSL and PartSize are required.
//
//nolint:tagliatelle // snake_case is the established config-file format
type S3Config struct {
Endpoint string `yaml:"endpoint"`
Bucket string `yaml:"bucket"`
@@ -116,17 +166,17 @@ type S3Config struct {
PartSize Size `yaml:"part_size"`
}
// ConfigPath wraps the config file path for fx dependency injection.
// Path wraps the config file path for fx dependency injection.
// This type allows the config file path to be injected as a distinct type
// rather than a plain string, avoiding conflicts with other string dependencies.
type ConfigPath string
type Path string
// New creates a new Config instance by loading from the specified path.
// This function is used by the fx dependency injection framework.
// Returns an error if the path is empty or if loading fails.
func New(path ConfigPath) (*Config, error) {
func New(path Path) (*Config, error) {
if path == "" {
return nil, fmt.Errorf("config path not provided")
return nil, errNoConfigPath
}
cfg, err := Load(string(path))
@@ -151,20 +201,22 @@ func Load(path string) (*Config, error) {
cfg := &Config{
// Set defaults
BlobSizeLimit: Size(10 * 1024 * 1024 * 1024), // 10GB
ChunkSize: Size(10 * 1024 * 1024), // 10MB
BlobSizeLimit: defaultBlobSizeLimit,
ChunkSize: defaultChunkSize,
IndexPath: filepath.Join(xdg.DataHome, appName, "index.sqlite"),
CompressionLevel: 3,
CompressionLevel: defaultCompressionLevel,
}
// Convert smartconfig data to YAML then unmarshal
configData := sc.Data()
yamlBytes, err := yaml.Marshal(configData)
if err != nil {
return nil, fmt.Errorf("failed to marshal config data: %w", err)
}
if err := yaml.Unmarshal(yamlBytes, cfg); err != nil {
err = yaml.Unmarshal(yamlBytes, cfg)
if err != nil {
return nil, fmt.Errorf("failed to parse config: %w", err)
}
@@ -177,6 +229,7 @@ func Load(path string) (*Config, error) {
for i, path := range snap.Paths {
snap.Paths[i] = expandTilde(path)
}
cfg.Snapshots[name] = snap
}
@@ -196,6 +249,7 @@ func Load(path string) (*Config, error) {
if err != nil {
return nil, fmt.Errorf("failed to get hostname: %w", err)
}
cfg.Hostname = hostname
}
@@ -203,12 +257,15 @@ func Load(path string) (*Config, error) {
if cfg.S3.Region == "" {
cfg.S3.Region = "us-east-1"
}
if cfg.S3.PartSize == 0 {
cfg.S3.PartSize = Size(5 * 1024 * 1024) // 5MB
cfg.S3.PartSize = defaultS3PartSize
}
// Check config file permissions (warn if world or group readable)
if info, err := os.Stat(path); err == nil {
//nolint:gosec // G703: config path is operator-supplied by design
info, statErr := os.Stat(path)
if statErr == nil {
mode := info.Mode().Perm()
if mode&0044 != 0 { // group or world readable
log.Warn("Config file has insecure permissions (contains S3 credentials)",
@@ -218,7 +275,8 @@ func Load(path string) (*Config, error) {
}
}
if err := cfg.Validate(); err != nil {
err = cfg.Validate()
if err != nil {
return nil, fmt.Errorf("invalid config: %w", err)
}
@@ -236,34 +294,36 @@ func Load(path string) (*Config, error) {
// Returns an error describing the first validation failure encountered.
func (c *Config) Validate() error {
if len(c.AgeRecipients) == 0 {
return fmt.Errorf("at least one age_recipient is required (generate with: age-keygen)")
return errNoAgeRecipients
}
if len(c.Snapshots) == 0 {
return fmt.Errorf("at least one snapshot must be configured (see config.example.yml)")
return errNoSnapshots
}
for name, snap := range c.Snapshots {
if len(snap.Paths) == 0 {
return fmt.Errorf("snapshot %q must have at least one path", name)
return fmt.Errorf("%w: %q", errSnapshotNoPaths, name)
}
}
// Validate storage configuration
if err := c.validateStorage(); err != nil {
err := c.validateStorage()
if err != nil {
return err
}
if c.ChunkSize.Int64() < 1024*1024 { // 1MB minimum
return fmt.Errorf("chunk_size must be at least 1MB")
if c.ChunkSize.Int64() < minChunkSize {
return errChunkSizeTooSmall
}
if c.BlobSizeLimit.Int64() < c.ChunkSize.Int64() {
return fmt.Errorf("blob_size_limit must be at least chunk_size")
return errBlobSizeTooSmall
}
if c.CompressionLevel < 1 || c.CompressionLevel > 19 {
return fmt.Errorf("compression_level must be between 1 and 19")
if c.CompressionLevel < minCompressionLevel ||
c.CompressionLevel > maxCompressionLevel {
return errBadCompression
}
return nil
@@ -275,48 +335,56 @@ func (c *Config) Validate() error {
// If StorageURL is not set, legacy S3 configuration is required.
func (c *Config) validateStorage() error {
if c.StorageURL != "" {
// URL-based configuration
if strings.HasPrefix(c.StorageURL, "file://") {
// File storage doesn't need S3 credentials
return nil
}
if strings.HasPrefix(c.StorageURL, "s3://") {
// S3 storage needs credentials
if c.S3.AccessKeyID == "" {
return fmt.Errorf("s3.access_key_id is required for s3:// URLs")
}
if c.S3.SecretAccessKey == "" {
return fmt.Errorf("s3.secret_access_key is required for s3:// URLs")
}
return nil
}
if strings.HasPrefix(c.StorageURL, "rclone://") {
// Rclone storage uses rclone's own config
return nil
}
return fmt.Errorf("storage_url must start with s3://, file://, or rclone://")
return c.validateStorageURL()
}
// Legacy S3 configuration
if c.S3.Endpoint == "" {
return fmt.Errorf("storage not configured; set storage_url or provide s3.endpoint + s3.bucket + credentials")
return errStorageNotConfigured
}
if c.S3.Bucket == "" {
return fmt.Errorf("s3.bucket is required (or set storage_url)")
return errS3BucketRequired
}
if c.S3.AccessKeyID == "" {
return fmt.Errorf("s3.access_key_id is required")
return errS3KeyIDRequired
}
if c.S3.SecretAccessKey == "" {
return fmt.Errorf("s3.secret_access_key is required")
return errS3SecretRequired
}
return nil
}
// validateStorageURL validates URL-based storage configuration. File and
// rclone URLs need no credentials; S3 URLs require the legacy s3.*
// credential fields.
func (c *Config) validateStorageURL() error {
switch {
case strings.HasPrefix(c.StorageURL, "file://"):
// File storage doesn't need S3 credentials
return nil
case strings.HasPrefix(c.StorageURL, "rclone://"):
// Rclone storage uses rclone's own config
return nil
case strings.HasPrefix(c.StorageURL, "s3://"):
// S3 storage needs credentials
if c.S3.AccessKeyID == "" {
return fmt.Errorf("%w for s3:// URLs", errS3KeyIDRequired)
}
if c.S3.SecretAccessKey == "" {
return fmt.Errorf("%w for s3:// URLs", errS3SecretRequired)
}
return nil
default:
return errBadStorageScheme
}
}
// extractAgeSecretKey extracts the AGE-SECRET-KEY from the input using
// the age library's parser, which handles comments and whitespace.
func extractAgeSecretKey(input string) string {
@@ -329,11 +397,14 @@ func extractAgeSecretKey(input string) string {
if id, ok := identities[0].(*age.X25519Identity); ok {
return id.String()
}
return strings.TrimSpace(input)
}
// Module exports the config module for fx dependency injection.
// It provides the Config type to other modules in the application.
//
//nolint:gochecknoglobals // fx module definitions are package globals
var Module = fx.Module("config",
fx.Provide(New),
)

View File

@@ -1,4 +1,4 @@
package config
package config //nolint:testpackage // exercises unexported extractAgeSecretKey
import (
"os"
@@ -7,15 +7,20 @@ import (
)
const (
TEST_SNEAK_AGE_PUBLIC_KEY = "age1278m9q7dp3chsh2dcy82qk27v047zywyvtxwnj4cvt0z65jw6a7q5dqhfj"
TEST_INTEGRATION_AGE_PUBLIC_KEY = "age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05gl0sjq9q9wjg"
TEST_INTEGRATION_AGE_PRIVATE_KEY = "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5"
testSneakAgePublicKey = "age1278m9q7dp3chsh2dcy82qk27v047zywyvt" +
"xwnj4cvt0z65jw6a7q5dqhfj"
testIntegrationAgePublicKey = "age1ezrjmfpwsc95svdg0y54mums3zevgzu" +
"0x0ecq2f7tp8a05gl0sjq9q9wjg"
testIntegrationAgePrivateKey = "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GX" +
"VEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5"
)
func TestMain(m *testing.M) {
// Set up test environment
testConfigPath := filepath.Join("..", "..", "test", "config.yaml")
if absPath, err := filepath.Abs(testConfigPath); err == nil {
absPath, err := filepath.Abs(testConfigPath)
if err == nil {
_ = os.Setenv("VAULTIK_CONFIG", absPath)
}
@@ -23,8 +28,11 @@ func TestMain(m *testing.M) {
os.Exit(code)
}
// TestConfigLoad ensures the config package can be imported and basic functionality works
// TestConfigLoad ensures the config package can be imported and basic
// functionality works.
func TestConfigLoad(t *testing.T) {
t.Parallel()
// Use the test config file
configPath := os.Getenv("VAULTIK_CONFIG")
if configPath == "" {
@@ -41,8 +49,10 @@ func TestConfigLoad(t *testing.T) {
if len(cfg.AgeRecipients) != 2 {
t.Errorf("Expected 2 age recipients, got %d", len(cfg.AgeRecipients))
}
if cfg.AgeRecipients[0] != TEST_SNEAK_AGE_PUBLIC_KEY {
t.Errorf("Expected first age recipient to be %s, got '%s'", TEST_SNEAK_AGE_PUBLIC_KEY, cfg.AgeRecipients[0])
if cfg.AgeRecipients[0] != testSneakAgePublicKey {
t.Errorf("Expected first age recipient to be %s, got '%s'",
testSneakAgePublicKey, cfg.AgeRecipients[0])
}
if len(cfg.Snapshots) != 1 {
@@ -59,11 +69,13 @@ func TestConfigLoad(t *testing.T) {
}
if testSnap.Paths[0] != "/tmp/vaultik-test-source" {
t.Errorf("Expected first path to be '/tmp/vaultik-test-source', got '%s'", testSnap.Paths[0])
t.Errorf("Expected first path to be '/tmp/vaultik-test-source', got '%s'",
testSnap.Paths[0])
}
if cfg.S3.Bucket != "vaultik-test-bucket" {
t.Errorf("Expected S3 bucket to be 'vaultik-test-bucket', got '%s'", cfg.S3.Bucket)
t.Errorf("Expected S3 bucket to be 'vaultik-test-bucket', got '%s'",
cfg.S3.Bucket)
}
if cfg.Hostname != "test-host" {
@@ -73,19 +85,26 @@ func TestConfigLoad(t *testing.T) {
// TestConfigFromEnv tests loading config path from environment variable
func TestConfigFromEnv(t *testing.T) {
t.Parallel()
configPath := os.Getenv("VAULTIK_CONFIG")
if configPath == "" {
t.Skip("VAULTIK_CONFIG not set")
}
// Verify the file exists
if _, err := os.Stat(configPath); os.IsNotExist(err) {
t.Errorf("Config file does not exist at path from VAULTIK_CONFIG: %s", configPath)
//nolint:gosec // G703: test config path comes from the test environment
_, err := os.Stat(configPath)
if os.IsNotExist(err) {
t.Errorf("Config file does not exist at path from VAULTIK_CONFIG: %s",
configPath)
}
}
// TestExtractAgeSecretKey tests extraction of AGE-SECRET-KEY from various inputs
func TestExtractAgeSecretKey(t *testing.T) {
t.Parallel()
tests := []struct {
name string
input string
@@ -93,36 +112,32 @@ func TestExtractAgeSecretKey(t *testing.T) {
}{
{
name: "plain key",
input: "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5",
expected: "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5",
input: testIntegrationAgePrivateKey,
expected: testIntegrationAgePrivateKey,
},
{
name: "key with trailing newline",
input: "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5\n",
expected: "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5",
input: testIntegrationAgePrivateKey + "\n",
expected: testIntegrationAgePrivateKey,
},
{
name: "full age-keygen output",
input: `# created: 2025-01-14T12:00:00Z
# public key: age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05gl0sjq9q9wjg
AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5
`,
expected: "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5",
input: "# created: 2025-01-14T12:00:00Z\n" +
"# public key: " + testIntegrationAgePublicKey + "\n" +
testIntegrationAgePrivateKey + "\n",
expected: testIntegrationAgePrivateKey,
},
{
name: "age-keygen output with extra blank lines",
input: `# created: 2025-01-14T12:00:00Z
# public key: age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05gl0sjq9q9wjg
AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5
`,
expected: "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5",
input: "# created: 2025-01-14T12:00:00Z\n" +
"# public key: " + testIntegrationAgePublicKey + "\n\n" +
testIntegrationAgePrivateKey + "\n\n",
expected: testIntegrationAgePrivateKey,
},
{
name: "key with leading whitespace",
input: " AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5 ",
expected: "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5",
input: " " + testIntegrationAgePrivateKey + " ",
expected: testIntegrationAgePrivateKey,
},
{
name: "empty input",
@@ -138,9 +153,12 @@ AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
result := extractAgeSecretKey(tt.input)
if result != tt.expected {
t.Errorf("extractAgeSecretKey(%q) = %q, want %q", tt.input, result, tt.expected)
t.Errorf("extractAgeSecretKey(%q) = %q, want %q",
tt.input, result, tt.expected)
}
})
}

View File

@@ -1,31 +1,45 @@
package config
import (
"errors"
"fmt"
"math"
"github.com/dustin/go-humanize"
)
var (
errSizeType = errors.New("size must be a number or string")
errSizeTooLarge = errors.New("size exceeds maximum supported value")
)
// Size represents a byte size that can be specified in configuration files.
// It can unmarshal from both numeric values (interpreted as bytes) and
// human-readable strings like "10MB", "2.5GB", or "1TB".
//
//nolint:recvcheck // UnmarshalYAML requires a pointer; String/Int64 are value reads
type Size int64
// UnmarshalYAML implements yaml.Unmarshaler for Size, allowing it to be
// parsed from YAML configuration files. It accepts both numeric values
// (interpreted as bytes) and string values with units (e.g., "10MB").
func (s *Size) UnmarshalYAML(unmarshal func(interface{}) error) error {
func (s *Size) UnmarshalYAML(unmarshal func(any) error) error {
// Try to unmarshal as int64 first
var intVal int64
if err := unmarshal(&intVal); err == nil {
err := unmarshal(&intVal)
if err == nil {
*s = Size(intVal)
return nil
}
// Try to unmarshal as string
var strVal string
if err := unmarshal(&strVal); err != nil {
return fmt.Errorf("size must be a number or string")
err = unmarshal(&strVal)
if err != nil {
return errSizeType
}
// Parse the string using go-humanize
@@ -34,7 +48,12 @@ func (s *Size) UnmarshalYAML(unmarshal func(interface{}) error) error {
return fmt.Errorf("invalid size format: %w", err)
}
if bytes > math.MaxInt64 {
return fmt.Errorf("%w: %s", errSizeTooLarge, strVal)
}
*s = Size(bytes)
return nil
}
@@ -49,6 +68,7 @@ func (s Size) Int64() int64 {
// For example, 1048576 bytes would be formatted as "1.0 MB".
// This implements the fmt.Stringer interface.
func (s Size) String() string {
//nolint:gosec // G115: sizes are non-negative by construction
return humanize.Bytes(uint64(s))
}
@@ -58,5 +78,10 @@ func ParseSize(s string) (Size, error) {
if err != nil {
return 0, fmt.Errorf("invalid size format: %w", err)
}
if bytes > math.MaxInt64 {
return 0, fmt.Errorf("%w: %s", errSizeTooLarge, s)
}
return Size(bytes), nil
}

View File

@@ -1,7 +1,10 @@
// Package crypto provides thread-safe age encryption and decryption
// helpers used to protect blob and metadata content.
package crypto
import (
"bytes"
"errors"
"fmt"
"io"
"sync"
@@ -10,6 +13,10 @@ import (
"go.uber.org/fx"
)
// ErrNoRecipients is returned when an encryptor is created or updated
// without any recipient public keys.
var ErrNoRecipients = errors.New("at least one recipient is required")
// Encryptor provides thread-safe encryption using the age encryption library.
// It supports encrypting data for multiple recipients simultaneously, allowing
// any of the corresponding private keys to decrypt the data. This is useful
@@ -25,7 +32,7 @@ type Encryptor struct {
// public keys are invalid or if no recipients are specified.
func NewEncryptor(publicKeys []string) (*Encryptor, error) {
if len(publicKeys) == 0 {
return nil, fmt.Errorf("at least one recipient is required")
return nil, ErrNoRecipients
}
recipients := make([]age.Recipient, 0, len(publicKeys))
@@ -34,6 +41,7 @@ func NewEncryptor(publicKeys []string) (*Encryptor, error) {
if err != nil {
return nil, fmt.Errorf("parsing age recipient %s: %w", key, err)
}
recipients = append(recipients, recipient)
}
@@ -60,12 +68,14 @@ func (e *Encryptor) Encrypt(data []byte) ([]byte, error) {
}
// Write data
if _, err := w.Write(data); err != nil {
_, err = w.Write(data)
if err != nil {
return nil, fmt.Errorf("writing encrypted data: %w", err)
}
// Close to flush
if err := w.Close(); err != nil {
err = w.Close()
if err != nil {
return nil, fmt.Errorf("closing encrypted writer: %w", err)
}
@@ -88,12 +98,14 @@ func (e *Encryptor) EncryptStream(dst io.Writer, src io.Reader) error {
}
// Copy data
if _, err := io.Copy(w, src); err != nil {
_, err = io.Copy(w, src)
if err != nil {
return fmt.Errorf("copying encrypted data: %w", err)
}
// Close to flush
if err := w.Close(); err != nil {
err = w.Close()
if err != nil {
return fmt.Errorf("closing encrypted writer: %w", err)
}
@@ -126,7 +138,7 @@ func (e *Encryptor) EncryptWriter(dst io.Writer) (io.WriteCloser, error) {
// of the public keys are invalid or if no recipients are specified.
func (e *Encryptor) UpdateRecipients(publicKeys []string) error {
if len(publicKeys) == 0 {
return fmt.Errorf("at least one recipient is required")
return ErrNoRecipients
}
recipients := make([]age.Recipient, 0, len(publicKeys))
@@ -135,6 +147,7 @@ func (e *Encryptor) UpdateRecipients(publicKeys []string) error {
if err != nil {
return fmt.Errorf("parsing age recipient %s: %w", key, err)
}
recipients = append(recipients, recipient)
}
@@ -206,4 +219,6 @@ func (d *Decryptor) DecryptStream(src io.Reader) (io.Reader, error) {
}
// Module exports the crypto module for fx dependency injection.
//
//nolint:gochecknoglobals // fx module definitions are package globals
var Module = fx.Module("crypto")

View File

@@ -1,13 +1,16 @@
package crypto
package crypto_test
import (
"bytes"
"testing"
"filippo.io/age"
"sneak.berlin/go/vaultik/internal/crypto"
)
func TestEncryptor(t *testing.T) {
t.Parallel()
// Generate a test key pair
identity, err := age.GenerateX25519Identity()
if err != nil {
@@ -17,7 +20,7 @@ func TestEncryptor(t *testing.T) {
publicKey := identity.Recipient().String()
// Create encryptor
enc, err := NewEncryptor([]string{publicKey})
enc, err := crypto.NewEncryptor([]string{publicKey})
if err != nil {
t.Fatalf("failed to create encryptor: %v", err)
}
@@ -43,7 +46,9 @@ func TestEncryptor(t *testing.T) {
}
var decrypted bytes.Buffer
if _, err := decrypted.ReadFrom(r); err != nil {
_, err = decrypted.ReadFrom(r)
if err != nil {
t.Fatalf("failed to read decrypted data: %v", err)
}
@@ -53,15 +58,19 @@ func TestEncryptor(t *testing.T) {
}
func TestEncryptorMultipleRecipients(t *testing.T) {
t.Parallel()
// Generate three test key pairs
identity1, err := age.GenerateX25519Identity()
if err != nil {
t.Fatalf("failed to generate identity1: %v", err)
}
identity2, err := age.GenerateX25519Identity()
if err != nil {
t.Fatalf("failed to generate identity2: %v", err)
}
identity3, err := age.GenerateX25519Identity()
if err != nil {
t.Fatalf("failed to generate identity3: %v", err)
@@ -74,7 +83,7 @@ func TestEncryptorMultipleRecipients(t *testing.T) {
}
// Create encryptor with multiple recipients
enc, err := NewEncryptor(publicKeys)
enc, err := crypto.NewEncryptor(publicKeys)
if err != nil {
t.Fatalf("failed to create encryptor: %v", err)
}
@@ -97,7 +106,9 @@ func TestEncryptorMultipleRecipients(t *testing.T) {
}
var decrypted bytes.Buffer
if _, err := decrypted.ReadFrom(r); err != nil {
_, err = decrypted.ReadFrom(r)
if err != nil {
t.Fatalf("recipient %d failed to read decrypted data: %v", i+1, err)
}
@@ -108,6 +119,8 @@ func TestEncryptorMultipleRecipients(t *testing.T) {
}
func TestEncryptorUpdateRecipients(t *testing.T) {
t.Parallel()
// Generate two identities
identity1, _ := age.GenerateX25519Identity()
identity2, _ := age.GenerateX25519Identity()
@@ -116,20 +129,22 @@ func TestEncryptorUpdateRecipients(t *testing.T) {
publicKey2 := identity2.Recipient().String()
// Create encryptor with first key
enc, err := NewEncryptor([]string{publicKey1})
enc, err := crypto.NewEncryptor([]string{publicKey1})
if err != nil {
t.Fatalf("failed to create encryptor: %v", err)
}
// Encrypt with first key
plaintext := []byte("test data")
ciphertext1, err := enc.Encrypt(plaintext)
if err != nil {
t.Fatalf("failed to encrypt: %v", err)
}
// Update to second key
if err := enc.UpdateRecipients([]string{publicKey2}); err != nil {
err = enc.UpdateRecipients([]string{publicKey2})
if err != nil {
t.Fatalf("failed to update recipients: %v", err)
}
@@ -140,18 +155,24 @@ func TestEncryptorUpdateRecipients(t *testing.T) {
}
// First ciphertext should only decrypt with first identity
if _, err := age.Decrypt(bytes.NewReader(ciphertext1), identity1); err != nil {
_, err = age.Decrypt(bytes.NewReader(ciphertext1), identity1)
if err != nil {
t.Error("failed to decrypt with identity1")
}
if _, err := age.Decrypt(bytes.NewReader(ciphertext1), identity2); err == nil {
_, err = age.Decrypt(bytes.NewReader(ciphertext1), identity2)
if err == nil {
t.Error("should not decrypt with identity2")
}
// Second ciphertext should only decrypt with second identity
if _, err := age.Decrypt(bytes.NewReader(ciphertext2), identity2); err != nil {
_, err = age.Decrypt(bytes.NewReader(ciphertext2), identity2)
if err != nil {
t.Error("failed to decrypt with identity2")
}
if _, err := age.Decrypt(bytes.NewReader(ciphertext2), identity1); err == nil {
_, err = age.Decrypt(bytes.NewReader(ciphertext2), identity1)
if err == nil {
t.Error("should not decrypt with identity1")
}
}

View File

@@ -3,18 +3,25 @@ package database
import (
"context"
"database/sql"
"errors"
"fmt"
)
// BlobChunkRepository provides access to the blob_chunks table, which maps
// blobs to the chunks they contain (with offset and length).
type BlobChunkRepository struct {
db *DB
}
// NewBlobChunkRepository creates a BlobChunkRepository backed by db.
func NewBlobChunkRepository(db *DB) *BlobChunkRepository {
return &BlobChunkRepository{db: db}
}
func (r *BlobChunkRepository) Create(ctx context.Context, tx *sql.Tx, bc *BlobChunk) error {
// Create inserts a blob_chunks row, using tx when non-nil.
func (r *BlobChunkRepository) Create(
ctx context.Context, tx *sql.Tx, bc *BlobChunk,
) error {
query := `
INSERT INTO blob_chunks (blob_id, chunk_hash, offset, length)
VALUES (?, ?, ?, ?)
@@ -34,7 +41,11 @@ func (r *BlobChunkRepository) Create(ctx context.Context, tx *sql.Tx, bc *BlobCh
return nil
}
func (r *BlobChunkRepository) GetByBlobID(ctx context.Context, blobID string) ([]*BlobChunk, error) {
// GetByBlobID returns all chunks contained in the given blob, ordered by
// their offset within the blob.
func (r *BlobChunkRepository) GetByBlobID(
ctx context.Context, blobID string,
) ([]*BlobChunk, error) {
query := `
SELECT blob_id, chunk_hash, offset, length
FROM blob_chunks
@@ -49,19 +60,26 @@ func (r *BlobChunkRepository) GetByBlobID(ctx context.Context, blobID string) ([
defer CloseRows(rows)
var blobChunks []*BlobChunk
for rows.Next() {
var bc BlobChunk
err := rows.Scan(&bc.BlobID, &bc.ChunkHash, &bc.Offset, &bc.Length)
if err != nil {
return nil, fmt.Errorf("scanning blob chunk: %w", err)
}
blobChunks = append(blobChunks, &bc)
}
return blobChunks, rows.Err()
}
func (r *BlobChunkRepository) GetByChunkHash(ctx context.Context, chunkHash string) (*BlobChunk, error) {
// GetByChunkHash returns one blob_chunks row containing the given chunk,
// or nil if the chunk is not packed in any blob.
func (r *BlobChunkRepository) GetByChunkHash(
ctx context.Context, chunkHash string,
) (*BlobChunk, error) {
query := `
SELECT blob_id, chunk_hash, offset, length
FROM blob_chunks
@@ -70,7 +88,9 @@ func (r *BlobChunkRepository) GetByChunkHash(ctx context.Context, chunkHash stri
`
LogSQL("GetByChunkHash", query, chunkHash)
var bc BlobChunk
err := r.db.conn.QueryRowContext(ctx, query, chunkHash).Scan(
&bc.BlobID,
&bc.ChunkHash,
@@ -78,21 +98,27 @@ func (r *BlobChunkRepository) GetByChunkHash(ctx context.Context, chunkHash stri
&bc.Length,
)
if err == sql.ErrNoRows {
if errors.Is(err, sql.ErrNoRows) {
LogSQL("GetByChunkHash", "No rows found", chunkHash)
return nil, nil
return nil, nil //nolint:nilnil // nil,nil signals not-found; callers check nil
}
if err != nil {
LogSQL("GetByChunkHash", "Error", chunkHash, err)
return nil, fmt.Errorf("querying blob chunk: %w", err)
}
LogSQL("GetByChunkHash", "Found blob", chunkHash, "blob", bc.BlobID)
return &bc, nil
}
// GetByChunkHashTx retrieves a blob chunk within a transaction
func (r *BlobChunkRepository) GetByChunkHashTx(ctx context.Context, tx *sql.Tx, chunkHash string) (*BlobChunk, error) {
func (r *BlobChunkRepository) GetByChunkHashTx(
ctx context.Context, tx *sql.Tx, chunkHash string,
) (*BlobChunk, error) {
query := `
SELECT blob_id, chunk_hash, offset, length
FROM blob_chunks
@@ -101,7 +127,9 @@ func (r *BlobChunkRepository) GetByChunkHashTx(ctx context.Context, tx *sql.Tx,
`
LogSQL("GetByChunkHashTx", query, chunkHash)
var bc BlobChunk
err := tx.QueryRowContext(ctx, query, chunkHash).Scan(
&bc.BlobID,
&bc.ChunkHash,
@@ -109,20 +137,25 @@ func (r *BlobChunkRepository) GetByChunkHashTx(ctx context.Context, tx *sql.Tx,
&bc.Length,
)
if err == sql.ErrNoRows {
if errors.Is(err, sql.ErrNoRows) {
LogSQL("GetByChunkHashTx", "No rows found", chunkHash)
return nil, nil
return nil, nil //nolint:nilnil // nil,nil signals not-found; callers check nil
}
if err != nil {
LogSQL("GetByChunkHashTx", "Error", chunkHash, err)
return nil, fmt.Errorf("querying blob chunk: %w", err)
}
LogSQL("GetByChunkHashTx", "Found blob", chunkHash, "blob", bc.BlobID)
return &bc, nil
}
// DeleteOrphaned deletes blob_chunks entries where either the blob or chunk no longer exists
// DeleteOrphaned deletes blob_chunks entries where either the blob or the
// chunk no longer exists.
func (r *BlobChunkRepository) DeleteOrphaned(ctx context.Context) error {
// Delete blob_chunks where the blob doesn't exist
query1 := `
@@ -132,7 +165,9 @@ func (r *BlobChunkRepository) DeleteOrphaned(ctx context.Context) error {
WHERE blobs.id = blob_chunks.blob_id
)
`
if _, err := r.db.ExecWithLog(ctx, query1); err != nil {
_, err := r.db.ExecWithLog(ctx, query1)
if err != nil {
return fmt.Errorf("deleting blob_chunks with missing blobs: %w", err)
}
@@ -144,7 +179,9 @@ func (r *BlobChunkRepository) DeleteOrphaned(ctx context.Context) error {
WHERE chunks.chunk_hash = blob_chunks.chunk_hash
)
`
if _, err := r.db.ExecWithLog(ctx, query2); err != nil {
_, err = r.db.ExecWithLog(ctx, query2)
if err != nil {
return fmt.Errorf("deleting blob_chunks with missing chunks: %w", err)
}

View File

@@ -1,4 +1,4 @@
package database
package database_test
import (
"context"
@@ -6,71 +6,107 @@ import (
"testing"
"time"
"sneak.berlin/go/vaultik/internal/database"
"sneak.berlin/go/vaultik/internal/types"
)
func TestBlobChunkRepository(t *testing.T) {
db, cleanup := setupTestDB(t)
defer cleanup()
// Chunk hashes used across the blob_chunks tests.
const (
chunk1Hash = "chunk1"
chunk2Hash = "chunk2"
chunk3Hash = "chunk3"
)
// mustCreateChunks registers the given chunk hashes (1024 bytes each).
func mustCreateChunks(
t *testing.T,
repos *database.Repositories,
hashes ...types.ChunkHash,
) {
t.Helper()
ctx := context.Background()
repos := NewRepositories(db)
// Create blob first
blob := &Blob{
ID: types.NewBlobID(),
Hash: types.BlobHash("blob1-hash"),
CreatedTS: time.Now(),
}
err := repos.Blobs.Create(ctx, nil, blob)
if err != nil {
t.Fatalf("failed to create blob: %v", err)
}
// Create chunks
chunks := []types.ChunkHash{"chunk1", "chunk2", "chunk3"}
for _, chunkHash := range chunks {
chunk := &Chunk{
for _, chunkHash := range hashes {
chunk := &database.Chunk{
ChunkHash: chunkHash,
Size: 1024,
}
err = repos.Chunks.Create(ctx, nil, chunk)
err := repos.Chunks.Create(ctx, nil, chunk)
if err != nil {
t.Fatalf("failed to create chunk %s: %v", chunkHash, err)
}
}
}
// mustCreateBlob creates a blob row with the given hash.
func mustCreateBlob(
t *testing.T,
repos *database.Repositories,
hash types.BlobHash,
) *database.Blob {
t.Helper()
blob := &database.Blob{
ID: types.NewBlobID(),
Hash: hash,
CreatedTS: time.Now(),
}
err := repos.Blobs.Create(context.Background(), nil, blob)
if err != nil {
t.Fatalf("failed to create blob %s: %v", hash, err)
}
return blob
}
func TestBlobChunkRepository(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repos := database.NewRepositories(db)
blob := mustCreateBlob(t, repos, "blob1-hash")
mustCreateChunks(t, repos, chunk1Hash, chunk2Hash, chunk3Hash)
// Test Create
bc1 := &BlobChunk{
bc1 := &database.BlobChunk{
BlobID: blob.ID,
ChunkHash: types.ChunkHash("chunk1"),
ChunkHash: types.ChunkHash(chunk1Hash),
Offset: 0,
Length: 1024,
}
err = repos.BlobChunks.Create(ctx, nil, bc1)
err := repos.BlobChunks.Create(ctx, nil, bc1)
if err != nil {
t.Fatalf("failed to create blob chunk: %v", err)
}
// Add more chunks to the same blob
bc2 := &BlobChunk{
bc2 := &database.BlobChunk{
BlobID: blob.ID,
ChunkHash: types.ChunkHash("chunk2"),
ChunkHash: types.ChunkHash(chunk2Hash),
Offset: 1024,
Length: 2048,
}
err = repos.BlobChunks.Create(ctx, nil, bc2)
if err != nil {
t.Fatalf("failed to create second blob chunk: %v", err)
}
bc3 := &BlobChunk{
bc3 := &database.BlobChunk{
BlobID: blob.ID,
ChunkHash: types.ChunkHash("chunk3"),
ChunkHash: types.ChunkHash(chunk3Hash),
Offset: 3072,
Length: 512,
}
err = repos.BlobChunks.Create(ctx, nil, bc3)
if err != nil {
t.Fatalf("failed to create third blob chunk: %v", err)
@@ -81,6 +117,7 @@ func TestBlobChunkRepository(t *testing.T) {
if err != nil {
t.Fatalf("failed to get blob chunks: %v", err)
}
if len(blobChunks) != 3 {
t.Errorf("expected 3 chunks, got %d", len(blobChunks))
}
@@ -89,92 +126,97 @@ func TestBlobChunkRepository(t *testing.T) {
expectedOffsets := []int64{0, 1024, 3072}
for i, bc := range blobChunks {
if bc.Offset != expectedOffsets[i] {
t.Errorf("wrong chunk order: expected offset %d, got %d", expectedOffsets[i], bc.Offset)
t.Errorf("wrong chunk order: expected offset %d, got %d",
expectedOffsets[i], bc.Offset)
}
}
// Test GetByChunkHash
bc, err := repos.BlobChunks.GetByChunkHash(ctx, "chunk2")
if err != nil {
t.Fatalf("failed to get blob chunk by chunk hash: %v", err)
}
if bc == nil {
t.Fatal("expected blob chunk, got nil")
}
if bc.BlobID != blob.ID {
t.Errorf("wrong blob ID: expected %s, got %s", blob.ID, bc.BlobID)
}
if bc.Offset != 1024 {
t.Errorf("wrong offset: expected 1024, got %d", bc.Offset)
}
// Test duplicate insert (should fail due to primary key constraint)
err = repos.BlobChunks.Create(ctx, nil, bc1)
if err == nil {
t.Fatal("duplicate blob_chunk insert should fail due to primary key constraint")
}
if !strings.Contains(err.Error(), "UNIQUE") && !strings.Contains(err.Error(), "constraint") {
if !strings.Contains(err.Error(), "UNIQUE") &&
!strings.Contains(err.Error(), "constraint") {
t.Fatalf("expected constraint error, got: %v", err)
}
}
func TestBlobChunkRepositoryGetByChunkHash(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repos := database.NewRepositories(db)
blob := mustCreateBlob(t, repos, "blob-gbch-hash")
mustCreateChunks(t, repos, chunk2Hash)
bc2 := &database.BlobChunk{
BlobID: blob.ID,
ChunkHash: types.ChunkHash(chunk2Hash),
Offset: 1024,
Length: 2048,
}
err := repos.BlobChunks.Create(ctx, nil, bc2)
if err != nil {
t.Fatalf("failed to create blob chunk: %v", err)
}
// Test GetByChunkHash
bc, err := repos.BlobChunks.GetByChunkHash(ctx, chunk2Hash)
if err != nil {
t.Fatalf("failed to get blob chunk by chunk hash: %v", err)
}
if bc == nil {
t.Fatal("expected blob chunk, got nil")
}
if bc.BlobID != blob.ID {
t.Errorf("wrong blob ID: expected %s, got %s", blob.ID, bc.BlobID)
}
if bc.Offset != 1024 {
t.Errorf("wrong offset: expected 1024, got %d", bc.Offset)
}
// Test non-existent chunk
bc, err = repos.BlobChunks.GetByChunkHash(ctx, "nonexistent")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if bc != nil {
t.Error("expected nil for non-existent chunk")
}
}
func TestBlobChunkRepositoryMultipleBlobs(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repos := NewRepositories(db)
repos := database.NewRepositories(db)
// Create blobs
blob1 := &Blob{
ID: types.NewBlobID(),
Hash: types.BlobHash("blob1-hash"),
CreatedTS: time.Now(),
}
blob2 := &Blob{
ID: types.NewBlobID(),
Hash: types.BlobHash("blob2-hash"),
CreatedTS: time.Now(),
}
err := repos.Blobs.Create(ctx, nil, blob1)
if err != nil {
t.Fatalf("failed to create blob1: %v", err)
}
err = repos.Blobs.Create(ctx, nil, blob2)
if err != nil {
t.Fatalf("failed to create blob2: %v", err)
}
// Create chunks
chunkHashes := []types.ChunkHash{"chunk1", "chunk2", "chunk3"}
for _, chunkHash := range chunkHashes {
chunk := &Chunk{
ChunkHash: chunkHash,
Size: 1024,
}
err = repos.Chunks.Create(ctx, nil, chunk)
if err != nil {
t.Fatalf("failed to create chunk %s: %v", chunkHash, err)
}
}
blob1 := mustCreateBlob(t, repos, "blob1-hash")
blob2 := mustCreateBlob(t, repos, "blob2-hash")
mustCreateChunks(t, repos, chunk1Hash, chunk2Hash, chunk3Hash)
// Create chunks across multiple blobs
// Some chunks are shared between blobs (deduplication scenario)
blobChunks := []BlobChunk{
{BlobID: blob1.ID, ChunkHash: types.ChunkHash("chunk1"), Offset: 0, Length: 1024},
{BlobID: blob1.ID, ChunkHash: types.ChunkHash("chunk2"), Offset: 1024, Length: 1024},
{BlobID: blob2.ID, ChunkHash: types.ChunkHash("chunk2"), Offset: 0, Length: 1024}, // chunk2 is shared
{BlobID: blob2.ID, ChunkHash: types.ChunkHash("chunk3"), Offset: 1024, Length: 1024},
blobChunks := []database.BlobChunk{
{BlobID: blob1.ID, ChunkHash: chunk1Hash, Offset: 0, Length: 1024},
{BlobID: blob1.ID, ChunkHash: chunk2Hash, Offset: 1024, Length: 1024},
// chunk2 is shared between the blobs
{BlobID: blob2.ID, ChunkHash: chunk2Hash, Offset: 0, Length: 1024},
{BlobID: blob2.ID, ChunkHash: chunk3Hash, Offset: 1024, Length: 1024},
}
for _, bc := range blobChunks {
@@ -189,6 +231,7 @@ func TestBlobChunkRepositoryMultipleBlobs(t *testing.T) {
if err != nil {
t.Fatalf("failed to get blob1 chunks: %v", err)
}
if len(chunks) != 2 {
t.Errorf("expected 2 chunks for blob1, got %d", len(chunks))
}
@@ -198,15 +241,17 @@ func TestBlobChunkRepositoryMultipleBlobs(t *testing.T) {
if err != nil {
t.Fatalf("failed to get blob2 chunks: %v", err)
}
if len(chunks) != 2 {
t.Errorf("expected 2 chunks for blob2, got %d", len(chunks))
}
// Verify shared chunk
bc, err := repos.BlobChunks.GetByChunkHash(ctx, "chunk2")
bc, err := repos.BlobChunks.GetByChunkHash(ctx, chunk2Hash)
if err != nil {
t.Fatalf("failed to get shared chunk: %v", err)
}
if bc == nil {
t.Fatal("expected shared chunk, got nil")
}

View File

@@ -3,31 +3,39 @@ package database
import (
"context"
"database/sql"
"errors"
"fmt"
"time"
"sneak.berlin/go/vaultik/internal/log"
)
// BlobRepository provides access to the blobs table, which tracks the
// packed, encrypted storage units uploaded to the destination.
type BlobRepository struct {
db *DB
}
// NewBlobRepository creates a BlobRepository backed by db.
func NewBlobRepository(db *DB) *BlobRepository {
return &BlobRepository{db: db}
}
// Create inserts a blob row, using tx when non-nil.
func (r *BlobRepository) Create(ctx context.Context, tx *sql.Tx, blob *Blob) error {
query := `
INSERT INTO blobs (id, blob_hash, created_ts, finished_ts, uncompressed_size, compressed_size, uploaded_ts)
INSERT INTO blobs (id, blob_hash, created_ts, finished_ts,
uncompressed_size, compressed_size, uploaded_ts)
VALUES (?, ?, ?, ?, ?, ?, ?)
`
var finishedTS, uploadedTS *int64
if blob.FinishedTS != nil {
ts := blob.FinishedTS.Unix()
finishedTS = &ts
}
if blob.UploadedTS != nil {
ts := blob.UploadedTS.Unix()
uploadedTS = &ts
@@ -49,85 +57,15 @@ func (r *BlobRepository) Create(ctx context.Context, tx *sql.Tx, blob *Blob) err
return nil
}
// GetByHash returns the blob with the given content hash, or nil if no
// such blob exists.
func (r *BlobRepository) GetByHash(ctx context.Context, hash string) (*Blob, error) {
query := `
SELECT id, blob_hash, created_ts, finished_ts, uncompressed_size, compressed_size, uploaded_ts
FROM blobs
WHERE blob_hash = ?
`
var blob Blob
var createdTSUnix int64
var finishedTSUnix, uploadedTSUnix sql.NullInt64
err := r.db.conn.QueryRowContext(ctx, query, hash).Scan(
&blob.ID,
&blob.Hash,
&createdTSUnix,
&finishedTSUnix,
&blob.UncompressedSize,
&blob.CompressedSize,
&uploadedTSUnix,
)
if err == sql.ErrNoRows {
return nil, nil
}
if err != nil {
return nil, fmt.Errorf("querying blob: %w", err)
}
blob.CreatedTS = time.Unix(createdTSUnix, 0).UTC()
if finishedTSUnix.Valid {
ts := time.Unix(finishedTSUnix.Int64, 0).UTC()
blob.FinishedTS = &ts
}
if uploadedTSUnix.Valid {
ts := time.Unix(uploadedTSUnix.Int64, 0).UTC()
blob.UploadedTS = &ts
}
return &blob, nil
return r.getOne(ctx, "blob_hash", hash)
}
// GetByID retrieves a blob by its ID
func (r *BlobRepository) GetByID(ctx context.Context, id string) (*Blob, error) {
query := `
SELECT id, blob_hash, created_ts, finished_ts, uncompressed_size, compressed_size, uploaded_ts
FROM blobs
WHERE id = ?
`
var blob Blob
var createdTSUnix int64
var finishedTSUnix, uploadedTSUnix sql.NullInt64
err := r.db.conn.QueryRowContext(ctx, query, id).Scan(
&blob.ID,
&blob.Hash,
&createdTSUnix,
&finishedTSUnix,
&blob.UncompressedSize,
&blob.CompressedSize,
&uploadedTSUnix,
)
if err == sql.ErrNoRows {
return nil, nil
}
if err != nil {
return nil, fmt.Errorf("querying blob: %w", err)
}
blob.CreatedTS = time.Unix(createdTSUnix, 0).UTC()
if finishedTSUnix.Valid {
ts := time.Unix(finishedTSUnix.Int64, 0).UTC()
blob.FinishedTS = &ts
}
if uploadedTSUnix.Valid {
ts := time.Unix(uploadedTSUnix.Int64, 0).UTC()
blob.UploadedTS = &ts
}
return &blob, nil
return r.getOne(ctx, "id", id)
}
// GetAll returns every blob row keyed by blob ID. Useful at restore
@@ -135,7 +73,8 @@ func (r *BlobRepository) GetByID(ctx context.Context, id string) (*Blob, error)
// into blob hashes without doing one GetByID query per chunk.
func (r *BlobRepository) GetAll(ctx context.Context) (map[string]*Blob, error) {
query := `
SELECT id, blob_hash, created_ts, finished_ts, uncompressed_size, compressed_size, uploaded_ts
SELECT id, blob_hash, created_ts, finished_ts,
uncompressed_size, compressed_size, uploaded_ts
FROM blobs
`
@@ -146,11 +85,15 @@ func (r *BlobRepository) GetAll(ctx context.Context) (map[string]*Blob, error) {
defer CloseRows(rows)
out := make(map[string]*Blob)
for rows.Next() {
var blob Blob
var createdTSUnix int64
var finishedTSUnix, uploadedTSUnix sql.NullInt64
if err := rows.Scan(
var (
blob Blob
createdTSUnix int64
finishedTSUnix, uploadedTSUnix sql.NullInt64
)
err := rows.Scan(
&blob.ID,
&blob.Hash,
&createdTSUnix,
@@ -158,25 +101,36 @@ func (r *BlobRepository) GetAll(ctx context.Context) (map[string]*Blob, error) {
&blob.UncompressedSize,
&blob.CompressedSize,
&uploadedTSUnix,
); err != nil {
)
if err != nil {
return nil, fmt.Errorf("scanning blob: %w", err)
}
blob.CreatedTS = time.Unix(createdTSUnix, 0).UTC()
if finishedTSUnix.Valid {
ts := time.Unix(finishedTSUnix.Int64, 0).UTC()
blob.FinishedTS = &ts
}
if uploadedTSUnix.Valid {
ts := time.Unix(uploadedTSUnix.Int64, 0).UTC()
blob.UploadedTS = &ts
}
out[blob.ID.String()] = &blob
}
return out, rows.Err()
}
// UpdateFinished updates a blob when it's finalized
func (r *BlobRepository) UpdateFinished(ctx context.Context, tx *sql.Tx, id string, hash string, uncompressedSize, compressedSize int64) error {
func (r *BlobRepository) UpdateFinished(
ctx context.Context,
tx *sql.Tx,
id string,
hash string,
uncompressedSize, compressedSize int64,
) error {
query := `
UPDATE blobs
SET blob_hash = ?, finished_ts = ?, uncompressed_size = ?, compressed_size = ?
@@ -184,6 +138,7 @@ func (r *BlobRepository) UpdateFinished(ctx context.Context, tx *sql.Tx, id stri
`
now := time.Now().UTC().Unix()
var err error
if tx != nil {
_, err = tx.ExecContext(ctx, query, hash, now, uncompressedSize, compressedSize, id)
@@ -199,7 +154,9 @@ func (r *BlobRepository) UpdateFinished(ctx context.Context, tx *sql.Tx, id stri
}
// UpdateUploaded marks a blob as uploaded
func (r *BlobRepository) UpdateUploaded(ctx context.Context, tx *sql.Tx, id string) error {
func (r *BlobRepository) UpdateUploaded(
ctx context.Context, tx *sql.Tx, id string,
) error {
query := `
UPDATE blobs
SET uploaded_ts = ?
@@ -207,6 +164,7 @@ func (r *BlobRepository) UpdateUploaded(ctx context.Context, tx *sql.Tx, id stri
`
now := time.Now().UTC().Unix()
var err error
if tx != nil {
_, err = tx.ExecContext(ctx, query, now, id)
@@ -243,3 +201,52 @@ func (r *BlobRepository) DeleteOrphaned(ctx context.Context) error {
return nil
}
// getOne fetches a single blob row matched on the given column, or
// (nil, nil) when no row matches.
func (r *BlobRepository) getOne(
ctx context.Context, column, value string,
) (*Blob, error) {
query := `
SELECT id, blob_hash, created_ts, finished_ts,
uncompressed_size, compressed_size, uploaded_ts
FROM blobs
WHERE ` + column + ` = ?`
var (
blob Blob
createdTSUnix int64
finishedTSUnix, uploadedTSUnix sql.NullInt64
)
err := r.db.conn.QueryRowContext(ctx, query, value).Scan(
&blob.ID,
&blob.Hash,
&createdTSUnix,
&finishedTSUnix,
&blob.UncompressedSize,
&blob.CompressedSize,
&uploadedTSUnix,
)
if errors.Is(err, sql.ErrNoRows) {
return nil, nil //nolint:nilnil // nil,nil signals not-found; callers check nil
}
if err != nil {
return nil, fmt.Errorf("querying blob: %w", err)
}
blob.CreatedTS = time.Unix(createdTSUnix, 0).UTC()
if finishedTSUnix.Valid {
ts := time.Unix(finishedTSUnix.Int64, 0).UTC()
blob.FinishedTS = &ts
}
if uploadedTSUnix.Valid {
ts := time.Unix(uploadedTSUnix.Int64, 0).UTC()
blob.UploadedTS = &ts
}
return &blob, nil
}

View File

@@ -1,22 +1,25 @@
package database
package database_test
import (
"context"
"testing"
"time"
"sneak.berlin/go/vaultik/internal/database"
"sneak.berlin/go/vaultik/internal/types"
)
func TestBlobRepository(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repo := NewBlobRepository(db)
repo := database.NewBlobRepository(db)
// Test Create
blob := &Blob{
blob := &database.Blob{
ID: types.NewBlobID(),
Hash: types.BlobHash("blobhash123"),
CreatedTS: time.Now().Truncate(time.Second),
@@ -32,14 +35,18 @@ func TestBlobRepository(t *testing.T) {
if err != nil {
t.Fatalf("failed to get blob: %v", err)
}
if retrieved == nil {
t.Fatal("expected blob, got nil")
}
if retrieved.Hash != blob.Hash {
t.Errorf("blob hash mismatch: got %s, want %s", retrieved.Hash, blob.Hash)
}
if !retrieved.CreatedTS.Equal(blob.CreatedTS) {
t.Errorf("created timestamp mismatch: got %v, want %v", retrieved.CreatedTS, blob.CreatedTS)
t.Errorf("created timestamp mismatch: got %v, want %v",
retrieved.CreatedTS, blob.CreatedTS)
}
// Test GetByID
@@ -47,26 +54,51 @@ func TestBlobRepository(t *testing.T) {
if err != nil {
t.Fatalf("failed to get blob by ID: %v", err)
}
if retrievedByID == nil {
t.Fatal("expected blob, got nil")
}
if retrievedByID.ID != blob.ID {
t.Errorf("blob ID mismatch: got %s, want %s", retrievedByID.ID, blob.ID)
}
// Test with second blob
blob2 := &Blob{
blob2 := &database.Blob{
ID: types.NewBlobID(),
Hash: types.BlobHash("blobhash456"),
CreatedTS: time.Now().Truncate(time.Second),
}
err = repo.Create(ctx, nil, blob2)
if err != nil {
t.Fatalf("failed to create second blob: %v", err)
}
}
func TestBlobRepositoryUpdates(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repo := database.NewBlobRepository(db)
blob := &database.Blob{
ID: types.NewBlobID(),
Hash: types.BlobHash("blobhash123"),
CreatedTS: time.Now().Truncate(time.Second),
}
err := repo.Create(ctx, nil, blob)
if err != nil {
t.Fatalf("failed to create blob: %v", err)
}
// Test UpdateFinished
now := time.Now()
err = repo.UpdateFinished(ctx, nil, blob.ID.String(), blob.Hash.String(), 1000, 500)
if err != nil {
t.Fatalf("failed to update blob as finished: %v", err)
@@ -77,12 +109,15 @@ func TestBlobRepository(t *testing.T) {
if err != nil {
t.Fatalf("failed to get updated blob: %v", err)
}
if updated.FinishedTS == nil {
t.Fatal("expected finished timestamp to be set")
}
if updated.UncompressedSize != 1000 {
t.Errorf("expected uncompressed size 1000, got %d", updated.UncompressedSize)
}
if updated.CompressedSize != 500 {
t.Errorf("expected compressed size 500, got %d", updated.CompressedSize)
}
@@ -98,6 +133,7 @@ func TestBlobRepository(t *testing.T) {
if err != nil {
t.Fatalf("failed to get uploaded blob: %v", err)
}
if uploaded.UploadedTS == nil {
t.Fatal("expected uploaded timestamp to be set")
}
@@ -108,13 +144,15 @@ func TestBlobRepository(t *testing.T) {
}
func TestBlobRepositoryDuplicate(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repo := NewBlobRepository(db)
repo := database.NewBlobRepository(db)
blob := &Blob{
blob := &database.Blob{
ID: types.NewBlobID(),
Hash: types.BlobHash("duplicate_blob"),
CreatedTS: time.Now().Truncate(time.Second),

View File

@@ -1,3 +1,4 @@
//nolint:testpackage // inspects the unexported database connection
package database
import (
@@ -9,23 +10,13 @@ import (
"sneak.berlin/go/vaultik/internal/types"
)
// TestCascadeDeleteDebug tests cascade delete with debug output
func TestCascadeDeleteDebug(t *testing.T) {
db, cleanup := setupTestDB(t)
defer cleanup()
// createCascadeFixtures creates a file with three chunk mappings for the
// cascade-delete test.
func createCascadeFixtures(t *testing.T, repos *Repositories) *File {
t.Helper()
ctx := context.Background()
repos := NewRepositories(db)
// Check if foreign keys are enabled
var fkEnabled int
err := db.conn.QueryRow("PRAGMA foreign_keys").Scan(&fkEnabled)
if err != nil {
t.Fatal(err)
}
t.Logf("Foreign keys enabled: %d", fkEnabled)
// Create a file
file := &File{
Path: "/cascade-test.txt",
MTime: time.Now().Truncate(time.Second),
@@ -34,18 +25,21 @@ func TestCascadeDeleteDebug(t *testing.T) {
UID: 1000,
GID: 1000,
}
err = repos.Files.Create(ctx, nil, file)
err := repos.Files.Create(ctx, nil, file)
if err != nil {
t.Fatalf("failed to create file: %v", err)
}
t.Logf("Created file with ID: %s", file.ID)
// Create chunks and file-chunk mappings
for i := 0; i < 3; i++ {
for i := range 3 {
chunk := &Chunk{
ChunkHash: types.ChunkHash(fmt.Sprintf("cascade-chunk-%d", i)),
Size: 1024,
}
err = repos.Chunks.Create(ctx, nil, chunk)
if err != nil {
t.Fatalf("failed to create chunk: %v", err)
@@ -56,33 +50,73 @@ func TestCascadeDeleteDebug(t *testing.T) {
Idx: i,
ChunkHash: chunk.ChunkHash,
}
err = repos.FileChunks.Create(ctx, nil, fc)
if err != nil {
t.Fatalf("failed to create file chunk: %v", err)
}
t.Logf("Created file chunk mapping: file_id=%s, idx=%d, chunk=%s", fc.FileID, fc.Idx, fc.ChunkHash)
t.Logf("Created file chunk mapping: file_id=%s, idx=%d, chunk=%s",
fc.FileID, fc.Idx, fc.ChunkHash)
}
// Verify file chunks exist
fileChunks, err := repos.FileChunks.GetByFileID(ctx, file.ID)
return file
}
// logCascadeDebugInfo logs foreign-key state and the file_chunks table
// definition for cascade-delete debugging.
func logCascadeDebugInfo(ctx context.Context, t *testing.T, db *DB) {
t.Helper()
// Check if foreign keys are enabled
var fkEnabled int
err := db.conn.QueryRowContext(ctx, "PRAGMA foreign_keys").Scan(&fkEnabled)
if err != nil {
t.Fatal(err)
}
t.Logf("File chunks before delete: %d", len(fileChunks))
t.Logf("Foreign keys enabled: %d", fkEnabled)
// Check the foreign key constraint
var fkInfo string
err = db.conn.QueryRow(`
err = db.conn.QueryRowContext(ctx, `
SELECT sql FROM sqlite_master
WHERE type='table' AND name='file_chunks'
`).Scan(&fkInfo)
if err != nil {
t.Fatal(err)
}
t.Logf("file_chunks table definition:\n%s", fkInfo)
}
// TestCascadeDeleteDebug tests cascade delete with debug output
func TestCascadeDeleteDebug(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repos := NewRepositories(db)
logCascadeDebugInfo(ctx, t, db)
file := createCascadeFixtures(t, repos)
// Verify file chunks exist
fileChunks, err := repos.FileChunks.GetByFileID(ctx, file.ID)
if err != nil {
t.Fatal(err)
}
t.Logf("File chunks before delete: %d", len(fileChunks))
// Delete the file
t.Log("Deleting file...")
err = repos.Files.DeleteByID(ctx, nil, file.ID)
if err != nil {
t.Fatalf("failed to delete file: %v", err)
@@ -93,6 +127,7 @@ func TestCascadeDeleteDebug(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if deletedFile != nil {
t.Error("file should have been deleted")
} else {
@@ -104,21 +139,27 @@ func TestCascadeDeleteDebug(t *testing.T) {
if err != nil {
t.Fatal(err)
}
t.Logf("File chunks after delete: %d", len(fileChunks))
// Manually check the database
var count int
err = db.conn.QueryRow("SELECT COUNT(*) FROM file_chunks WHERE file_id = ?", file.ID).Scan(&count)
err = db.conn.QueryRowContext(ctx,
"SELECT COUNT(*) FROM file_chunks WHERE file_id = ?", file.ID,
).Scan(&count)
if err != nil {
t.Fatal(err)
}
t.Logf("Manual count of file_chunks for deleted file: %d", count)
if len(fileChunks) != 0 {
t.Errorf("expected 0 file chunks after cascade delete, got %d", len(fileChunks))
// List the remaining chunks
for _, fc := range fileChunks {
t.Logf("Remaining chunk: file_id=%s, idx=%d, chunk=%s", fc.FileID, fc.Idx, fc.ChunkHash)
t.Logf("Remaining chunk: file_id=%s, idx=%d, chunk=%s",
fc.FileID, fc.Idx, fc.ChunkHash)
}
}
}

View File

@@ -4,19 +4,26 @@ import (
"context"
"database/sql"
"fmt"
"strings"
"sneak.berlin/go/vaultik/internal/types"
)
// ChunkFileRepository provides access to the chunk_files table, the
// reverse mapping from chunks to the files that contain them.
type ChunkFileRepository struct {
db *DB
}
// NewChunkFileRepository creates a ChunkFileRepository backed by db.
func NewChunkFileRepository(db *DB) *ChunkFileRepository {
return &ChunkFileRepository{db: db}
}
func (r *ChunkFileRepository) Create(ctx context.Context, tx *sql.Tx, cf *ChunkFile) error {
// Create inserts a chunk_files row (idempotently), using tx when non-nil.
func (r *ChunkFileRepository) Create(
ctx context.Context, tx *sql.Tx, cf *ChunkFile,
) error {
query := `
INSERT INTO chunk_files (chunk_hash, file_id, file_offset, length)
VALUES (?, ?, ?, ?)
@@ -25,9 +32,11 @@ func (r *ChunkFileRepository) Create(ctx context.Context, tx *sql.Tx, cf *ChunkF
var err error
if tx != nil {
_, err = tx.ExecContext(ctx, query, cf.ChunkHash.String(), cf.FileID.String(), cf.FileOffset, cf.Length)
_, err = tx.ExecContext(ctx, query,
cf.ChunkHash.String(), cf.FileID.String(), cf.FileOffset, cf.Length)
} else {
_, err = r.db.ExecWithLog(ctx, query, cf.ChunkHash.String(), cf.FileID.String(), cf.FileOffset, cf.Length)
_, err = r.db.ExecWithLog(ctx, query,
cf.ChunkHash.String(), cf.FileID.String(), cf.FileOffset, cf.Length)
}
if err != nil {
@@ -37,7 +46,10 @@ func (r *ChunkFileRepository) Create(ctx context.Context, tx *sql.Tx, cf *ChunkF
return nil
}
func (r *ChunkFileRepository) GetByChunkHash(ctx context.Context, chunkHash types.ChunkHash) ([]*ChunkFile, error) {
// GetByChunkHash returns all chunk_files rows for the given chunk hash.
func (r *ChunkFileRepository) GetByChunkHash(
ctx context.Context, chunkHash types.ChunkHash,
) ([]*ChunkFile, error) {
query := `
SELECT chunk_hash, file_id, file_offset, length
FROM chunk_files
@@ -53,7 +65,10 @@ func (r *ChunkFileRepository) GetByChunkHash(ctx context.Context, chunkHash type
return r.scanChunkFiles(rows)
}
func (r *ChunkFileRepository) GetByFilePath(ctx context.Context, filePath string) ([]*ChunkFile, error) {
// GetByFilePath returns all chunk_files rows for the file at the given path.
func (r *ChunkFileRepository) GetByFilePath(
ctx context.Context, filePath string,
) ([]*ChunkFile, error) {
query := `
SELECT cf.chunk_hash, cf.file_id, cf.file_offset, cf.length
FROM chunk_files cf
@@ -71,7 +86,9 @@ func (r *ChunkFileRepository) GetByFilePath(ctx context.Context, filePath string
}
// GetByFileID retrieves chunk files by file ID
func (r *ChunkFileRepository) GetByFileID(ctx context.Context, fileID types.FileID) ([]*ChunkFile, error) {
func (r *ChunkFileRepository) GetByFileID(
ctx context.Context, fileID types.FileID,
) ([]*ChunkFile, error) {
query := `
SELECT chunk_hash, file_id, file_offset, length
FROM chunk_files
@@ -87,29 +104,10 @@ func (r *ChunkFileRepository) GetByFileID(ctx context.Context, fileID types.File
return r.scanChunkFiles(rows)
}
// scanChunkFiles is a helper that scans chunk file rows
func (r *ChunkFileRepository) scanChunkFiles(rows *sql.Rows) ([]*ChunkFile, error) {
var chunkFiles []*ChunkFile
for rows.Next() {
var cf ChunkFile
var chunkHashStr, fileIDStr string
err := rows.Scan(&chunkHashStr, &fileIDStr, &cf.FileOffset, &cf.Length)
if err != nil {
return nil, fmt.Errorf("scanning chunk file: %w", err)
}
cf.ChunkHash = types.ChunkHash(chunkHashStr)
cf.FileID, err = types.ParseFileID(fileIDStr)
if err != nil {
return nil, fmt.Errorf("parsing file ID: %w", err)
}
chunkFiles = append(chunkFiles, &cf)
}
return chunkFiles, rows.Err()
}
// DeleteByFileID deletes all chunk_files entries for a given file ID
func (r *ChunkFileRepository) DeleteByFileID(ctx context.Context, tx *sql.Tx, fileID types.FileID) error {
func (r *ChunkFileRepository) DeleteByFileID(
ctx context.Context, tx *sql.Tx, fileID types.FileID,
) error {
query := `DELETE FROM chunk_files WHERE file_id = ?`
var err error
@@ -127,7 +125,11 @@ func (r *ChunkFileRepository) DeleteByFileID(ctx context.Context, tx *sql.Tx, fi
}
// DeleteByFileIDs deletes all chunk_files for multiple files in a single statement.
func (r *ChunkFileRepository) DeleteByFileIDs(ctx context.Context, tx *sql.Tx, fileIDs []types.FileID) error {
//
//nolint:dupl // symmetric implementation for a parallel association table
func (r *ChunkFileRepository) DeleteByFileIDs(
ctx context.Context, tx *sql.Tx, fileIDs []types.FileID,
) error {
if len(fileIDs) == 0 {
return nil
}
@@ -136,14 +138,15 @@ func (r *ChunkFileRepository) DeleteByFileIDs(ctx context.Context, tx *sql.Tx, f
const batchSize = 500
for i := 0; i < len(fileIDs); i += batchSize {
end := i + batchSize
if end > len(fileIDs) {
end = len(fileIDs)
}
end := min(i+batchSize, len(fileIDs))
batch := fileIDs[i:end]
query := "DELETE FROM chunk_files WHERE file_id IN (?" + repeatPlaceholder(len(batch)-1) + ")"
args := make([]interface{}, len(batch))
//nolint:gosec // G202: concatenates constant SQL and "?" placeholders only
query := "DELETE FROM chunk_files WHERE file_id IN (?" +
repeatPlaceholder(len(batch)-1) + ")"
args := make([]any, len(batch))
for j, id := range batch {
args[j] = id.String()
}
@@ -154,6 +157,7 @@ func (r *ChunkFileRepository) DeleteByFileIDs(ctx context.Context, tx *sql.Tx, f
} else {
_, err = r.db.ExecWithLog(ctx, query, args...)
}
if err != nil {
return fmt.Errorf("batch deleting chunk_files: %w", err)
}
@@ -163,30 +167,43 @@ func (r *ChunkFileRepository) DeleteByFileIDs(ctx context.Context, tx *sql.Tx, f
}
// CreateBatch inserts multiple chunk_files in a single statement for efficiency.
func (r *ChunkFileRepository) CreateBatch(ctx context.Context, tx *sql.Tx, cfs []ChunkFile) error {
func (r *ChunkFileRepository) CreateBatch(
ctx context.Context, tx *sql.Tx, cfs []ChunkFile,
) error {
if len(cfs) == 0 {
return nil
}
// Each ChunkFile has 4 values, so batch at 200 to be safe with SQLite's variable limit
// Each chunk_files row binds this many SQL variables.
const chunkFileCols = 4
// Batch at 200 rows to be safe with SQLite's variable limit.
const batchSize = 200
for i := 0; i < len(cfs); i += batchSize {
end := i + batchSize
if end > len(cfs) {
end = len(cfs)
}
end := min(i+batchSize, len(cfs))
batch := cfs[i:end]
query := "INSERT INTO chunk_files (chunk_hash, file_id, file_offset, length) VALUES "
args := make([]interface{}, 0, len(batch)*4)
args := make([]any, 0, len(batch)*chunkFileCols)
var querySb183 strings.Builder
for j, cf := range batch {
if j > 0 {
query += ", "
querySb183.WriteString(", ")
}
query += "(?, ?, ?, ?)"
args = append(args, cf.ChunkHash.String(), cf.FileID.String(), cf.FileOffset, cf.Length)
querySb183.WriteString("(?, ?, ?, ?)")
args = append(args,
cf.ChunkHash.String(), cf.FileID.String(), cf.FileOffset, cf.Length)
}
query += querySb183.String() //nolint:gosec // G202: appends "?" placeholders only
query += " ON CONFLICT(chunk_hash, file_id) DO NOTHING"
var err error
@@ -195,6 +212,7 @@ func (r *ChunkFileRepository) CreateBatch(ctx context.Context, tx *sql.Tx, cfs [
} else {
_, err = r.db.ExecWithLog(ctx, query, args...)
}
if err != nil {
return fmt.Errorf("batch inserting chunk_files: %w", err)
}
@@ -202,3 +220,31 @@ func (r *ChunkFileRepository) CreateBatch(ctx context.Context, tx *sql.Tx, cfs [
return nil
}
// scanChunkFiles is a helper that scans chunk file rows.
func (r *ChunkFileRepository) scanChunkFiles(rows *sql.Rows) ([]*ChunkFile, error) {
var chunkFiles []*ChunkFile
for rows.Next() {
var (
cf ChunkFile
chunkHashStr, fileIDStr string
)
err := rows.Scan(&chunkHashStr, &fileIDStr, &cf.FileOffset, &cf.Length)
if err != nil {
return nil, fmt.Errorf("scanning chunk file: %w", err)
}
cf.ChunkHash = types.ChunkHash(chunkHashStr)
cf.FileID, err = types.ParseFileID(fileIDStr)
if err != nil {
return nil, fmt.Errorf("parsing file ID: %w", err)
}
chunkFiles = append(chunkFiles, &cf)
}
return chunkFiles, rows.Err()
}

View File

@@ -1,120 +1,139 @@
package database
package database_test
import (
"context"
"testing"
"time"
"sneak.berlin/go/vaultik/internal/database"
"sneak.berlin/go/vaultik/internal/types"
)
const chunk4Hash = "chunk4"
// verifyChunkFilePair asserts that the chunk-file rows cover both test
// files at their expected offsets.
func verifyChunkFilePair(
t *testing.T, chunkFiles []*database.ChunkFile,
file1ID, file2ID types.FileID,
) {
t.Helper()
foundFile1 := false
foundFile2 := false
for _, cf := range chunkFiles {
if cf.FileID == file1ID && cf.FileOffset == 0 {
foundFile1 = true
}
if cf.FileID == file2ID && cf.FileOffset == 2048 {
foundFile2 = true
}
}
if !foundFile1 || !foundFile2 {
t.Error("not all expected files found")
}
}
// createChunkFileTestFiles creates the two files used by the chunk-file
// repository tests.
func createChunkFileTestFiles(
t *testing.T, fileRepo *database.FileRepository,
) (*database.File, *database.File) {
t.Helper()
testTime := time.Now().Truncate(time.Second)
file1 := &database.File{
Path: testFilePath1,
MTime: testTime,
Size: 1024,
Mode: 0644,
UID: 1000,
GID: 1000,
LinkTarget: "",
}
file2 := &database.File{
Path: testFilePath2,
MTime: testTime,
Size: 1024,
Mode: 0644,
UID: 1000,
GID: 1000,
LinkTarget: "",
}
mustCreateFile(t, fileRepo, file1)
mustCreateFile(t, fileRepo, file2)
return file1, file2
}
func TestChunkFileRepository(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repo := NewChunkFileRepository(db)
fileRepo := NewFileRepository(db)
chunksRepo := NewChunkRepository(db)
repo := database.NewChunkFileRepository(db)
fileRepo := database.NewFileRepository(db)
repos := database.NewRepositories(db)
// Create test files first
testTime := time.Now().Truncate(time.Second)
file1 := &File{
Path: "/file1.txt",
MTime: testTime,
Size: 1024,
Mode: 0644,
UID: 1000,
GID: 1000,
LinkTarget: "",
}
err := fileRepo.Create(ctx, nil, file1)
if err != nil {
t.Fatalf("failed to create file1: %v", err)
}
file2 := &File{
Path: "/file2.txt",
MTime: testTime,
Size: 1024,
Mode: 0644,
UID: 1000,
GID: 1000,
LinkTarget: "",
}
err = fileRepo.Create(ctx, nil, file2)
if err != nil {
t.Fatalf("failed to create file2: %v", err)
}
// Create chunk first
chunk := &Chunk{
ChunkHash: types.ChunkHash("chunk1"),
Size: 1024,
}
err = chunksRepo.Create(ctx, nil, chunk)
if err != nil {
t.Fatalf("failed to create chunk: %v", err)
}
file1, file2 := createChunkFileTestFiles(t, fileRepo)
mustCreateChunks(t, repos, chunk1Hash)
// Test Create
cf1 := &ChunkFile{
ChunkHash: types.ChunkHash("chunk1"),
cf1 := &database.ChunkFile{
ChunkHash: types.ChunkHash(chunk1Hash),
FileID: file1.ID,
FileOffset: 0,
Length: 1024,
}
err = repo.Create(ctx, nil, cf1)
err := repo.Create(ctx, nil, cf1)
if err != nil {
t.Fatalf("failed to create chunk file: %v", err)
}
// Add same chunk in different file (deduplication scenario)
cf2 := &ChunkFile{
ChunkHash: types.ChunkHash("chunk1"),
cf2 := &database.ChunkFile{
ChunkHash: types.ChunkHash(chunk1Hash),
FileID: file2.ID,
FileOffset: 2048,
Length: 1024,
}
err = repo.Create(ctx, nil, cf2)
if err != nil {
t.Fatalf("failed to create second chunk file: %v", err)
}
// Test GetByChunkHash
chunkFiles, err := repo.GetByChunkHash(ctx, "chunk1")
chunkFiles, err := repo.GetByChunkHash(ctx, chunk1Hash)
if err != nil {
t.Fatalf("failed to get chunk files: %v", err)
}
if len(chunkFiles) != 2 {
t.Errorf("expected 2 files for chunk, got %d", len(chunkFiles))
}
// Verify both files are returned
foundFile1 := false
foundFile2 := false
for _, cf := range chunkFiles {
if cf.FileID == file1.ID && cf.FileOffset == 0 {
foundFile1 = true
}
if cf.FileID == file2.ID && cf.FileOffset == 2048 {
foundFile2 = true
}
}
if !foundFile1 || !foundFile2 {
t.Error("not all expected files found")
}
verifyChunkFilePair(t, chunkFiles, file1.ID, file2.ID)
// Test GetByFileID
chunkFiles, err = repo.GetByFileID(ctx, file1.ID)
if err != nil {
t.Fatalf("failed to get chunks by file ID: %v", err)
}
if len(chunkFiles) != 1 {
t.Errorf("expected 1 chunk for file, got %d", len(chunkFiles))
}
if chunkFiles[0].ChunkHash != types.ChunkHash("chunk1") {
if chunkFiles[0].ChunkHash != types.ChunkHash(chunk1Hash) {
t.Errorf("wrong chunk hash: expected chunk1, got %s", chunkFiles[0].ChunkHash)
}
@@ -126,60 +145,53 @@ func TestChunkFileRepository(t *testing.T) {
}
func TestChunkFileRepositoryComplexDeduplication(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repo := NewChunkFileRepository(db)
fileRepo := NewFileRepository(db)
chunksRepo := NewChunkRepository(db)
repo := database.NewChunkFileRepository(db)
fileRepo := database.NewFileRepository(db)
repos := database.NewRepositories(db)
// Create test files
testTime := time.Now().Truncate(time.Second)
file1 := &File{Path: "/file1.txt", MTime: testTime, Size: 3072, Mode: 0644, UID: 1000, GID: 1000}
file2 := &File{Path: "/file2.txt", MTime: testTime, Size: 3072, Mode: 0644, UID: 1000, GID: 1000}
file3 := &File{Path: "/file3.txt", MTime: testTime, Size: 2048, Mode: 0644, UID: 1000, GID: 1000}
if err := fileRepo.Create(ctx, nil, file1); err != nil {
t.Fatalf("failed to create file1: %v", err)
file1 := &database.File{
Path: testFilePath1, MTime: testTime, Size: 3072,
Mode: 0644, UID: 1000, GID: 1000,
}
if err := fileRepo.Create(ctx, nil, file2); err != nil {
t.Fatalf("failed to create file2: %v", err)
file2 := &database.File{
Path: testFilePath2, MTime: testTime, Size: 3072,
Mode: 0644, UID: 1000, GID: 1000,
}
if err := fileRepo.Create(ctx, nil, file3); err != nil {
t.Fatalf("failed to create file3: %v", err)
file3 := &database.File{
Path: "/file3.txt", MTime: testTime, Size: 2048,
Mode: 0644, UID: 1000, GID: 1000,
}
// Create chunks first
chunks := []types.ChunkHash{"chunk1", "chunk2", "chunk3", "chunk4"}
for _, chunkHash := range chunks {
chunk := &Chunk{
ChunkHash: chunkHash,
Size: 1024,
}
err := chunksRepo.Create(ctx, nil, chunk)
if err != nil {
t.Fatalf("failed to create chunk %s: %v", chunkHash, err)
}
}
mustCreateFile(t, fileRepo, file1)
mustCreateFile(t, fileRepo, file2)
mustCreateFile(t, fileRepo, file3)
mustCreateChunks(t, repos, chunk1Hash, chunk2Hash, chunk3Hash, chunk4Hash)
// Simulate a scenario where multiple files share chunks
// File1: chunk1, chunk2, chunk3
// File2: chunk2, chunk3, chunk4
// File3: chunk1, chunk4
chunkFiles := []ChunkFile{
chunkFiles := []database.ChunkFile{
// File1
{ChunkHash: types.ChunkHash("chunk1"), FileID: file1.ID, FileOffset: 0, Length: 1024},
{ChunkHash: types.ChunkHash("chunk2"), FileID: file1.ID, FileOffset: 1024, Length: 1024},
{ChunkHash: types.ChunkHash("chunk3"), FileID: file1.ID, FileOffset: 2048, Length: 1024},
{ChunkHash: chunk1Hash, FileID: file1.ID, FileOffset: 0, Length: 1024},
{ChunkHash: chunk2Hash, FileID: file1.ID, FileOffset: 1024, Length: 1024},
{ChunkHash: chunk3Hash, FileID: file1.ID, FileOffset: 2048, Length: 1024},
// File2
{ChunkHash: types.ChunkHash("chunk2"), FileID: file2.ID, FileOffset: 0, Length: 1024},
{ChunkHash: types.ChunkHash("chunk3"), FileID: file2.ID, FileOffset: 1024, Length: 1024},
{ChunkHash: types.ChunkHash("chunk4"), FileID: file2.ID, FileOffset: 2048, Length: 1024},
{ChunkHash: chunk2Hash, FileID: file2.ID, FileOffset: 0, Length: 1024},
{ChunkHash: chunk3Hash, FileID: file2.ID, FileOffset: 1024, Length: 1024},
{ChunkHash: chunk4Hash, FileID: file2.ID, FileOffset: 2048, Length: 1024},
// File3
{ChunkHash: types.ChunkHash("chunk1"), FileID: file3.ID, FileOffset: 0, Length: 1024},
{ChunkHash: types.ChunkHash("chunk4"), FileID: file3.ID, FileOffset: 1024, Length: 1024},
{ChunkHash: chunk1Hash, FileID: file3.ID, FileOffset: 0, Length: 1024},
{ChunkHash: chunk4Hash, FileID: file3.ID, FileOffset: 1024, Length: 1024},
}
for _, cf := range chunkFiles {
@@ -190,19 +202,21 @@ func TestChunkFileRepositoryComplexDeduplication(t *testing.T) {
}
// Test chunk1 (used by file1 and file3)
files, err := repo.GetByChunkHash(ctx, "chunk1")
files, err := repo.GetByChunkHash(ctx, chunk1Hash)
if err != nil {
t.Fatalf("failed to get files for chunk1: %v", err)
}
if len(files) != 2 {
t.Errorf("expected 2 files for chunk1, got %d", len(files))
}
// Test chunk2 (used by file1 and file2)
files, err = repo.GetByChunkHash(ctx, "chunk2")
files, err = repo.GetByChunkHash(ctx, chunk2Hash)
if err != nil {
t.Fatalf("failed to get files for chunk2: %v", err)
}
if len(files) != 2 {
t.Errorf("expected 2 files for chunk2, got %d", len(files))
}
@@ -212,6 +226,7 @@ func TestChunkFileRepositoryComplexDeduplication(t *testing.T) {
if err != nil {
t.Fatalf("failed to get chunks for file2: %v", err)
}
if len(file2Chunks) != 3 {
t.Errorf("expected 3 chunks for file2, got %d", len(file2Chunks))
}

View File

@@ -3,19 +3,25 @@ package database
import (
"context"
"database/sql"
"errors"
"fmt"
"strings"
"sneak.berlin/go/vaultik/internal/log"
)
// ChunkRepository provides access to the chunks table, which tracks
// content-defined chunks by hash and size.
type ChunkRepository struct {
db *DB
}
// NewChunkRepository creates a ChunkRepository backed by db.
func NewChunkRepository(db *DB) *ChunkRepository {
return &ChunkRepository{db: db}
}
// Create inserts a chunk row (idempotently), using tx when non-nil.
func (r *ChunkRepository) Create(ctx context.Context, tx *sql.Tx, chunk *Chunk) error {
query := `
INSERT INTO chunks (chunk_hash, size)
@@ -37,6 +43,8 @@ func (r *ChunkRepository) Create(ctx context.Context, tx *sql.Tx, chunk *Chunk)
return nil
}
// GetByHash returns the chunk with the given hash, or nil if it is not
// known to the index.
func (r *ChunkRepository) GetByHash(ctx context.Context, hash string) (*Chunk, error) {
query := `
SELECT chunk_hash, size
@@ -51,9 +59,10 @@ func (r *ChunkRepository) GetByHash(ctx context.Context, hash string) (*Chunk, e
&chunk.Size,
)
if err == sql.ErrNoRows {
return nil, nil
if errors.Is(err, sql.ErrNoRows) {
return nil, nil //nolint:nilnil // nil,nil signals not-found; callers check nil
}
if err != nil {
return nil, fmt.Errorf("querying chunk: %w", err)
}
@@ -61,7 +70,11 @@ func (r *ChunkRepository) GetByHash(ctx context.Context, hash string) (*Chunk, e
return &chunk, nil
}
func (r *ChunkRepository) GetByHashes(ctx context.Context, hashes []string) ([]*Chunk, error) {
// GetByHashes returns the chunks whose hashes appear in hashes, ordered by
// chunk hash. Unknown hashes are silently omitted from the result.
func (r *ChunkRepository) GetByHashes(
ctx context.Context, hashes []string,
) ([]*Chunk, error) {
if len(hashes) == 0 {
return nil, nil
}
@@ -71,14 +84,22 @@ func (r *ChunkRepository) GetByHashes(ctx context.Context, hashes []string) ([]*
FROM chunks
WHERE chunk_hash IN (`
args := make([]interface{}, len(hashes))
args := make([]any, len(hashes))
var querySb75 strings.Builder
for i, hash := range hashes {
if i > 0 {
query += ", "
querySb75.WriteString(", ")
}
query += "?"
querySb75.WriteString("?")
args[i] = hash
}
query += querySb75.String() //nolint:gosec // G202: appends "?" placeholders only
query += ") ORDER BY chunk_hash"
rows, err := r.db.conn.QueryContext(ctx, query, args...)
@@ -88,6 +109,7 @@ func (r *ChunkRepository) GetByHashes(ctx context.Context, hashes []string) ([]*
defer CloseRows(rows)
var chunks []*Chunk
for rows.Next() {
var chunk Chunk
@@ -105,7 +127,11 @@ func (r *ChunkRepository) GetByHashes(ctx context.Context, hashes []string) ([]*
return chunks, rows.Err()
}
func (r *ChunkRepository) ListUnpacked(ctx context.Context, limit int) ([]*Chunk, error) {
// ListUnpacked returns up to limit chunks that are not yet stored in any
// blob, ordered by chunk hash.
func (r *ChunkRepository) ListUnpacked(
ctx context.Context, limit int,
) ([]*Chunk, error) {
query := `
SELECT c.chunk_hash, c.size
FROM chunks c
@@ -122,6 +148,7 @@ func (r *ChunkRepository) ListUnpacked(ctx context.Context, limit int) ([]*Chunk
defer CloseRows(rows)
var chunks []*Chunk
for rows.Next() {
var chunk Chunk

View File

@@ -5,6 +5,7 @@ import (
"fmt"
)
// List returns every chunk in the index, ordered by chunk hash.
func (r *ChunkRepository) List(ctx context.Context) ([]*Chunk, error) {
query := `
SELECT chunk_hash, size
@@ -19,6 +20,7 @@ func (r *ChunkRepository) List(ctx context.Context) ([]*Chunk, error) {
defer CloseRows(rows)
var chunks []*Chunk
for rows.Next() {
var chunk Chunk

View File

@@ -1,21 +1,24 @@
package database
package database_test
import (
"context"
"testing"
"sneak.berlin/go/vaultik/internal/database"
"sneak.berlin/go/vaultik/internal/types"
)
func TestChunkRepository(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repo := NewChunkRepository(db)
repo := database.NewChunkRepository(db)
// Test Create
chunk := &Chunk{
chunk := &database.Chunk{
ChunkHash: types.ChunkHash("chunkhash123"),
Size: 4096,
}
@@ -30,12 +33,15 @@ func TestChunkRepository(t *testing.T) {
if err != nil {
t.Fatalf("failed to get chunk: %v", err)
}
if retrieved == nil {
t.Fatal("expected chunk, got nil")
}
if retrieved.ChunkHash != chunk.ChunkHash {
t.Errorf("chunk hash mismatch: got %s, want %s", retrieved.ChunkHash, chunk.ChunkHash)
}
if retrieved.Size != chunk.Size {
t.Errorf("size mismatch: got %d, want %d", retrieved.Size, chunk.Size)
}
@@ -47,19 +53,23 @@ func TestChunkRepository(t *testing.T) {
}
// Test GetByHashes
chunk2 := &Chunk{
chunk2 := &database.Chunk{
ChunkHash: types.ChunkHash("chunkhash456"),
Size: 8192,
}
err = repo.Create(ctx, nil, chunk2)
if err != nil {
t.Fatalf("failed to create second chunk: %v", err)
}
chunks, err := repo.GetByHashes(ctx, []string{chunk.ChunkHash.String(), chunk2.ChunkHash.String()})
chunks, err := repo.GetByHashes(ctx, []string{
chunk.ChunkHash.String(), chunk2.ChunkHash.String(),
})
if err != nil {
t.Fatalf("failed to get chunks by hashes: %v", err)
}
if len(chunks) != 2 {
t.Errorf("expected 2 chunks, got %d", len(chunks))
}
@@ -69,23 +79,27 @@ func TestChunkRepository(t *testing.T) {
if err != nil {
t.Fatalf("failed to list unpacked chunks: %v", err)
}
if len(unpacked) != 2 {
t.Errorf("expected 2 unpacked chunks, got %d", len(unpacked))
}
}
func TestChunkRepositoryNotFound(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repo := NewChunkRepository(db)
repo := database.NewChunkRepository(db)
// Test GetByHash with non-existent hash
chunk, err := repo.GetByHash(ctx, "nonexistent")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if chunk != nil {
t.Error("expected nil for non-existent chunk")
}
@@ -95,6 +109,7 @@ func TestChunkRepositoryNotFound(t *testing.T) {
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if chunks != nil {
t.Error("expected nil for empty hash list")
}

View File

@@ -15,6 +15,7 @@ import (
"context"
"database/sql"
"embed"
"errors"
"fmt"
"os"
"path/filepath"
@@ -22,10 +23,15 @@ import (
"strconv"
"strings"
// Register the pure-Go sqlite driver.
_ "modernc.org/sqlite"
"sneak.berlin/go/vaultik/internal/log"
)
// errInvalidMigrationFilename is returned when an embedded migration file
// does not follow the "<version>[_<description>].sql" naming pattern.
var errInvalidMigrationFilename = errors.New("invalid migration filename")
//go:embed schema/*.sql
var schemaFS embed.FS
@@ -51,26 +57,28 @@ type DB struct {
func ParseMigrationVersion(filename string) (int, error) {
name := strings.TrimSuffix(filename, filepath.Ext(filename))
if name == "" {
return 0, fmt.Errorf("invalid migration filename %q: empty name", filename)
return 0, fmt.Errorf("%w %q: empty name", errInvalidMigrationFilename, filename)
}
// Split on underscore to separate version from description.
// If there's no underscore, the entire stem is the version.
versionStr := name
if idx := strings.IndexByte(name, '_'); idx >= 0 {
versionStr = name[:idx]
if before, _, ok := strings.Cut(name, "_"); ok {
versionStr = before
}
if versionStr == "" {
return 0, fmt.Errorf("invalid migration filename %q: empty version prefix", filename)
return 0, fmt.Errorf(
"%w %q: empty version prefix", errInvalidMigrationFilename, filename,
)
}
// Validate the version is purely numeric.
for _, ch := range versionStr {
if ch < '0' || ch > '9' {
return 0, fmt.Errorf(
"invalid migration filename %q: version %q contains non-numeric character %q",
filename, versionStr, string(ch),
"%w %q: version %q contains non-numeric character %q",
errInvalidMigrationFilename, filename, versionStr, string(ch),
)
}
}
@@ -98,61 +106,93 @@ func New(ctx context.Context, path string) (*DB, error) {
// First attempt with standard WAL mode
log.Debug("Attempting to open database with WAL mode", "path", path)
conn, err := sql.Open(
"sqlite",
path+"?_journal_mode=WAL&_synchronous=NORMAL&_busy_timeout=10000&_locking_mode=NORMAL&_foreign_keys=ON",
path+"?_journal_mode=WAL&_synchronous=NORMAL&_busy_timeout=10000"+
"&_locking_mode=NORMAL&_foreign_keys=ON",
)
if err == nil {
// Set connection pool settings
// SQLite can handle multiple readers but only one writer at a time.
// Setting MaxOpenConns to 1 ensures all writes are serialized through
// a single connection, preventing SQLITE_BUSY errors.
conn.SetMaxOpenConns(1)
conn.SetMaxIdleConns(1)
configureConnPool(conn)
if err := conn.PingContext(ctx); err == nil {
err = conn.PingContext(ctx)
if err == nil {
// Success on first try
log.Debug("Database opened successfully with WAL mode", "path", path)
// Enable foreign keys explicitly
if _, err := conn.ExecContext(ctx, "PRAGMA foreign_keys = ON"); err != nil {
log.Warn("Failed to enable foreign keys", "error", err)
return finishOpen(ctx, conn, path)
}
db := &DB{conn: conn, path: path}
if err := applyMigrations(ctx, conn); err != nil {
_ = conn.Close()
return nil, fmt.Errorf("applying migrations: %w", err)
}
return db, nil
}
log.Debug("Failed to ping database, closing connection", "path", path, "error", err)
log.Debug(
"Failed to ping database, closing connection",
"path", path, "error", err,
)
_ = conn.Close()
}
// If first attempt failed, try with TRUNCATE mode to clear any locks
return openWithRecovery(ctx, path)
}
// configureConnPool serializes all database access through one connection.
// SQLite can handle multiple readers but only one writer at a time; setting
// MaxOpenConns to 1 ensures all writes go through a single connection,
// preventing SQLITE_BUSY errors.
func configureConnPool(conn *sql.DB) {
conn.SetMaxOpenConns(1)
conn.SetMaxIdleConns(1)
}
// finishOpen enables foreign keys, wraps the connection, and applies any
// pending migrations. On migration failure the connection is closed.
func finishOpen(ctx context.Context, conn *sql.DB, path string) (*DB, error) {
// Enable foreign keys explicitly
_, err := conn.ExecContext(ctx, "PRAGMA foreign_keys = ON")
if err != nil {
log.Warn("Failed to enable foreign keys", "path", path, "error", err)
}
db := &DB{conn: conn, path: path}
err = applyMigrations(ctx, conn)
if err != nil {
_ = conn.Close()
return nil, fmt.Errorf("applying migrations: %w", err)
}
return db, nil
}
// openWithRecovery retries opening the database in TRUNCATE journal mode to
// clear stale locks, then switches back to WAL mode.
func openWithRecovery(ctx context.Context, path string) (*DB, error) {
log.Info(
"Database appears locked, attempting recovery with TRUNCATE mode",
"path", path,
)
conn, err = sql.Open(
conn, err := sql.Open(
"sqlite",
path+"?_journal_mode=TRUNCATE&_synchronous=NORMAL&_busy_timeout=10000&_foreign_keys=ON",
path+"?_journal_mode=TRUNCATE&_synchronous=NORMAL&_busy_timeout=10000"+
"&_foreign_keys=ON",
)
if err != nil {
return nil, fmt.Errorf("opening database in recovery mode: %w", err)
}
// Set connection pool settings
// SQLite can handle multiple readers but only one writer at a time.
// Setting MaxOpenConns to 1 ensures all writes are serialized through
// a single connection, preventing SQLITE_BUSY errors.
conn.SetMaxOpenConns(1)
conn.SetMaxIdleConns(1)
configureConnPool(conn)
err = conn.PingContext(ctx)
if err != nil {
log.Debug(
"Failed to ping database in recovery mode, closing",
"path", path, "error", err,
)
if err := conn.PingContext(ctx); err != nil {
log.Debug("Failed to ping database in recovery mode, closing", "path", path, "error", err)
_ = conn.Close()
return nil, fmt.Errorf(
"database still locked after recovery attempt: %w",
err,
@@ -163,35 +203,44 @@ func New(ctx context.Context, path string) (*DB, error) {
// Switch back to WAL mode
log.Debug("Switching database back to WAL mode", "path", path)
if _, err := conn.ExecContext(ctx, "PRAGMA journal_mode=WAL"); err != nil {
_, err = conn.ExecContext(ctx, "PRAGMA journal_mode=WAL")
if err != nil {
log.Warn("Failed to switch back to WAL mode", "path", path, "error", err)
}
// Ensure foreign keys are enabled
if _, err := conn.ExecContext(ctx, "PRAGMA foreign_keys=ON"); err != nil {
log.Warn("Failed to enable foreign keys", "path", path, "error", err)
}
db := &DB{conn: conn, path: path}
if err := applyMigrations(ctx, conn); err != nil {
_ = conn.Close()
return nil, fmt.Errorf("applying migrations: %w", err)
db, err := finishOpen(ctx, conn, path)
if err != nil {
return nil, err
}
log.Debug("Database connection established successfully", "path", path)
return db, nil
}
// NewTestDB creates an in-memory SQLite database for testing purposes.
// The database is automatically initialized with the schema and is ready
// for use. Each call creates a new independent database instance.
func NewTestDB() (*DB, error) {
return New(context.Background(), ":memory:")
}
// Close closes the database connection.
// It ensures all pending operations are completed before closing.
// Returns an error if the database connection cannot be closed properly.
func (db *DB) Close() error {
log.Debug("Closing database connection", "path", db.path)
if err := db.conn.Close(); err != nil {
err := db.conn.Close()
if err != nil {
log.Error("Failed to close database", "path", db.path, "error", err)
return fmt.Errorf("failed to close database: %w", err)
}
log.Debug("Database connection closed successfully", "path", db.path)
return nil
}
@@ -227,22 +276,25 @@ func (db *DB) BeginTx(
func (db *DB) ExecWithLog(
ctx context.Context,
query string,
args ...interface{},
args ...any,
) (sql.Result, error) {
LogSQL("Execute", query, args...)
return db.conn.ExecContext(ctx, query, args...)
}
// QueryRowWithLog executes a query that returns at most one row with SQL logging.
// This is useful for queries that modify data and return values (e.g., INSERT ... RETURNING).
// SQLite handles its own locking internally.
// The query and args parameters follow the same format as sql.DB.QueryRowContext.
// QueryRowWithLog executes a query that returns at most one row with SQL
// logging. This is useful for queries that modify data and return values
// (e.g., INSERT ... RETURNING). SQLite handles its own locking internally.
// The query and args parameters follow the same format as
// sql.DB.QueryRowContext.
func (db *DB) QueryRowWithLog(
ctx context.Context,
query string,
args ...interface{},
args ...any,
) *sql.Row {
LogSQL("QueryRow", query, args...)
return db.conn.QueryRowContext(ctx, query, args...)
}
@@ -302,7 +354,8 @@ func bootstrapMigrationsTable(ctx context.Context, db *sql.DB) error {
// the schema_migrations table via 000.sql, then iterates through remaining
// migration files in order.
func applyMigrations(ctx context.Context, db *sql.DB) error {
if err := bootstrapMigrationsTable(ctx, db); err != nil {
err := bootstrapMigrationsTable(ctx, db)
if err != nil {
return err
}
@@ -362,30 +415,26 @@ func applyMigrations(ctx context.Context, db *sql.DB) error {
return nil
}
// NewTestDB creates an in-memory SQLite database for testing purposes.
// The database is automatically initialized with the schema and is ready for use.
// Each call creates a new independent database instance.
func NewTestDB() (*DB, error) {
return New(context.Background(), ":memory:")
}
// repeatPlaceholder generates a string of ", ?" repeated n times for IN clause construction.
// For example, repeatPlaceholder(2) returns ", ?, ?".
// repeatPlaceholder generates a string of ", ?" repeated n times for IN
// clause construction. For example, repeatPlaceholder(2) returns ", ?, ?".
func repeatPlaceholder(n int) string {
if n <= 0 {
return ""
}
return strings.Repeat(", ?", n)
}
// LogSQL logs SQL queries and their arguments when debug mode is enabled.
// Debug mode is activated by setting the GODEBUG environment variable to include "vaultik".
// This is useful for troubleshooting database operations and understanding query patterns.
// Debug mode is activated by setting the GODEBUG environment variable to
// include "vaultik". This is useful for troubleshooting database operations
// and understanding query patterns.
//
// The operation parameter describes the type of SQL operation (e.g., "Execute", "Query").
// The query parameter is the SQL statement being executed.
// The args parameter contains the query arguments that will be interpolated.
func LogSQL(operation, query string, args ...interface{}) {
// The operation parameter describes the type of SQL operation (e.g.,
// "Execute", "Query"). The query parameter is the SQL statement being
// executed. The args parameter contains the query arguments that will be
// interpolated.
func LogSQL(operation, query string, args ...any) {
if strings.Contains(os.Getenv("GODEBUG"), "vaultik") {
log.Debug(
"SQL "+operation,

View File

@@ -1,3 +1,4 @@
//nolint:testpackage // exercises unexported migration internals
package database
import (
@@ -9,6 +10,8 @@ import (
)
func TestDatabase(t *testing.T) {
t.Parallel()
ctx := context.Background()
dbPath := filepath.Join(t.TempDir(), "test.db")
@@ -17,7 +20,8 @@ func TestDatabase(t *testing.T) {
t.Fatalf("failed to create database: %v", err)
}
defer func() {
if err := db.Close(); err != nil {
err := db.Close()
if err != nil {
t.Errorf("failed to close database: %v", err)
}
}()
@@ -37,7 +41,10 @@ func TestDatabase(t *testing.T) {
for _, table := range tables {
var name string
err := db.conn.QueryRow("SELECT name FROM sqlite_master WHERE type='table' AND name=?", table).Scan(&name)
err := db.conn.QueryRowContext(ctx,
"SELECT name FROM sqlite_master WHERE type='table' AND name=?", table,
).Scan(&name)
if err != nil {
t.Errorf("table %s does not exist: %v", table, err)
}
@@ -45,6 +52,8 @@ func TestDatabase(t *testing.T) {
}
func TestDatabaseInvalidPath(t *testing.T) {
t.Parallel()
ctx := context.Background()
// Test with invalid path
@@ -55,6 +64,8 @@ func TestDatabaseInvalidPath(t *testing.T) {
}
func TestDatabaseConcurrentAccess(t *testing.T) {
t.Parallel()
ctx := context.Background()
dbPath := filepath.Join(t.TempDir(), "test.db")
@@ -63,7 +74,8 @@ func TestDatabaseConcurrentAccess(t *testing.T) {
t.Fatalf("failed to create database: %v", err)
}
defer func() {
if err := db.Close(); err != nil {
err := db.Close()
if err != nil {
t.Errorf("failed to close database: %v", err)
}
}()
@@ -73,18 +85,20 @@ func TestDatabaseConcurrentAccess(t *testing.T) {
index int
err error
}
results := make(chan result, 10)
for i := 0; i < 10; i++ {
for i := range 10 {
go func(i int) {
_, err := db.ExecWithLog(ctx, "INSERT INTO chunks (chunk_hash, size) VALUES (?, ?)",
_, err := db.ExecWithLog(ctx,
"INSERT INTO chunks (chunk_hash, size) VALUES (?, ?)",
fmt.Sprintf("hash%d", i), i*1024)
results <- result{index: i, err: err}
}(i)
}
// Wait for all goroutines and check results
for i := 0; i < 10; i++ {
for range 10 {
r := <-results
if r.err != nil {
t.Fatalf("concurrent insert %d failed: %v", r.index, r.err)
@@ -93,16 +107,20 @@ func TestDatabaseConcurrentAccess(t *testing.T) {
// Verify all inserts succeeded
var count int
err = db.conn.QueryRowContext(ctx, "SELECT COUNT(*) FROM chunks").Scan(&count)
if err != nil {
t.Fatalf("failed to count chunks: %v", err)
}
if count != 10 {
t.Errorf("expected 10 chunks, got %d", count)
}
}
func TestParseMigrationVersion(t *testing.T) {
t.Parallel()
tests := []struct {
name string
filename string
@@ -112,8 +130,14 @@ func TestParseMigrationVersion(t *testing.T) {
{name: "valid 000.sql", filename: "000.sql", wantVer: 0, wantError: false},
{name: "valid 001.sql", filename: "001.sql", wantVer: 1, wantError: false},
{name: "valid 099.sql", filename: "099.sql", wantVer: 99, wantError: false},
{name: "valid with description", filename: "001_initial_schema.sql", wantVer: 1, wantError: false},
{name: "valid large version", filename: "123_big_migration.sql", wantVer: 123, wantError: false},
{
name: "valid with description", filename: "001_initial_schema.sql",
wantVer: 1, wantError: false,
},
{
name: "valid large version", filename: "123_big_migration.sql",
wantVer: 123, wantError: false,
},
{name: "invalid alpha version", filename: "abc.sql", wantVer: 0, wantError: true},
{name: "invalid mixed chars", filename: "12a.sql", wantVer: 0, wantError: true},
{name: "invalid no extension", filename: "schema.sql", wantVer: 0, wantError: true},
@@ -122,25 +146,36 @@ func TestParseMigrationVersion(t *testing.T) {
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
got, err := ParseMigrationVersion(tc.filename)
if tc.wantError {
if err == nil {
t.Errorf("ParseMigrationVersion(%q) = %d, nil; want error", tc.filename, got)
t.Errorf("ParseMigrationVersion(%q) = %d, nil; want error",
tc.filename, got)
}
return
}
if err != nil {
t.Errorf("ParseMigrationVersion(%q) unexpected error: %v", tc.filename, err)
t.Errorf("ParseMigrationVersion(%q) unexpected error: %v",
tc.filename, err)
return
}
if got != tc.wantVer {
t.Errorf("ParseMigrationVersion(%q) = %d; want %d", tc.filename, got, tc.wantVer)
t.Errorf("ParseMigrationVersion(%q) = %d; want %d",
tc.filename, got, tc.wantVer)
}
})
}
}
func TestApplyMigrations_Idempotent(t *testing.T) {
t.Parallel()
ctx := context.Background()
conn, err := sql.Open("sqlite", ":memory:?_foreign_keys=ON")
@@ -148,7 +183,8 @@ func TestApplyMigrations_Idempotent(t *testing.T) {
t.Fatalf("failed to open database: %v", err)
}
defer func() {
if err := conn.Close(); err != nil {
err := conn.Close()
if err != nil {
t.Errorf("failed to close database: %v", err)
}
}()
@@ -157,33 +193,46 @@ func TestApplyMigrations_Idempotent(t *testing.T) {
conn.SetMaxIdleConns(1)
// First run: apply all migrations.
if err := applyMigrations(ctx, conn); err != nil {
err = applyMigrations(ctx, conn)
if err != nil {
t.Fatalf("first applyMigrations failed: %v", err)
}
// Count rows in schema_migrations after first run.
var countBefore int
if err := conn.QueryRowContext(ctx, "SELECT COUNT(*) FROM schema_migrations").Scan(&countBefore); err != nil {
err = conn.QueryRowContext(ctx,
"SELECT COUNT(*) FROM schema_migrations",
).Scan(&countBefore)
if err != nil {
t.Fatalf("failed to count schema_migrations after first run: %v", err)
}
// Second run: must be a no-op.
if err := applyMigrations(ctx, conn); err != nil {
err = applyMigrations(ctx, conn)
if err != nil {
t.Fatalf("second applyMigrations failed: %v", err)
}
// Count rows in schema_migrations after second run — must be unchanged.
var countAfter int
if err := conn.QueryRowContext(ctx, "SELECT COUNT(*) FROM schema_migrations").Scan(&countAfter); err != nil {
err = conn.QueryRowContext(ctx,
"SELECT COUNT(*) FROM schema_migrations",
).Scan(&countAfter)
if err != nil {
t.Fatalf("failed to count schema_migrations after second run: %v", err)
}
if countBefore != countAfter {
t.Errorf("schema_migrations row count changed: before=%d, after=%d", countBefore, countAfter)
t.Errorf("schema_migrations row count changed: before=%d, after=%d",
countBefore, countAfter)
}
}
func TestBootstrapMigrationsTable_FreshDatabase(t *testing.T) {
t.Parallel()
ctx := context.Background()
conn, err := sql.Open("sqlite", ":memory:?_foreign_keys=ON")
@@ -191,7 +240,8 @@ func TestBootstrapMigrationsTable_FreshDatabase(t *testing.T) {
t.Fatalf("failed to open database: %v", err)
}
defer func() {
if err := conn.Close(); err != nil {
err := conn.Close()
if err != nil {
t.Errorf("failed to close database: %v", err)
}
}()
@@ -201,38 +251,49 @@ func TestBootstrapMigrationsTable_FreshDatabase(t *testing.T) {
// Verify schema_migrations does NOT exist yet.
var tableBefore int
if err := conn.QueryRowContext(ctx,
err = conn.QueryRowContext(ctx,
"SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='schema_migrations'",
).Scan(&tableBefore); err != nil {
).Scan(&tableBefore)
if err != nil {
t.Fatalf("failed to check for table before bootstrap: %v", err)
}
if tableBefore != 0 {
t.Fatal("schema_migrations table should not exist before bootstrap")
}
// Run bootstrap.
if err := bootstrapMigrationsTable(ctx, conn); err != nil {
err = bootstrapMigrationsTable(ctx, conn)
if err != nil {
t.Fatalf("bootstrapMigrationsTable failed: %v", err)
}
// Verify schema_migrations now exists.
var tableAfter int
if err := conn.QueryRowContext(ctx,
err = conn.QueryRowContext(ctx,
"SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='schema_migrations'",
).Scan(&tableAfter); err != nil {
).Scan(&tableAfter)
if err != nil {
t.Fatalf("failed to check for table after bootstrap: %v", err)
}
if tableAfter != 1 {
t.Fatalf("schema_migrations table should exist after bootstrap, got count=%d", tableAfter)
t.Fatalf("schema_migrations table should exist after bootstrap, got count=%d",
tableAfter)
}
// Verify version 0 row exists.
var version int
if err := conn.QueryRowContext(ctx,
err = conn.QueryRowContext(ctx,
"SELECT version FROM schema_migrations WHERE version = 0",
).Scan(&version); err != nil {
).Scan(&version)
if err != nil {
t.Fatalf("version 0 row not found in schema_migrations: %v", err)
}
if version != 0 {
t.Errorf("expected version 0, got %d", version)
}

View File

@@ -6,15 +6,16 @@ import (
"os"
)
// Fatal prints an error message to stderr and exits with status 1
func Fatal(format string, args ...interface{}) {
// Fatalf prints an error message to stderr and exits with status 1
func Fatalf(format string, args ...any) {
fmt.Fprintf(os.Stderr, "FATAL: "+format+"\n", args...)
os.Exit(1)
}
// CloseRows closes rows and exits on error
func CloseRows(rows *sql.Rows) {
if err := rows.Close(); err != nil {
Fatal("failed to close rows: %v", err)
err := rows.Close()
if err != nil {
Fatalf("failed to close rows: %v", err)
}
}

View File

@@ -4,19 +4,26 @@ import (
"context"
"database/sql"
"fmt"
"strings"
"sneak.berlin/go/vaultik/internal/types"
)
// FileChunkRepository provides access to the file_chunks table, which maps
// files to their ordered constituent chunks.
type FileChunkRepository struct {
db *DB
}
// NewFileChunkRepository creates a FileChunkRepository backed by db.
func NewFileChunkRepository(db *DB) *FileChunkRepository {
return &FileChunkRepository{db: db}
}
func (r *FileChunkRepository) Create(ctx context.Context, tx *sql.Tx, fc *FileChunk) error {
// Create inserts a file_chunks row (idempotently), using tx when non-nil.
func (r *FileChunkRepository) Create(
ctx context.Context, tx *sql.Tx, fc *FileChunk,
) error {
query := `
INSERT INTO file_chunks (file_id, idx, chunk_hash)
VALUES (?, ?, ?)
@@ -27,7 +34,8 @@ func (r *FileChunkRepository) Create(ctx context.Context, tx *sql.Tx, fc *FileCh
if tx != nil {
_, err = tx.ExecContext(ctx, query, fc.FileID.String(), fc.Idx, fc.ChunkHash.String())
} else {
_, err = r.db.ExecWithLog(ctx, query, fc.FileID.String(), fc.Idx, fc.ChunkHash.String())
_, err = r.db.ExecWithLog(ctx, query,
fc.FileID.String(), fc.Idx, fc.ChunkHash.String())
}
if err != nil {
@@ -37,7 +45,10 @@ func (r *FileChunkRepository) Create(ctx context.Context, tx *sql.Tx, fc *FileCh
return nil
}
func (r *FileChunkRepository) GetByPath(ctx context.Context, path string) ([]*FileChunk, error) {
// GetByPath returns the ordered chunks of the file at the given path.
func (r *FileChunkRepository) GetByPath(
ctx context.Context, path string,
) ([]*FileChunk, error) {
query := `
SELECT fc.file_id, fc.idx, fc.chunk_hash
FROM file_chunks fc
@@ -56,7 +67,9 @@ func (r *FileChunkRepository) GetByPath(ctx context.Context, path string) ([]*Fi
}
// GetByFileID retrieves file chunks by file ID
func (r *FileChunkRepository) GetByFileID(ctx context.Context, fileID types.FileID) ([]*FileChunk, error) {
func (r *FileChunkRepository) GetByFileID(
ctx context.Context, fileID types.FileID,
) ([]*FileChunk, error) {
query := `
SELECT file_id, idx, chunk_hash
FROM file_chunks
@@ -74,7 +87,9 @@ func (r *FileChunkRepository) GetByFileID(ctx context.Context, fileID types.File
}
// GetByPathTx retrieves file chunks within a transaction
func (r *FileChunkRepository) GetByPathTx(ctx context.Context, tx *sql.Tx, path string) ([]*FileChunk, error) {
func (r *FileChunkRepository) GetByPathTx(
ctx context.Context, tx *sql.Tx, path string,
) ([]*FileChunk, error) {
query := `
SELECT fc.file_id, fc.idx, fc.chunk_hash
FROM file_chunks fc
@@ -84,6 +99,7 @@ func (r *FileChunkRepository) GetByPathTx(ctx context.Context, tx *sql.Tx, path
`
LogSQL("GetByPathTx", query, path)
rows, err := tx.QueryContext(ctx, query, path)
if err != nil {
return nil, fmt.Errorf("querying file chunks: %w", err)
@@ -92,32 +108,18 @@ func (r *FileChunkRepository) GetByPathTx(ctx context.Context, tx *sql.Tx, path
fileChunks, err := r.scanFileChunks(rows)
LogSQL("GetByPathTx", "Complete", path, "count", len(fileChunks))
return fileChunks, err
}
// scanFileChunks is a helper that scans file chunk rows
func (r *FileChunkRepository) scanFileChunks(rows *sql.Rows) ([]*FileChunk, error) {
var fileChunks []*FileChunk
for rows.Next() {
var fc FileChunk
var fileIDStr, chunkHashStr string
err := rows.Scan(&fileIDStr, &fc.Idx, &chunkHashStr)
if err != nil {
return nil, fmt.Errorf("scanning file chunk: %w", err)
}
fc.FileID, err = types.ParseFileID(fileIDStr)
if err != nil {
return nil, fmt.Errorf("parsing file ID: %w", err)
}
fc.ChunkHash = types.ChunkHash(chunkHashStr)
fileChunks = append(fileChunks, &fc)
}
return fileChunks, rows.Err()
}
func (r *FileChunkRepository) DeleteByPath(ctx context.Context, tx *sql.Tx, path string) error {
query := `DELETE FROM file_chunks WHERE file_id = (SELECT id FROM files WHERE path = ?)`
// DeleteByPath deletes all file_chunks rows for the file at the given path.
func (r *FileChunkRepository) DeleteByPath(
ctx context.Context, tx *sql.Tx, path string,
) error {
query := `
DELETE FROM file_chunks
WHERE file_id = (SELECT id FROM files WHERE path = ?)
`
var err error
if tx != nil {
@@ -134,7 +136,9 @@ func (r *FileChunkRepository) DeleteByPath(ctx context.Context, tx *sql.Tx, path
}
// DeleteByFileID deletes all chunks for a file by its UUID
func (r *FileChunkRepository) DeleteByFileID(ctx context.Context, tx *sql.Tx, fileID types.FileID) error {
func (r *FileChunkRepository) DeleteByFileID(
ctx context.Context, tx *sql.Tx, fileID types.FileID,
) error {
query := `DELETE FROM file_chunks WHERE file_id = ?`
var err error
@@ -152,7 +156,11 @@ func (r *FileChunkRepository) DeleteByFileID(ctx context.Context, tx *sql.Tx, fi
}
// DeleteByFileIDs deletes all chunks for multiple files in a single statement.
func (r *FileChunkRepository) DeleteByFileIDs(ctx context.Context, tx *sql.Tx, fileIDs []types.FileID) error {
//
//nolint:dupl // symmetric implementation for a parallel association table
func (r *FileChunkRepository) DeleteByFileIDs(
ctx context.Context, tx *sql.Tx, fileIDs []types.FileID,
) error {
if len(fileIDs) == 0 {
return nil
}
@@ -161,14 +169,15 @@ func (r *FileChunkRepository) DeleteByFileIDs(ctx context.Context, tx *sql.Tx, f
const batchSize = 500
for i := 0; i < len(fileIDs); i += batchSize {
end := i + batchSize
if end > len(fileIDs) {
end = len(fileIDs)
}
end := min(i+batchSize, len(fileIDs))
batch := fileIDs[i:end]
query := "DELETE FROM file_chunks WHERE file_id IN (?" + repeatPlaceholder(len(batch)-1) + ")"
args := make([]interface{}, len(batch))
//nolint:gosec // G202: concatenates constant SQL and "?" placeholders only
query := "DELETE FROM file_chunks WHERE file_id IN (?" +
repeatPlaceholder(len(batch)-1) + ")"
args := make([]any, len(batch))
for j, id := range batch {
args[j] = id.String()
}
@@ -179,6 +188,7 @@ func (r *FileChunkRepository) DeleteByFileIDs(ctx context.Context, tx *sql.Tx, f
} else {
_, err = r.db.ExecWithLog(ctx, query, args...)
}
if err != nil {
return fmt.Errorf("batch deleting file_chunks: %w", err)
}
@@ -189,32 +199,44 @@ func (r *FileChunkRepository) DeleteByFileIDs(ctx context.Context, tx *sql.Tx, f
// CreateBatch inserts multiple file_chunks in a single statement for efficiency.
// Batches are automatically split to stay within SQLite's variable limit.
func (r *FileChunkRepository) CreateBatch(ctx context.Context, tx *sql.Tx, fcs []FileChunk) error {
func (r *FileChunkRepository) CreateBatch(
ctx context.Context, tx *sql.Tx, fcs []FileChunk,
) error {
if len(fcs) == 0 {
return nil
}
// SQLite has a limit on variables (typically 999 or 32766).
// Each FileChunk has 3 values, so batch at 300 to be safe.
// Each file_chunks row binds this many SQL variables.
const fileChunkCols = 3
// SQLite has a limit on variables (typically 999 or 32766), so batch
// at 300 rows to be safe.
const batchSize = 300
for i := 0; i < len(fcs); i += batchSize {
end := i + batchSize
if end > len(fcs) {
end = len(fcs)
}
end := min(i+batchSize, len(fcs))
batch := fcs[i:end]
// Build the query with multiple value sets
query := "INSERT INTO file_chunks (file_id, idx, chunk_hash) VALUES "
args := make([]interface{}, 0, len(batch)*3)
args := make([]any, 0, len(batch)*fileChunkCols)
var querySb211 strings.Builder
for j, fc := range batch {
if j > 0 {
query += ", "
querySb211.WriteString(", ")
}
query += "(?, ?, ?)"
querySb211.WriteString("(?, ?, ?)")
args = append(args, fc.FileID.String(), fc.Idx, fc.ChunkHash.String())
}
query += querySb211.String() //nolint:gosec // G202: appends "?" placeholders only
query += " ON CONFLICT(file_id, idx) DO NOTHING"
var err error
@@ -223,6 +245,7 @@ func (r *FileChunkRepository) CreateBatch(ctx context.Context, tx *sql.Tx, fcs [
} else {
_, err = r.db.ExecWithLog(ctx, query, args...)
}
if err != nil {
return fmt.Errorf("batch inserting file_chunks: %w", err)
}
@@ -232,17 +255,50 @@ func (r *FileChunkRepository) CreateBatch(ctx context.Context, tx *sql.Tx, fcs [
}
// GetByFile is an alias for GetByPath for compatibility
func (r *FileChunkRepository) GetByFile(ctx context.Context, path string) ([]*FileChunk, error) {
func (r *FileChunkRepository) GetByFile(
ctx context.Context, path string,
) ([]*FileChunk, error) {
LogSQL("GetByFile", "Starting", path)
result, err := r.GetByPath(ctx, path)
LogSQL("GetByFile", "Complete", path, "count", len(result))
return result, err
}
// GetByFileTx retrieves file chunks within a transaction
func (r *FileChunkRepository) GetByFileTx(ctx context.Context, tx *sql.Tx, path string) ([]*FileChunk, error) {
func (r *FileChunkRepository) GetByFileTx(
ctx context.Context, tx *sql.Tx, path string,
) ([]*FileChunk, error) {
LogSQL("GetByFileTx", "Starting", path)
result, err := r.GetByPathTx(ctx, tx, path)
LogSQL("GetByFileTx", "Complete", path, "count", len(result))
return result, err
}
// scanFileChunks is a helper that scans file chunk rows
func (r *FileChunkRepository) scanFileChunks(rows *sql.Rows) ([]*FileChunk, error) {
var fileChunks []*FileChunk
for rows.Next() {
var (
fc FileChunk
fileIDStr, chunkHashStr string
)
err := rows.Scan(&fileIDStr, &fc.Idx, &chunkHashStr)
if err != nil {
return nil, fmt.Errorf("scanning file chunk: %w", err)
}
fc.FileID, err = types.ParseFileID(fileIDStr)
if err != nil {
return nil, fmt.Errorf("parsing file ID: %w", err)
}
fc.ChunkHash = types.ChunkHash(chunkHashStr)
fileChunks = append(fileChunks, &fc)
}
return fileChunks, rows.Err()
}

View File

@@ -1,4 +1,4 @@
package database
package database_test
import (
"context"
@@ -6,21 +6,25 @@ import (
"testing"
"time"
"sneak.berlin/go/vaultik/internal/database"
"sneak.berlin/go/vaultik/internal/types"
)
func TestFileChunkRepository(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repo := NewFileChunkRepository(db)
fileRepo := NewFileRepository(db)
repo := database.NewFileChunkRepository(db)
fileRepo := database.NewFileRepository(db)
repos := database.NewRepositories(db)
// Create test file first
testTime := time.Now().Truncate(time.Second)
file := &File{
Path: "/test/file.txt",
file := &database.File{
Path: testFileTxt,
MTime: testTime,
Size: 3072,
Mode: 0644,
@@ -28,63 +32,51 @@ func TestFileChunkRepository(t *testing.T) {
GID: 1000,
LinkTarget: "",
}
err := fileRepo.Create(ctx, nil, file)
if err != nil {
t.Fatalf("failed to create file: %v", err)
}
// Create chunks first
chunks := []types.ChunkHash{"chunk1", "chunk2", "chunk3"}
chunkRepo := NewChunkRepository(db)
for _, chunkHash := range chunks {
chunk := &Chunk{
ChunkHash: chunkHash,
Size: 1024,
}
err = chunkRepo.Create(ctx, nil, chunk)
if err != nil {
t.Fatalf("failed to create chunk %s: %v", chunkHash, err)
}
}
mustCreateFile(t, fileRepo, file)
mustCreateChunks(t, repos, chunk1Hash, chunk2Hash, chunk3Hash)
// Test Create
fc1 := &FileChunk{
fc1 := &database.FileChunk{
FileID: file.ID,
Idx: 0,
ChunkHash: types.ChunkHash("chunk1"),
ChunkHash: types.ChunkHash(chunk1Hash),
}
err = repo.Create(ctx, nil, fc1)
err := repo.Create(ctx, nil, fc1)
if err != nil {
t.Fatalf("failed to create file chunk: %v", err)
}
// Add more chunks for the same file
fc2 := &FileChunk{
fc2 := &database.FileChunk{
FileID: file.ID,
Idx: 1,
ChunkHash: types.ChunkHash("chunk2"),
ChunkHash: types.ChunkHash(chunk2Hash),
}
err = repo.Create(ctx, nil, fc2)
if err != nil {
t.Fatalf("failed to create second file chunk: %v", err)
}
fc3 := &FileChunk{
fc3 := &database.FileChunk{
FileID: file.ID,
Idx: 2,
ChunkHash: types.ChunkHash("chunk3"),
ChunkHash: types.ChunkHash(chunk3Hash),
}
err = repo.Create(ctx, nil, fc3)
if err != nil {
t.Fatalf("failed to create third file chunk: %v", err)
}
// Test GetByFile
fileChunks, err := repo.GetByFile(ctx, "/test/file.txt")
fileChunks, err := repo.GetByFile(ctx, testFileTxt)
if err != nil {
t.Fatalf("failed to get file chunks: %v", err)
}
if len(fileChunks) != 3 {
t.Errorf("expected 3 chunks, got %d", len(fileChunks))
}
@@ -101,6 +93,41 @@ func TestFileChunkRepository(t *testing.T) {
if err != nil {
t.Fatalf("failed to create duplicate file chunk: %v", err)
}
}
func TestFileChunkRepositoryDeleteByFileID(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repo := database.NewFileChunkRepository(db)
fileRepo := database.NewFileRepository(db)
repos := database.NewRepositories(db)
file := &database.File{
Path: testFileTxt,
MTime: time.Now().Truncate(time.Second),
Size: 1024,
Mode: 0644,
UID: 1000,
GID: 1000,
}
mustCreateFile(t, fileRepo, file)
mustCreateChunks(t, repos, chunk1Hash)
fc := &database.FileChunk{
FileID: file.ID,
Idx: 0,
ChunkHash: types.ChunkHash(chunk1Hash),
}
err := repo.Create(ctx, nil, fc)
if err != nil {
t.Fatalf("failed to create file chunk: %v", err)
}
// Test DeleteByFileID
err = repo.DeleteByFileID(ctx, nil, file.ID)
@@ -108,30 +135,33 @@ func TestFileChunkRepository(t *testing.T) {
t.Fatalf("failed to delete file chunks: %v", err)
}
fileChunks, err = repo.GetByFileID(ctx, file.ID)
fileChunks, err := repo.GetByFileID(ctx, file.ID)
if err != nil {
t.Fatalf("failed to get deleted file chunks: %v", err)
}
if len(fileChunks) != 0 {
t.Errorf("expected 0 chunks after delete, got %d", len(fileChunks))
}
}
func TestFileChunkRepositoryMultipleFiles(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repo := NewFileChunkRepository(db)
fileRepo := NewFileRepository(db)
repo := database.NewFileChunkRepository(db)
fileRepo := database.NewFileRepository(db)
// Create test files
testTime := time.Now().Truncate(time.Second)
filePaths := []string{"/file1.txt", "/file2.txt", "/file3.txt"}
files := make([]*File, len(filePaths))
filePaths := []string{testFilePath1, testFilePath2, "/file3.txt"}
files := make([]*database.File, len(filePaths))
for i, path := range filePaths {
file := &File{
file := &database.File{
Path: types.FilePath(path),
MTime: testTime,
Size: 2048,
@@ -140,22 +170,23 @@ func TestFileChunkRepositoryMultipleFiles(t *testing.T) {
GID: 1000,
LinkTarget: "",
}
err := fileRepo.Create(ctx, nil, file)
if err != nil {
t.Fatalf("failed to create file %s: %v", path, err)
}
mustCreateFile(t, fileRepo, file)
files[i] = file
}
// Create all chunks first
chunkRepo := NewChunkRepository(db)
chunkRepo := database.NewChunkRepository(db)
for i := range files {
for j := 0; j < 2; j++ {
for j := range 2 {
chunkHash := types.ChunkHash(fmt.Sprintf("file%d_chunk%d", i, j))
chunk := &Chunk{
chunk := &database.Chunk{
ChunkHash: chunkHash,
Size: 1024,
}
err := chunkRepo.Create(ctx, nil, chunk)
if err != nil {
t.Fatalf("failed to create chunk %s: %v", chunkHash, err)
@@ -165,12 +196,13 @@ func TestFileChunkRepositoryMultipleFiles(t *testing.T) {
// Create chunks for multiple files
for i, file := range files {
for j := 0; j < 2; j++ {
fc := &FileChunk{
for j := range 2 {
fc := &database.FileChunk{
FileID: file.ID,
Idx: j,
ChunkHash: types.ChunkHash(fmt.Sprintf("file%d_chunk%d", i, j)),
}
err := repo.Create(ctx, nil, fc)
if err != nil {
t.Fatalf("failed to create file chunk: %v", err)
@@ -184,6 +216,7 @@ func TestFileChunkRepositoryMultipleFiles(t *testing.T) {
if err != nil {
t.Fatalf("failed to get chunks for file %d: %v", i, err)
}
if len(chunks) != 2 {
t.Errorf("expected 2 chunks for file %d, got %d", i, len(chunks))
}

View File

@@ -3,21 +3,29 @@ package database
import (
"context"
"database/sql"
"errors"
"fmt"
"strings"
"time"
"sneak.berlin/go/vaultik/internal/log"
"sneak.berlin/go/vaultik/internal/types"
)
// FileRepository provides access to the files table, which stores file
// metadata (path, times, permissions, ownership, symlink targets).
type FileRepository struct {
db *DB
}
// NewFileRepository creates a FileRepository backed by db.
func NewFileRepository(db *DB) *FileRepository {
return &FileRepository{db: db}
}
// Create inserts or updates a file row (upsert on path), using tx when
// non-nil. The file's ID is generated when zero and updated from the
// database's RETURNING clause.
func (r *FileRepository) Create(ctx context.Context, tx *sql.Tx, file *File) error {
// Generate UUID if not provided
if file.ID.IsZero() {
@@ -38,13 +46,25 @@ func (r *FileRepository) Create(ctx context.Context, tx *sql.Tx, file *File) err
RETURNING id
`
var idStr string
var err error
var (
idStr string
err error
)
if tx != nil {
LogSQL("Execute", query, file.ID.String(), file.Path.String(), file.SourcePath.String(), file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID, file.LinkTarget.String())
err = tx.QueryRowContext(ctx, query, file.ID.String(), file.Path.String(), file.SourcePath.String(), file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID, file.LinkTarget.String()).Scan(&idStr)
LogSQL("Execute", query,
file.ID.String(), file.Path.String(), file.SourcePath.String(),
file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID,
file.LinkTarget.String())
err = tx.QueryRowContext(ctx, query,
file.ID.String(), file.Path.String(), file.SourcePath.String(),
file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID,
file.LinkTarget.String()).Scan(&idStr)
} else {
err = r.db.QueryRowWithLog(ctx, query, file.ID.String(), file.Path.String(), file.SourcePath.String(), file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID, file.LinkTarget.String()).Scan(&idStr)
err = r.db.QueryRowWithLog(ctx, query,
file.ID.String(), file.Path.String(), file.SourcePath.String(),
file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID,
file.LinkTarget.String()).Scan(&idStr)
}
if err != nil {
@@ -60,6 +80,8 @@ func (r *FileRepository) Create(ctx context.Context, tx *sql.Tx, file *File) err
return nil
}
// GetByPath returns the file at the given path, or nil if the path is not
// in the index.
func (r *FileRepository) GetByPath(ctx context.Context, path string) (*File, error) {
query := `
SELECT id, path, source_path, mtime, size, mode, uid, gid, link_target
@@ -68,9 +90,10 @@ func (r *FileRepository) GetByPath(ctx context.Context, path string) (*File, err
`
file, err := r.scanFile(r.db.conn.QueryRowContext(ctx, query, path))
if err == sql.ErrNoRows {
return nil, nil
if errors.Is(err, sql.ErrNoRows) {
return nil, nil //nolint:nilnil // nil,nil signals not-found; callers check nil
}
if err != nil {
return nil, fmt.Errorf("querying file: %w", err)
}
@@ -87,9 +110,10 @@ func (r *FileRepository) GetByID(ctx context.Context, id types.FileID) (*File, e
`
file, err := r.scanFile(r.db.conn.QueryRowContext(ctx, query, id.String()))
if err == sql.ErrNoRows {
return nil, nil
if errors.Is(err, sql.ErrNoRows) {
return nil, nil //nolint:nilnil // nil,nil signals not-found; callers check nil
}
if err != nil {
return nil, fmt.Errorf("querying file: %w", err)
}
@@ -97,7 +121,11 @@ func (r *FileRepository) GetByID(ctx context.Context, id types.FileID) (*File, e
return file, nil
}
func (r *FileRepository) GetByPathTx(ctx context.Context, tx *sql.Tx, path string) (*File, error) {
// GetByPathTx returns the file at the given path within a transaction, or
// nil if the path is not in the index.
func (r *FileRepository) GetByPathTx(
ctx context.Context, tx *sql.Tx, path string,
) (*File, error) {
query := `
SELECT id, path, source_path, mtime, size, mode, uid, gid, link_target
FROM files
@@ -108,9 +136,10 @@ func (r *FileRepository) GetByPathTx(ctx context.Context, tx *sql.Tx, path strin
file, err := r.scanFile(tx.QueryRowContext(ctx, query, path))
LogSQL("GetByPathTx Scan complete", query, path)
if err == sql.ErrNoRows {
return nil, nil
if errors.Is(err, sql.ErrNoRows) {
return nil, nil //nolint:nilnil // nil,nil signals not-found; callers check nil
}
if err != nil {
return nil, fmt.Errorf("querying file: %w", err)
}
@@ -118,79 +147,16 @@ func (r *FileRepository) GetByPathTx(ctx context.Context, tx *sql.Tx, path strin
return file, nil
}
// scanFile is a helper that scans a single file row
func (r *FileRepository) scanFile(row *sql.Row) (*File, error) {
var file File
var idStr, pathStr, sourcePathStr string
var mtimeUnix int64
var linkTarget sql.NullString
err := row.Scan(
&idStr,
&pathStr,
&sourcePathStr,
&mtimeUnix,
&file.Size,
&file.Mode,
&file.UID,
&file.GID,
&linkTarget,
)
if err != nil {
return nil, err
}
file.ID, err = types.ParseFileID(idStr)
if err != nil {
return nil, fmt.Errorf("parsing file ID: %w", err)
}
file.Path = types.FilePath(pathStr)
file.SourcePath = types.SourcePath(sourcePathStr)
file.MTime = time.Unix(mtimeUnix, 0).UTC()
if linkTarget.Valid {
file.LinkTarget = types.FilePath(linkTarget.String)
}
return &file, nil
// fileRowScanner abstracts *sql.Row and *sql.Rows for scanning a file row.
type fileRowScanner interface {
Scan(dest ...any) error
}
// scanFileRows is a helper that scans a file row from rows iterator
func (r *FileRepository) scanFileRows(rows *sql.Rows) (*File, error) {
var file File
var idStr, pathStr, sourcePathStr string
var mtimeUnix int64
var linkTarget sql.NullString
err := rows.Scan(
&idStr,
&pathStr,
&sourcePathStr,
&mtimeUnix,
&file.Size,
&file.Mode,
&file.UID,
&file.GID,
&linkTarget,
)
if err != nil {
return nil, err
}
file.ID, err = types.ParseFileID(idStr)
if err != nil {
return nil, fmt.Errorf("parsing file ID: %w", err)
}
file.Path = types.FilePath(pathStr)
file.SourcePath = types.SourcePath(sourcePathStr)
file.MTime = time.Unix(mtimeUnix, 0).UTC()
if linkTarget.Valid {
file.LinkTarget = types.FilePath(linkTarget.String)
}
return &file, nil
}
func (r *FileRepository) ListModifiedSince(ctx context.Context, since time.Time) ([]*File, error) {
// ListModifiedSince returns all files whose recorded mtime is at or after
// since, ordered by path.
func (r *FileRepository) ListModifiedSince(
ctx context.Context, since time.Time,
) ([]*File, error) {
query := `
SELECT id, path, source_path, mtime, size, mode, uid, gid, link_target
FROM files
@@ -205,17 +171,20 @@ func (r *FileRepository) ListModifiedSince(ctx context.Context, since time.Time)
defer CloseRows(rows)
var files []*File
for rows.Next() {
file, err := r.scanFileRows(rows)
if err != nil {
return nil, fmt.Errorf("scanning file: %w", err)
}
files = append(files, file)
}
return files, rows.Err()
}
// Delete removes the file row at the given path, using tx when non-nil.
func (r *FileRepository) Delete(ctx context.Context, tx *sql.Tx, path string) error {
query := `DELETE FROM files WHERE path = ?`
@@ -234,7 +203,9 @@ func (r *FileRepository) Delete(ctx context.Context, tx *sql.Tx, path string) er
}
// DeleteByID deletes a file by its UUID
func (r *FileRepository) DeleteByID(ctx context.Context, tx *sql.Tx, id types.FileID) error {
func (r *FileRepository) DeleteByID(
ctx context.Context, tx *sql.Tx, id types.FileID,
) error {
query := `DELETE FROM files WHERE id = ?`
var err error
@@ -251,7 +222,11 @@ func (r *FileRepository) DeleteByID(ctx context.Context, tx *sql.Tx, id types.Fi
return nil
}
func (r *FileRepository) ListByPrefix(ctx context.Context, prefix string) ([]*File, error) {
// ListByPrefix returns all files whose path starts with prefix, ordered by
// path.
func (r *FileRepository) ListByPrefix(
ctx context.Context, prefix string,
) ([]*File, error) {
query := `
SELECT id, path, source_path, mtime, size, mode, uid, gid, link_target
FROM files
@@ -266,11 +241,13 @@ func (r *FileRepository) ListByPrefix(ctx context.Context, prefix string) ([]*Fi
defer CloseRows(rows)
var files []*File
for rows.Next() {
file, err := r.scanFileRows(rows)
if err != nil {
return nil, fmt.Errorf("scanning file: %w", err)
}
files = append(files, file)
}
@@ -292,11 +269,13 @@ func (r *FileRepository) ListAll(ctx context.Context) ([]*File, error) {
defer CloseRows(rows)
var files []*File
for rows.Next() {
file, err := r.scanFileRows(rows)
if err != nil {
return nil, fmt.Errorf("scanning file: %w", err)
}
files = append(files, file)
}
@@ -305,30 +284,47 @@ func (r *FileRepository) ListAll(ctx context.Context) ([]*File, error) {
// CreateBatch inserts or updates multiple files in a single statement for efficiency.
// File IDs must be pre-generated before calling this method.
func (r *FileRepository) CreateBatch(ctx context.Context, tx *sql.Tx, files []*File) error {
func (r *FileRepository) CreateBatch(
ctx context.Context, tx *sql.Tx, files []*File,
) error {
if len(files) == 0 {
return nil
}
// Each File has 9 values, so batch at 100 to be safe with SQLite's variable limit
// Each files row binds this many SQL variables.
const fileCols = 9
// Batch at 100 rows to be safe with SQLite's variable limit.
const batchSize = 100
for i := 0; i < len(files); i += batchSize {
end := i + batchSize
if end > len(files) {
end = len(files)
}
end := min(i+batchSize, len(files))
batch := files[i:end]
query := `INSERT INTO files (id, path, source_path, mtime, size, mode, uid, gid, link_target) VALUES `
args := make([]interface{}, 0, len(batch)*9)
query := `INSERT INTO files
(id, path, source_path, mtime, size, mode, uid, gid, link_target)
VALUES `
args := make([]any, 0, len(batch)*fileCols)
var querySb325 strings.Builder
for j, f := range batch {
if j > 0 {
query += ", "
querySb325.WriteString(", ")
}
query += "(?, ?, ?, ?, ?, ?, ?, ?, ?)"
args = append(args, f.ID.String(), f.Path.String(), f.SourcePath.String(), f.MTime.Unix(), f.Size, f.Mode, f.UID, f.GID, f.LinkTarget.String())
querySb325.WriteString("(?, ?, ?, ?, ?, ?, ?, ?, ?)")
args = append(args,
f.ID.String(), f.Path.String(), f.SourcePath.String(),
f.MTime.Unix(), f.Size, f.Mode, f.UID, f.GID,
f.LinkTarget.String())
}
query += querySb325.String() //nolint:gosec // G202: appends "?" placeholders only
query += ` ON CONFLICT(path) DO UPDATE SET
source_path = excluded.source_path,
mtime = excluded.mtime,
@@ -344,6 +340,7 @@ func (r *FileRepository) CreateBatch(ctx context.Context, tx *sql.Tx, files []*F
} else {
_, err = r.db.ExecWithLog(ctx, query, args...)
}
if err != nil {
return fmt.Errorf("batch inserting files: %w", err)
}
@@ -374,3 +371,53 @@ func (r *FileRepository) DeleteOrphaned(ctx context.Context) error {
return nil
}
// scanFile is a helper that scans a single file row
func (r *FileRepository) scanFile(row *sql.Row) (*File, error) {
return r.scanFileFrom(row)
}
// scanFileRows is a helper that scans a file row from rows iterator
func (r *FileRepository) scanFileRows(rows *sql.Rows) (*File, error) {
return r.scanFileFrom(rows)
}
// scanFileFrom scans one file row from any row scanner.
func (r *FileRepository) scanFileFrom(row fileRowScanner) (*File, error) {
var (
file File
idStr, pathStr, sourcePathStr string
mtimeUnix int64
linkTarget sql.NullString
)
err := row.Scan(
&idStr,
&pathStr,
&sourcePathStr,
&mtimeUnix,
&file.Size,
&file.Mode,
&file.UID,
&file.GID,
&linkTarget,
)
if err != nil {
return nil, err
}
file.ID, err = types.ParseFileID(idStr)
if err != nil {
return nil, fmt.Errorf("parsing file ID: %w", err)
}
file.Path = types.FilePath(pathStr)
file.SourcePath = types.SourcePath(sourcePathStr)
file.MTime = time.Unix(mtimeUnix, 0).UTC()
if linkTarget.Valid {
file.LinkTarget = types.FilePath(linkTarget.String)
}
return &file, nil
}

View File

@@ -1,43 +1,32 @@
package database
package database_test
import (
"context"
"database/sql"
"fmt"
"errors"
"os"
"path/filepath"
"testing"
"time"
"sneak.berlin/go/vaultik/internal/database"
)
func setupTestDB(t *testing.T) (*DB, func()) {
ctx := context.Background()
dbPath := filepath.Join(t.TempDir(), "test.db")
db, err := New(ctx, dbPath)
if err != nil {
t.Fatalf("failed to create database: %v", err)
}
cleanup := func() {
if err := db.Close(); err != nil {
t.Errorf("failed to close database: %v", err)
}
}
return db, cleanup
}
// errTestRollback is the sentinel returned from transaction bodies to
// force a rollback in tests.
var errTestRollback = errors.New("test rollback")
func TestFileRepository(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repo := NewFileRepository(db)
repo := database.NewFileRepository(db)
// Test Create
file := &File{
Path: "/test/file.txt",
file := &database.File{
Path: testFileTxt,
MTime: time.Now().Truncate(time.Second),
Size: 1024,
Mode: 0644,
@@ -56,18 +45,23 @@ func TestFileRepository(t *testing.T) {
if err != nil {
t.Fatalf("failed to get file: %v", err)
}
if retrieved == nil {
t.Fatal("expected file, got nil")
}
if retrieved.Path != file.Path {
t.Errorf("path mismatch: got %s, want %s", retrieved.Path, file.Path)
}
if !retrieved.MTime.Equal(file.MTime) {
t.Errorf("mtime mismatch: got %v, want %v", retrieved.MTime, file.MTime)
}
if retrieved.Size != file.Size {
t.Errorf("size mismatch: got %d, want %d", retrieved.Size, file.Size)
}
if retrieved.Mode != file.Mode {
t.Errorf("mode mismatch: got %o, want %o", retrieved.Mode, file.Mode)
}
@@ -75,6 +69,7 @@ func TestFileRepository(t *testing.T) {
// Test Update (upsert)
file.Size = 2048
file.MTime = time.Now().Truncate(time.Second)
err = repo.Create(ctx, nil, file)
if err != nil {
t.Fatalf("failed to update file: %v", err)
@@ -84,15 +79,41 @@ func TestFileRepository(t *testing.T) {
if err != nil {
t.Fatalf("failed to get updated file: %v", err)
}
if retrieved.Size != 2048 {
t.Errorf("size not updated: got %d, want %d", retrieved.Size, 2048)
}
}
func TestFileRepositoryListDelete(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repo := database.NewFileRepository(db)
file := &database.File{
Path: testFileTxt,
MTime: time.Now().Truncate(time.Second),
Size: 1024,
Mode: 0644,
UID: 1000,
GID: 1000,
}
err := repo.Create(ctx, nil, file)
if err != nil {
t.Fatalf("failed to create file: %v", err)
}
// Test ListModifiedSince
files, err := repo.ListModifiedSince(ctx, time.Now().Add(-1*time.Hour))
if err != nil {
t.Fatalf("failed to list files: %v", err)
}
if len(files) != 1 {
t.Errorf("expected 1 file, got %d", len(files))
}
@@ -103,24 +124,27 @@ func TestFileRepository(t *testing.T) {
t.Fatalf("failed to delete file: %v", err)
}
retrieved, err = repo.GetByPath(ctx, file.Path.String())
retrieved, err := repo.GetByPath(ctx, file.Path.String())
if err != nil {
t.Fatalf("error getting deleted file: %v", err)
}
if retrieved != nil {
t.Error("expected nil for deleted file")
}
}
func TestFileRepositorySymlink(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repo := NewFileRepository(db)
repo := database.NewFileRepository(db)
// Test symlink
symlink := &File{
symlink := &database.File{
Path: "/test/link",
MTime: time.Now().Truncate(time.Second),
Size: 0,
@@ -139,25 +163,30 @@ func TestFileRepositorySymlink(t *testing.T) {
if err != nil {
t.Fatalf("failed to get symlink: %v", err)
}
if !retrieved.IsSymlink() {
t.Error("expected IsSymlink() to be true")
}
if retrieved.LinkTarget != symlink.LinkTarget {
t.Errorf("link target mismatch: got %s, want %s", retrieved.LinkTarget, symlink.LinkTarget)
t.Errorf("link target mismatch: got %s, want %s",
retrieved.LinkTarget, symlink.LinkTarget)
}
}
func TestFileRepositoryTransaction(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repos := NewRepositories(db)
repos := database.NewRepositories(db)
// Test transaction rollback
err := repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
file := &File{
Path: "/test/tx_file.txt",
file := &database.File{
Path: testTxFile,
MTime: time.Now().Truncate(time.Second),
Size: 1024,
Mode: 0644,
@@ -165,23 +194,24 @@ func TestFileRepositoryTransaction(t *testing.T) {
GID: 1000,
}
if err := repos.Files.Create(ctx, tx, file); err != nil {
err := repos.Files.Create(ctx, tx, file)
if err != nil {
return err
}
// Return error to trigger rollback
return fmt.Errorf("test rollback")
return errTestRollback
})
if err == nil || err.Error() != "test rollback" {
if !errors.Is(err, errTestRollback) {
t.Fatalf("expected rollback error, got: %v", err)
}
// Verify file was not created
retrieved, err := repos.Files.GetByPath(ctx, "/test/tx_file.txt")
retrieved, err := repos.Files.GetByPath(ctx, testTxFile)
if err != nil {
t.Fatalf("error checking for file: %v", err)
}
if retrieved != nil {
t.Error("file should not exist after rollback")
}

View File

@@ -0,0 +1,81 @@
package database
import (
"context"
"path/filepath"
"testing"
"sneak.berlin/go/vaultik/internal/types"
)
// Common fixture values shared by the internal repository tests.
const (
internalTestHost = "test-host"
internalTestSnapshotID = "test-snapshot"
internalTestFilePath = "/test.txt"
internalTestFile1 = "/file1.txt"
internalTestFile2 = "/file2.txt"
// countFilesQuery counts the rows of the files table.
countFilesQuery = "SELECT COUNT(*) FROM files"
)
// mustCreateFileRow inserts the file row, failing the test on error.
func mustCreateFileRow(t *testing.T, repos *Repositories, file *File) {
t.Helper()
err := repos.Files.Create(context.Background(), nil, file)
if err != nil {
t.Fatalf("failed to create file %s: %v", file.Path, err)
}
}
// mustAddFileToSnapshot associates a file with a snapshot, failing the
// test on error.
func mustAddFileToSnapshot(
t *testing.T, repos *Repositories, snapshotID string, fileID types.FileID,
) {
t.Helper()
err := repos.Snapshots.AddFileByID(context.Background(), nil, snapshotID, fileID)
if err != nil {
t.Fatal(err)
}
}
// setupTestDB creates an on-disk test database in a per-test temp
// directory and returns it along with a cleanup func that closes it.
func setupTestDB(t *testing.T) (*DB, func()) {
t.Helper()
ctx := context.Background()
dbPath := filepath.Join(t.TempDir(), "test.db")
db, err := New(ctx, dbPath)
if err != nil {
t.Fatalf("failed to create database: %v", err)
}
cleanup := func() {
err := db.Close()
if err != nil {
t.Errorf("failed to close database: %v", err)
}
}
return db, cleanup
}
// countRow runs a single-integer COUNT-style query and returns the value.
func countRow(t *testing.T, db *DB, query string, args ...any) int {
t.Helper()
var count int
err := db.conn.QueryRowContext(context.Background(), query, args...).Scan(&count)
if err != nil {
t.Fatal(err)
}
return count
}

View File

@@ -0,0 +1,52 @@
package database_test
import (
"context"
"path/filepath"
"testing"
"sneak.berlin/go/vaultik/internal/database"
)
// Common fixture values shared by the repository tests.
const (
testFilePath1 = "/file1.txt"
testFilePath2 = "/file2.txt"
testFileTxt = "/test/file.txt"
testTxFile = "/test/tx_file.txt"
testHostname = "test-host"
testVersion = "1.0.0"
)
// mustCreateFile inserts the given file row, failing the test on error.
func mustCreateFile(t *testing.T, repo *database.FileRepository, file *database.File) {
t.Helper()
err := repo.Create(context.Background(), nil, file)
if err != nil {
t.Fatalf("failed to create file %s: %v", file.Path, err)
}
}
// setupTestDB creates an on-disk test database in a per-test temp
// directory and returns it along with a cleanup func that closes it.
func setupTestDB(t *testing.T) (*database.DB, func()) {
t.Helper()
ctx := context.Background()
dbPath := filepath.Join(t.TempDir(), "test.db")
db, err := database.New(ctx, dbPath)
if err != nil {
t.Fatalf("failed to create database: %v", err)
}
cleanup := func() {
err := db.Close()
if err != nil {
t.Errorf("failed to close database: %v", err)
}
}
return db, cleanup
}

View File

@@ -18,6 +18,7 @@ type LocalMetaRepository struct {
db *DB
}
// NewLocalMetaRepository creates a LocalMetaRepository backed by db.
func NewLocalMetaRepository(db *DB) *LocalMetaRepository {
return &LocalMetaRepository{db: db}
}
@@ -27,15 +28,18 @@ func NewLocalMetaRepository(db *DB) *LocalMetaRepository {
// "unset" (bind on first use) from "set to something" (compare).
func (r *LocalMetaRepository) Get(ctx context.Context, key string) (string, error) {
var value string
err := r.db.conn.QueryRowContext(ctx,
"SELECT value FROM local_meta WHERE key = ?", key,
).Scan(&value)
if errors.Is(err, sql.ErrNoRows) {
return "", nil
}
if err != nil {
return "", fmt.Errorf("reading local_meta %q: %w", key, err)
}
return value, nil
}
@@ -49,5 +53,6 @@ func (r *LocalMetaRepository) Set(ctx context.Context, key, value string) error
if err != nil {
return fmt.Errorf("writing local_meta %q: %w", key, err)
}
return nil
}

View File

@@ -9,26 +9,33 @@ import (
)
func TestLocalMetaEmptyOnFresh(t *testing.T) {
t.Parallel()
db, err := database.NewTestDB()
require.NoError(t, err)
defer func() { _ = db.Close() }()
repos := database.NewRepositories(db)
got, err := repos.LocalMeta.Get(context.Background(), database.LocalMetaKeyStorageURL)
require.NoError(t, err)
require.Equal(t, "", got, "fresh DB must return empty for unset keys, not error")
require.Empty(t, got, "fresh DB must return empty for unset keys, not error")
}
func TestLocalMetaSetGetRoundTrip(t *testing.T) {
t.Parallel()
db, err := database.NewTestDB()
require.NoError(t, err)
defer func() { _ = db.Close() }()
repos := database.NewRepositories(db)
ctx := context.Background()
require.NoError(t, repos.LocalMeta.Set(ctx, database.LocalMetaKeyStorageURL, "file:///mnt/backups"))
require.NoError(t, repos.LocalMeta.Set(
ctx, database.LocalMetaKeyStorageURL, "file:///mnt/backups"))
got, err := repos.LocalMeta.Get(ctx, database.LocalMetaKeyStorageURL)
require.NoError(t, err)
@@ -36,15 +43,20 @@ func TestLocalMetaSetGetRoundTrip(t *testing.T) {
}
func TestLocalMetaSetOverwrites(t *testing.T) {
t.Parallel()
db, err := database.NewTestDB()
require.NoError(t, err)
defer func() { _ = db.Close() }()
repos := database.NewRepositories(db)
ctx := context.Background()
require.NoError(t, repos.LocalMeta.Set(ctx, database.LocalMetaKeyStorageURL, "s3://old"))
require.NoError(t, repos.LocalMeta.Set(ctx, database.LocalMetaKeyStorageURL, "s3://new"))
require.NoError(t, repos.LocalMeta.Set(
ctx, database.LocalMetaKeyStorageURL, "s3://old"))
require.NoError(t, repos.LocalMeta.Set(
ctx, database.LocalMetaKeyStorageURL, "s3://new"))
got, err := repos.LocalMeta.Get(ctx, database.LocalMetaKeyStorageURL)
require.NoError(t, err)

View File

@@ -1,5 +1,3 @@
// Package database provides data models and repository interfaces for the Vaultik backup system.
// It includes types for files, chunks, blobs, snapshots, and their relationships.
package database
import (
@@ -15,7 +13,10 @@ import (
type File struct {
ID types.FileID // UUID primary key
Path types.FilePath // Absolute path of the file
SourcePath types.SourcePath // The source directory this file came from (for restore path stripping)
// SourcePath is the source directory this file came from (used for
// restore path stripping).
SourcePath types.SourcePath
MTime time.Time
Size int64
Mode uint32
@@ -56,7 +57,10 @@ type Chunk struct {
// -> encrypted with age -> hashed -> uploaded to S3 with the hash as filename.
type Blob struct {
ID types.BlobID // UUID assigned when blob creation starts
Hash types.BlobHash // SHA256 of final compressed+encrypted content (empty until finalized)
// Hash is the SHA256 of the final compressed+encrypted content
// (empty until finalized).
Hash types.BlobHash
CreatedTS time.Time // When blob creation started
FinishedTS *time.Time // When blob was finalized (nil if still packing)
UncompressedSize int64 // Total size of raw chunks before compression
@@ -75,9 +79,10 @@ type BlobChunk struct {
Length int64
}
// ChunkFile represents the reverse mapping showing which files contain a specific chunk.
// This is used during deduplication to identify all files that share a chunk,
// which is important for garbage collection and integrity verification.
// ChunkFile represents the reverse mapping showing which files contain a
// specific chunk. This is used during deduplication to identify all files
// that share a chunk, which is important for garbage collection and
// integrity verification.
type ChunkFile struct {
ChunkHash types.ChunkHash
FileID types.FileID
@@ -97,7 +102,10 @@ type Snapshot struct {
ChunkCount int64
BlobCount int64
TotalSize int64 // Total size of all referenced files
BlobSize int64 // Total size of all referenced blobs (compressed and encrypted)
// BlobSize is the total size of all referenced blobs (compressed and
// encrypted).
BlobSize int64
BlobUncompressedSize int64 // Total uncompressed size of all referenced blobs
CompressionRatio float64 // Compression ratio (BlobSize / BlobUncompressedSize)
CompressionLevel int // Compression level used for this snapshot

View File

@@ -11,7 +11,13 @@ import (
"sneak.berlin/go/vaultik/internal/log"
)
// indexDirPerm restricts the local index directory to the owning user;
// the index describes the backed-up file tree and must stay private.
const indexDirPerm = 0o700
// Module provides database dependencies
//
//nolint:gochecknoglobals // fx module definitions are package globals by convention
var Module = fx.Module("database",
fx.Provide(
provideDatabase,
@@ -22,7 +28,9 @@ var Module = fx.Module("database",
func provideDatabase(lc fx.Lifecycle, cfg *config.Config) (*DB, error) {
// Ensure the index directory exists
indexDir := filepath.Dir(cfg.IndexPath)
if err := os.MkdirAll(indexDir, 0700); err != nil {
err := os.MkdirAll(indexDir, indexDirPerm)
if err != nil {
return nil, fmt.Errorf("creating index directory: %w", err)
}
@@ -32,13 +40,18 @@ func provideDatabase(lc fx.Lifecycle, cfg *config.Config) (*DB, error) {
}
lc.Append(fx.Hook{
OnStop: func(ctx context.Context) error {
OnStop: func(_ context.Context) error {
log.Debug("Database module OnStop hook called")
if err := db.Close(); err != nil {
err := db.Close()
if err != nil {
log.Error("Failed to close database in OnStop hook", "error", err)
return err
}
log.Debug("Database closed successfully in OnStop hook")
return nil
},
})

View File

@@ -50,21 +50,26 @@ type TxFunc func(ctx context.Context, tx *sql.Tx) error
// This method should be used for all write operations to ensure atomicity.
func (r *Repositories) WithTx(ctx context.Context, fn TxFunc) error {
LogSQL("WithTx", "Beginning transaction", "")
tx, err := r.db.BeginTx(ctx, nil)
if err != nil {
return fmt.Errorf("beginning transaction: %w", err)
}
LogSQL("WithTx", "Transaction started", "")
defer func() {
if p := recover(); p != nil {
if rollbackErr := tx.Rollback(); rollbackErr != nil {
Fatal("failed to rollback transaction: %v", rollbackErr)
rollbackErr := tx.Rollback()
if rollbackErr != nil {
Fatalf("failed to rollback transaction: %v", rollbackErr)
}
panic(p)
} else if err != nil {
if rollbackErr := tx.Rollback(); rollbackErr != nil {
Fatal("failed to rollback transaction: %v", rollbackErr)
rollbackErr := tx.Rollback()
if rollbackErr != nil {
Fatalf("failed to rollback transaction: %v", rollbackErr)
}
}
}()
@@ -90,6 +95,7 @@ func (r *Repositories) WithReadTx(ctx context.Context, fn TxFunc) error {
opts := &sql.TxOptions{
ReadOnly: true,
}
tx, err := r.db.BeginTx(ctx, opts)
if err != nil {
return fmt.Errorf("beginning read transaction: %w", err)
@@ -97,13 +103,16 @@ func (r *Repositories) WithReadTx(ctx context.Context, fn TxFunc) error {
defer func() {
if p := recover(); p != nil {
if rollbackErr := tx.Rollback(); rollbackErr != nil {
Fatal("failed to rollback transaction: %v", rollbackErr)
rollbackErr := tx.Rollback()
if rollbackErr != nil {
Fatalf("failed to rollback transaction: %v", rollbackErr)
}
panic(p)
} else if err != nil {
if rollbackErr := tx.Rollback(); rollbackErr != nil {
Fatal("failed to rollback transaction: %v", rollbackErr)
rollbackErr := tx.Rollback()
if rollbackErr != nil {
Fatalf("failed to rollback transaction: %v", rollbackErr)
}
}
}()

View File

@@ -1,124 +1,162 @@
package database
package database_test
import (
"context"
"database/sql"
"fmt"
"errors"
"testing"
"time"
"sneak.berlin/go/vaultik/internal/database"
"sneak.berlin/go/vaultik/internal/types"
)
func TestRepositoriesTransaction(t *testing.T) {
db, cleanup := setupTestDB(t)
defer cleanup()
// errIntentionalRollback forces a transaction rollback in tests.
var errIntentionalRollback = errors.New("intentional rollback")
ctx := context.Background()
repos := NewRepositories(db)
// Test successful transaction with multiple operations
err := repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
// Create a file
file := &File{
Path: "/test/tx_file.txt",
// createTxTestData returns a transaction body that creates a file with
// two chunks packed into one blob.
func createTxTestData(
repos *database.Repositories,
) func(context.Context, *sql.Tx) error {
return func(ctx context.Context, tx *sql.Tx) error {
file := &database.File{
Path: testTxFile,
MTime: time.Now().Truncate(time.Second),
Size: 1024,
Mode: 0644,
UID: 1000,
GID: 1000,
}
if err := repos.Files.Create(ctx, tx, file); err != nil {
err := repos.Files.Create(ctx, tx, file)
if err != nil {
return err
}
err = createTxFileChunks(ctx, tx, repos, file.ID)
if err != nil {
return err
}
return createTxBlob(ctx, tx, repos)
}
}
// createTxFileChunks creates the two test chunks and maps them to the file.
func createTxFileChunks(
ctx context.Context, tx *sql.Tx,
repos *database.Repositories, fileID types.FileID,
) error {
// Create chunks
chunk1 := &Chunk{
chunk1 := &database.Chunk{
ChunkHash: types.ChunkHash("tx_chunk1"),
Size: 512,
}
if err := repos.Chunks.Create(ctx, tx, chunk1); err != nil {
err := repos.Chunks.Create(ctx, tx, chunk1)
if err != nil {
return err
}
chunk2 := &Chunk{
chunk2 := &database.Chunk{
ChunkHash: types.ChunkHash("tx_chunk2"),
Size: 512,
}
if err := repos.Chunks.Create(ctx, tx, chunk2); err != nil {
err = repos.Chunks.Create(ctx, tx, chunk2)
if err != nil {
return err
}
// Map chunks to file
fc1 := &FileChunk{
FileID: file.ID,
fc1 := &database.FileChunk{
FileID: fileID,
Idx: 0,
ChunkHash: chunk1.ChunkHash,
}
if err := repos.FileChunks.Create(ctx, tx, fc1); err != nil {
err = repos.FileChunks.Create(ctx, tx, fc1)
if err != nil {
return err
}
fc2 := &FileChunk{
FileID: file.ID,
fc2 := &database.FileChunk{
FileID: fileID,
Idx: 1,
ChunkHash: chunk2.ChunkHash,
}
if err := repos.FileChunks.Create(ctx, tx, fc2); err != nil {
return err
}
// Create blob
blob := &Blob{
return repos.FileChunks.Create(ctx, tx, fc2)
}
// createTxBlob creates the test blob and maps both chunks into it.
func createTxBlob(
ctx context.Context, tx *sql.Tx, repos *database.Repositories,
) error {
blob := &database.Blob{
ID: types.NewBlobID(),
Hash: types.BlobHash("tx_blob1"),
CreatedTS: time.Now().Truncate(time.Second),
}
if err := repos.Blobs.Create(ctx, tx, blob); err != nil {
err := repos.Blobs.Create(ctx, tx, blob)
if err != nil {
return err
}
// Map chunks to blob
bc1 := &BlobChunk{
bc1 := &database.BlobChunk{
BlobID: blob.ID,
ChunkHash: chunk1.ChunkHash,
ChunkHash: types.ChunkHash("tx_chunk1"),
Offset: 0,
Length: 512,
}
if err := repos.BlobChunks.Create(ctx, tx, bc1); err != nil {
err = repos.BlobChunks.Create(ctx, tx, bc1)
if err != nil {
return err
}
bc2 := &BlobChunk{
bc2 := &database.BlobChunk{
BlobID: blob.ID,
ChunkHash: chunk2.ChunkHash,
ChunkHash: types.ChunkHash("tx_chunk2"),
Offset: 512,
Length: 512,
}
if err := repos.BlobChunks.Create(ctx, tx, bc2); err != nil {
return err
}
return nil
})
return repos.BlobChunks.Create(ctx, tx, bc2)
}
func TestRepositoriesTransaction(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repos := database.NewRepositories(db)
err := repos.WithTx(ctx, createTxTestData(repos))
if err != nil {
t.Fatalf("transaction failed: %v", err)
}
// Verify all data was committed
file, err := repos.Files.GetByPath(ctx, "/test/tx_file.txt")
file, err := repos.Files.GetByPath(ctx, testTxFile)
if err != nil {
t.Fatalf("failed to get file: %v", err)
}
if file == nil {
t.Error("expected file after transaction")
}
chunks, err := repos.FileChunks.GetByFile(ctx, "/test/tx_file.txt")
chunks, err := repos.FileChunks.GetByFile(ctx, testTxFile)
if err != nil {
t.Fatalf("failed to get file chunks: %v", err)
}
if len(chunks) != 2 {
t.Errorf("expected 2 file chunks, got %d", len(chunks))
}
@@ -127,22 +165,25 @@ func TestRepositoriesTransaction(t *testing.T) {
if err != nil {
t.Fatalf("failed to get blob: %v", err)
}
if blob == nil {
t.Error("expected blob after transaction")
}
}
func TestRepositoriesTransactionRollback(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repos := NewRepositories(db)
repos := database.NewRepositories(db)
// Test transaction rollback
err := repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
// Create a file
file := &File{
file := &database.File{
Path: "/test/rollback_file.txt",
MTime: time.Now().Truncate(time.Second),
Size: 1024,
@@ -150,24 +191,27 @@ func TestRepositoriesTransactionRollback(t *testing.T) {
UID: 1000,
GID: 1000,
}
if err := repos.Files.Create(ctx, tx, file); err != nil {
err := repos.Files.Create(ctx, tx, file)
if err != nil {
return err
}
// Create a chunk
chunk := &Chunk{
chunk := &database.Chunk{
ChunkHash: types.ChunkHash("rollback_chunk"),
Size: 1024,
}
if err := repos.Chunks.Create(ctx, tx, chunk); err != nil {
err = repos.Chunks.Create(ctx, tx, chunk)
if err != nil {
return err
}
// Return error to trigger rollback
return fmt.Errorf("intentional rollback")
return errIntentionalRollback
})
if err == nil || err.Error() != "intentional rollback" {
if !errors.Is(err, errIntentionalRollback) {
t.Fatalf("expected rollback error, got: %v", err)
}
@@ -176,6 +220,7 @@ func TestRepositoriesTransactionRollback(t *testing.T) {
if err != nil {
t.Fatalf("error checking for file: %v", err)
}
if file != nil {
t.Error("file should not exist after rollback")
}
@@ -184,20 +229,23 @@ func TestRepositoriesTransactionRollback(t *testing.T) {
if err != nil {
t.Fatalf("error checking for chunk: %v", err)
}
if chunk != nil {
t.Error("chunk should not exist after rollback")
}
}
func TestRepositoriesReadTransaction(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repos := NewRepositories(db)
repos := database.NewRepositories(db)
// First, create some data
file := &File{
file := &database.File{
Path: "/test/read_file.txt",
MTime: time.Now().Truncate(time.Second),
Size: 1024,
@@ -205,22 +253,25 @@ func TestRepositoriesReadTransaction(t *testing.T) {
UID: 1000,
GID: 1000,
}
err := repos.Files.Create(ctx, nil, file)
if err != nil {
t.Fatalf("failed to create file: %v", err)
}
// Test read-only transaction
var retrievedFile *File
var retrievedFile *database.File
err = repos.WithReadTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
var err error
retrievedFile, err = repos.Files.GetByPathTx(ctx, tx, "/test/read_file.txt")
if err != nil {
return err
}
// Try to write in read-only transaction (should fail)
_ = repos.Files.Create(ctx, tx, &File{
_ = repos.Files.Create(ctx, tx, &database.File{
Path: "/test/should_fail.txt",
MTime: time.Now(),
Size: 0,
@@ -232,7 +283,6 @@ func TestRepositoriesReadTransaction(t *testing.T) {
return nil
})
if err != nil {
t.Fatalf("read transaction failed: %v", err)
}

View File

@@ -1,8 +1,10 @@
//nolint:testpackage // inspects the unexported database connection
package database
import (
"context"
"database/sql"
"errors"
"fmt"
"testing"
"time"
@@ -10,8 +12,13 @@ import (
"sneak.berlin/go/vaultik/internal/types"
)
// errTxIntentionalRollback forces a transaction rollback in tests.
var errTxIntentionalRollback = errors.New("intentional rollback")
// TestFileRepositoryUUIDGeneration tests that files get unique UUIDs
func TestFileRepositoryUUIDGeneration(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
@@ -21,7 +28,7 @@ func TestFileRepositoryUUIDGeneration(t *testing.T) {
// Create multiple files
files := []*File{
{
Path: "/file1.txt",
Path: internalTestFile1,
MTime: time.Now().Truncate(time.Second),
Size: 1024,
Mode: 0644,
@@ -29,7 +36,7 @@ func TestFileRepositoryUUIDGeneration(t *testing.T) {
GID: 1000,
},
{
Path: "/file2.txt",
Path: internalTestFile2,
MTime: time.Now().Truncate(time.Second),
Size: 2048,
Mode: 0644,
@@ -39,6 +46,7 @@ func TestFileRepositoryUUIDGeneration(t *testing.T) {
}
uuids := make(map[string]bool)
for _, file := range files {
err := repo.Create(ctx, nil, file)
if err != nil {
@@ -54,12 +62,15 @@ func TestFileRepositoryUUIDGeneration(t *testing.T) {
if uuids[file.ID.String()] {
t.Errorf("duplicate UUID generated: %s", file.ID)
}
uuids[file.ID.String()] = true
}
}
// TestFileRepositoryGetByID tests retrieving files by UUID
func TestFileRepositoryGetByID(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
@@ -68,7 +79,7 @@ func TestFileRepositoryGetByID(t *testing.T) {
// Create a file
file := &File{
Path: "/test.txt",
Path: internalTestFilePath,
MTime: time.Now().Truncate(time.Second),
Size: 1024,
Mode: 0644,
@@ -90,16 +101,20 @@ func TestFileRepositoryGetByID(t *testing.T) {
if retrieved.ID != file.ID {
t.Errorf("ID mismatch: expected %s, got %s", file.ID, retrieved.ID)
}
if retrieved.Path != file.Path {
t.Errorf("Path mismatch: expected %s, got %s", file.Path, retrieved.Path)
}
// Test non-existent ID
nonExistentID := types.NewFileID() // Generate a new UUID that won't exist in the database
// Test non-existent ID: generate a new UUID that won't exist in the
// database.
nonExistentID := types.NewFileID()
nonExistent, err := repo.GetByID(ctx, nonExistentID)
if err != nil {
t.Fatalf("GetByID should not return error for non-existent ID: %v", err)
}
if nonExistent != nil {
t.Error("expected nil for non-existent ID")
}
@@ -107,6 +122,8 @@ func TestFileRepositoryGetByID(t *testing.T) {
// TestOrphanedFileCleanup tests the cleanup of orphaned files
func TestOrphanedFileCleanup(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
@@ -135,6 +152,7 @@ func TestOrphanedFileCleanup(t *testing.T) {
if err != nil {
t.Fatalf("failed to create file1: %v", err)
}
err = repos.Files.Create(ctx, nil, file2)
if err != nil {
t.Fatalf("failed to create file2: %v", err)
@@ -142,20 +160,18 @@ func TestOrphanedFileCleanup(t *testing.T) {
// Create a snapshot and reference only file2
snapshot := &Snapshot{
ID: "test-snapshot",
Hostname: "test-host",
ID: internalTestSnapshotID,
Hostname: internalTestHost,
StartedAt: time.Now(),
}
err = repos.Snapshots.Create(ctx, nil, snapshot)
if err != nil {
t.Fatalf("failed to create snapshot: %v", err)
}
// Add file2 to snapshot
err = repos.Snapshots.AddFileByID(ctx, nil, snapshot.ID.String(), file2.ID)
if err != nil {
t.Fatalf("failed to add file to snapshot: %v", err)
}
mustAddFileToSnapshot(t, repos, snapshot.ID.String(), file2.ID)
// Run orphaned cleanup
err = repos.Files.DeleteOrphaned(ctx)
@@ -168,6 +184,7 @@ func TestOrphanedFileCleanup(t *testing.T) {
if err != nil {
t.Fatalf("error getting file: %v", err)
}
if orphanedFile != nil {
t.Error("orphaned file should have been deleted")
}
@@ -177,6 +194,7 @@ func TestOrphanedFileCleanup(t *testing.T) {
if err != nil {
t.Fatalf("error getting file: %v", err)
}
if referencedFile == nil {
t.Error("referenced file should not have been deleted")
}
@@ -184,6 +202,8 @@ func TestOrphanedFileCleanup(t *testing.T) {
// TestOrphanedChunkCleanup tests the cleanup of orphaned chunks
func TestOrphanedChunkCleanup(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
@@ -204,6 +224,7 @@ func TestOrphanedChunkCleanup(t *testing.T) {
if err != nil {
t.Fatalf("failed to create chunk1: %v", err)
}
err = repos.Chunks.Create(ctx, nil, chunk2)
if err != nil {
t.Fatalf("failed to create chunk2: %v", err)
@@ -211,13 +232,14 @@ func TestOrphanedChunkCleanup(t *testing.T) {
// Create a file and reference only chunk2
file := &File{
Path: "/test.txt",
Path: internalTestFilePath,
MTime: time.Now().Truncate(time.Second),
Size: 1024,
Mode: 0644,
UID: 1000,
GID: 1000,
}
err = repos.Files.Create(ctx, nil, file)
if err != nil {
t.Fatalf("failed to create file: %v", err)
@@ -229,6 +251,7 @@ func TestOrphanedChunkCleanup(t *testing.T) {
Idx: 0,
ChunkHash: chunk2.ChunkHash,
}
err = repos.FileChunks.Create(ctx, nil, fc)
if err != nil {
t.Fatalf("failed to create file chunk: %v", err)
@@ -245,6 +268,7 @@ func TestOrphanedChunkCleanup(t *testing.T) {
if err != nil {
t.Fatalf("error getting chunk: %v", err)
}
if orphanedChunk != nil {
t.Error("orphaned chunk should have been deleted")
}
@@ -254,6 +278,7 @@ func TestOrphanedChunkCleanup(t *testing.T) {
if err != nil {
t.Fatalf("error getting chunk: %v", err)
}
if referencedChunk == nil {
t.Error("referenced chunk should not have been deleted")
}
@@ -261,6 +286,8 @@ func TestOrphanedChunkCleanup(t *testing.T) {
// TestOrphanedBlobCleanup tests the cleanup of orphaned blobs
func TestOrphanedBlobCleanup(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
@@ -283,6 +310,7 @@ func TestOrphanedBlobCleanup(t *testing.T) {
if err != nil {
t.Fatalf("failed to create blob1: %v", err)
}
err = repos.Blobs.Create(ctx, nil, blob2)
if err != nil {
t.Fatalf("failed to create blob2: %v", err)
@@ -290,10 +318,11 @@ func TestOrphanedBlobCleanup(t *testing.T) {
// Create a snapshot and reference only blob2
snapshot := &Snapshot{
ID: "test-snapshot",
Hostname: "test-host",
ID: internalTestSnapshotID,
Hostname: internalTestHost,
StartedAt: time.Now(),
}
err = repos.Snapshots.Create(ctx, nil, snapshot)
if err != nil {
t.Fatalf("failed to create snapshot: %v", err)
@@ -316,6 +345,7 @@ func TestOrphanedBlobCleanup(t *testing.T) {
if err != nil {
t.Fatalf("error getting blob: %v", err)
}
if orphanedBlob != nil {
t.Error("orphaned blob should have been deleted")
}
@@ -325,6 +355,7 @@ func TestOrphanedBlobCleanup(t *testing.T) {
if err != nil {
t.Fatalf("error getting blob: %v", err)
}
if referencedBlob == nil {
t.Error("referenced blob should not have been deleted")
}
@@ -332,6 +363,8 @@ func TestOrphanedBlobCleanup(t *testing.T) {
// TestFileChunkRepositoryWithUUIDs tests file-chunk relationships with UUIDs
func TestFileChunkRepositoryWithUUIDs(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
@@ -340,17 +373,15 @@ func TestFileChunkRepositoryWithUUIDs(t *testing.T) {
// Create a file
file := &File{
Path: "/test.txt",
Path: internalTestFilePath,
MTime: time.Now().Truncate(time.Second),
Size: 3072,
Mode: 0644,
UID: 1000,
GID: 1000,
}
err := repos.Files.Create(ctx, nil, file)
if err != nil {
t.Fatalf("failed to create file: %v", err)
}
mustCreateFileRow(t, repos, file)
// Create chunks
chunks := []types.ChunkHash{"chunk1", "chunk2", "chunk3"}
@@ -359,7 +390,8 @@ func TestFileChunkRepositoryWithUUIDs(t *testing.T) {
ChunkHash: chunkHash,
Size: 1024,
}
err = repos.Chunks.Create(ctx, nil, chunk)
err := repos.Chunks.Create(ctx, nil, chunk)
if err != nil {
t.Fatalf("failed to create chunk: %v", err)
}
@@ -370,6 +402,7 @@ func TestFileChunkRepositoryWithUUIDs(t *testing.T) {
Idx: i,
ChunkHash: chunkHash,
}
err = repos.FileChunks.Create(ctx, nil, fc)
if err != nil {
t.Fatalf("failed to create file chunk: %v", err)
@@ -381,6 +414,7 @@ func TestFileChunkRepositoryWithUUIDs(t *testing.T) {
if err != nil {
t.Fatalf("failed to get file chunks: %v", err)
}
if len(fileChunks) != 3 {
t.Errorf("expected 3 chunks, got %d", len(fileChunks))
}
@@ -395,6 +429,7 @@ func TestFileChunkRepositoryWithUUIDs(t *testing.T) {
if err != nil {
t.Fatalf("failed to get file chunks after delete: %v", err)
}
if len(fileChunks) != 0 {
t.Errorf("expected 0 chunks after delete, got %d", len(fileChunks))
}
@@ -402,6 +437,8 @@ func TestFileChunkRepositoryWithUUIDs(t *testing.T) {
// TestChunkFileRepositoryWithUUIDs tests chunk-file relationships with UUIDs
func TestChunkFileRepositoryWithUUIDs(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
@@ -410,7 +447,7 @@ func TestChunkFileRepositoryWithUUIDs(t *testing.T) {
// Create files
file1 := &File{
Path: "/file1.txt",
Path: internalTestFile1,
MTime: time.Now().Truncate(time.Second),
Size: 1024,
Mode: 0644,
@@ -418,7 +455,7 @@ func TestChunkFileRepositoryWithUUIDs(t *testing.T) {
GID: 1000,
}
file2 := &File{
Path: "/file2.txt",
Path: internalTestFile2,
MTime: time.Now().Truncate(time.Second),
Size: 1024,
Mode: 0644,
@@ -426,21 +463,16 @@ func TestChunkFileRepositoryWithUUIDs(t *testing.T) {
GID: 1000,
}
err := repos.Files.Create(ctx, nil, file1)
if err != nil {
t.Fatalf("failed to create file1: %v", err)
}
err = repos.Files.Create(ctx, nil, file2)
if err != nil {
t.Fatalf("failed to create file2: %v", err)
}
mustCreateFileRow(t, repos, file1)
mustCreateFileRow(t, repos, file2)
// Create a chunk that appears in both files (deduplication)
chunk := &Chunk{
ChunkHash: types.ChunkHash("shared-chunk"),
Size: 1024,
}
err = repos.Chunks.Create(ctx, nil, chunk)
err := repos.Chunks.Create(ctx, nil, chunk)
if err != nil {
t.Fatalf("failed to create chunk: %v", err)
}
@@ -463,6 +495,7 @@ func TestChunkFileRepositoryWithUUIDs(t *testing.T) {
if err != nil {
t.Fatalf("failed to create chunk file 1: %v", err)
}
err = repos.ChunkFiles.Create(ctx, nil, cf2)
if err != nil {
t.Fatalf("failed to create chunk file 2: %v", err)
@@ -473,6 +506,7 @@ func TestChunkFileRepositoryWithUUIDs(t *testing.T) {
if err != nil {
t.Fatalf("failed to get chunk files: %v", err)
}
if len(chunkFiles) != 2 {
t.Errorf("expected 2 files for chunk, got %d", len(chunkFiles))
}
@@ -482,6 +516,7 @@ func TestChunkFileRepositoryWithUUIDs(t *testing.T) {
if err != nil {
t.Fatalf("failed to get chunks by file ID: %v", err)
}
if len(chunkFiles) != 1 {
t.Errorf("expected 1 chunk for file, got %d", len(chunkFiles))
}
@@ -489,6 +524,8 @@ func TestChunkFileRepositoryWithUUIDs(t *testing.T) {
// TestSnapshotRepositoryExtendedFields tests snapshot with version and git revision
func TestSnapshotRepositoryExtendedFields(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
@@ -498,7 +535,7 @@ func TestSnapshotRepositoryExtendedFields(t *testing.T) {
// Create snapshot with extended fields
snapshot := &Snapshot{
ID: "test-20250722-120000Z",
Hostname: "test-host",
Hostname: internalTestHost,
VaultikVersion: "0.0.1",
VaultikGitRevision: "abc123def456",
StartedAt: time.Now(),
@@ -526,31 +563,39 @@ func TestSnapshotRepositoryExtendedFields(t *testing.T) {
}
if retrieved.VaultikVersion != snapshot.VaultikVersion {
t.Errorf("version mismatch: expected %s, got %s", snapshot.VaultikVersion, retrieved.VaultikVersion)
t.Errorf("version mismatch: expected %s, got %s",
snapshot.VaultikVersion, retrieved.VaultikVersion)
}
if retrieved.VaultikGitRevision != snapshot.VaultikGitRevision {
t.Errorf("git revision mismatch: expected %s, got %s", snapshot.VaultikGitRevision, retrieved.VaultikGitRevision)
t.Errorf("git revision mismatch: expected %s, got %s",
snapshot.VaultikGitRevision, retrieved.VaultikGitRevision)
}
if retrieved.CompressionLevel != snapshot.CompressionLevel {
t.Errorf("compression level mismatch: expected %d, got %d", snapshot.CompressionLevel, retrieved.CompressionLevel)
t.Errorf("compression level mismatch: expected %d, got %d",
snapshot.CompressionLevel, retrieved.CompressionLevel)
}
if retrieved.BlobUncompressedSize != snapshot.BlobUncompressedSize {
t.Errorf("uncompressed size mismatch: expected %d, got %d", snapshot.BlobUncompressedSize, retrieved.BlobUncompressedSize)
t.Errorf("uncompressed size mismatch: expected %d, got %d",
snapshot.BlobUncompressedSize, retrieved.BlobUncompressedSize)
}
if retrieved.UploadDurationMs != snapshot.UploadDurationMs {
t.Errorf("upload duration mismatch: expected %d, got %d", snapshot.UploadDurationMs, retrieved.UploadDurationMs)
t.Errorf("upload duration mismatch: expected %d, got %d",
snapshot.UploadDurationMs, retrieved.UploadDurationMs)
}
}
// TestComplexOrphanedDataScenario tests a complex scenario with multiple relationships
func TestComplexOrphanedDataScenario(t *testing.T) {
db, cleanup := setupTestDB(t)
defer cleanup()
// createOrphanScenarioFixtures creates two snapshots and three files for
// the orphaned-data cleanup scenario.
func createOrphanScenarioFixtures(
ctx context.Context, t *testing.T, repos *Repositories,
) (*Snapshot, *Snapshot, []*File) {
t.Helper()
ctx := context.Background()
repos := NewRepositories(db)
// Create snapshots
snapshot1 := &Snapshot{
ID: "snapshot1",
Hostname: "host1",
@@ -566,6 +611,7 @@ func TestComplexOrphanedDataScenario(t *testing.T) {
if err != nil {
t.Fatalf("failed to create snapshot1: %v", err)
}
err = repos.Snapshots.Create(ctx, nil, snapshot2)
if err != nil {
t.Fatalf("failed to create snapshot2: %v", err)
@@ -582,40 +628,44 @@ func TestComplexOrphanedDataScenario(t *testing.T) {
UID: 1000,
GID: 1000,
}
err = repos.Files.Create(ctx, nil, files[i])
if err != nil {
t.Fatalf("failed to create file%d: %v", i, err)
}
}
return snapshot1, snapshot2, files
}
func TestComplexOrphanedDataScenario(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repos := NewRepositories(db)
snapshot1, snapshot2, files := createOrphanScenarioFixtures(ctx, t, repos)
// Add files to snapshots
// Snapshot1: file0, file1
// Snapshot2: file1, file2
// file0: only in snapshot1
// file1: in both snapshots
// file2: only in snapshot2
err = repos.Snapshots.AddFileByID(ctx, nil, snapshot1.ID.String(), files[0].ID)
if err != nil {
t.Fatal(err)
}
err = repos.Snapshots.AddFileByID(ctx, nil, snapshot1.ID.String(), files[1].ID)
if err != nil {
t.Fatal(err)
}
err = repos.Snapshots.AddFileByID(ctx, nil, snapshot2.ID.String(), files[1].ID)
if err != nil {
t.Fatal(err)
}
err = repos.Snapshots.AddFileByID(ctx, nil, snapshot2.ID.String(), files[2].ID)
mustAddFileToSnapshot(t, repos, snapshot1.ID.String(), files[0].ID)
mustAddFileToSnapshot(t, repos, snapshot1.ID.String(), files[1].ID)
mustAddFileToSnapshot(t, repos, snapshot2.ID.String(), files[1].ID)
mustAddFileToSnapshot(t, repos, snapshot2.ID.String(), files[2].ID)
// Delete snapshot1
err := repos.Snapshots.DeleteSnapshotFiles(ctx, snapshot1.ID.String())
if err != nil {
t.Fatal(err)
}
// Delete snapshot1
err = repos.Snapshots.DeleteSnapshotFiles(ctx, snapshot1.ID.String())
if err != nil {
t.Fatal(err)
}
err = repos.Snapshots.Delete(ctx, snapshot1.ID.String())
if err != nil {
t.Fatal(err)
@@ -633,6 +683,7 @@ func TestComplexOrphanedDataScenario(t *testing.T) {
if err != nil {
t.Fatalf("error getting file0: %v", err)
}
if file0 != nil {
t.Error("file0 should have been deleted")
}
@@ -642,6 +693,7 @@ func TestComplexOrphanedDataScenario(t *testing.T) {
if err != nil {
t.Fatalf("error getting file1: %v", err)
}
if file1 == nil {
t.Error("file1 should still exist")
}
@@ -651,6 +703,7 @@ func TestComplexOrphanedDataScenario(t *testing.T) {
if err != nil {
t.Fatalf("error getting file2: %v", err)
}
if file2 == nil {
t.Error("file2 should still exist")
}
@@ -658,6 +711,8 @@ func TestComplexOrphanedDataScenario(t *testing.T) {
// TestCascadeDelete tests that cascade deletes work properly
func TestCascadeDelete(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
@@ -673,17 +728,19 @@ func TestCascadeDelete(t *testing.T) {
UID: 1000,
GID: 1000,
}
err := repos.Files.Create(ctx, nil, file)
if err != nil {
t.Fatalf("failed to create file: %v", err)
}
// Create chunks and file-chunk mappings
for i := 0; i < 3; i++ {
for i := range 3 {
chunk := &Chunk{
ChunkHash: types.ChunkHash(fmt.Sprintf("cascade-chunk-%d", i)),
Size: 1024,
}
err = repos.Chunks.Create(ctx, nil, chunk)
if err != nil {
t.Fatalf("failed to create chunk: %v", err)
@@ -694,6 +751,7 @@ func TestCascadeDelete(t *testing.T) {
Idx: i,
ChunkHash: chunk.ChunkHash,
}
err = repos.FileChunks.Create(ctx, nil, fc)
if err != nil {
t.Fatalf("failed to create file chunk: %v", err)
@@ -705,6 +763,7 @@ func TestCascadeDelete(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if len(fileChunks) != 3 {
t.Errorf("expected 3 file chunks, got %d", len(fileChunks))
}
@@ -720,6 +779,7 @@ func TestCascadeDelete(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if len(fileChunks) != 0 {
t.Errorf("expected 0 file chunks after cascade delete, got %d", len(fileChunks))
}
@@ -727,6 +787,8 @@ func TestCascadeDelete(t *testing.T) {
// TestTransactionIsolation tests that transactions properly isolate changes
func TestTransactionIsolation(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
@@ -744,6 +806,7 @@ func TestTransactionIsolation(t *testing.T) {
UID: 1000,
GID: 1000,
}
err := repos.Files.Create(ctx, tx, file)
if err != nil {
return err
@@ -754,9 +817,8 @@ func TestTransactionIsolation(t *testing.T) {
// For now, we'll just test that rollback works
// Return an error to trigger rollback
return fmt.Errorf("intentional rollback")
return errTxIntentionalRollback
})
if err == nil {
t.Fatal("expected error from transaction")
}
@@ -766,37 +828,22 @@ func TestTransactionIsolation(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if len(files) != 0 {
t.Error("file should not exist after rollback")
}
}
// TestConcurrentOrphanedCleanup tests that concurrent cleanup operations don't interfere
func TestConcurrentOrphanedCleanup(t *testing.T) {
db, cleanup := setupTestDB(t)
defer cleanup()
// TestConcurrentOrphanedCleanup tests that concurrent cleanup operations
// don't interfere.
// createConcurrentCleanupFiles creates 20 files and associates the
// even-numbered ones with the snapshot, leaving the rest orphaned.
func createConcurrentCleanupFiles(
ctx context.Context, t *testing.T, repos *Repositories, snapshotID string,
) {
t.Helper()
ctx := context.Background()
repos := NewRepositories(db)
// Set a 5-second busy timeout to handle concurrent operations
if _, err := db.conn.Exec("PRAGMA busy_timeout = 5000"); err != nil {
t.Fatalf("failed to set busy timeout: %v", err)
}
// Create a snapshot
snapshot := &Snapshot{
ID: "concurrent-test",
Hostname: "test-host",
StartedAt: time.Now(),
}
err := repos.Snapshots.Create(ctx, nil, snapshot)
if err != nil {
t.Fatal(err)
}
// Create many files, some orphaned
for i := 0; i < 20; i++ {
for i := range 20 {
file := &File{
Path: types.FilePath(fmt.Sprintf("/concurrent-%d.txt", i)),
MTime: time.Now().Truncate(time.Second),
@@ -805,31 +852,63 @@ func TestConcurrentOrphanedCleanup(t *testing.T) {
UID: 1000,
GID: 1000,
}
err = repos.Files.Create(ctx, nil, file)
err := repos.Files.Create(ctx, nil, file)
if err != nil {
t.Fatal(err)
}
// Add even-numbered files to snapshot
if i%2 == 0 {
err = repos.Snapshots.AddFileByID(ctx, nil, snapshot.ID.String(), file.ID)
err = repos.Snapshots.AddFileByID(ctx, nil, snapshotID, file.ID)
if err != nil {
t.Fatal(err)
}
}
}
}
func TestConcurrentOrphanedCleanup(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repos := NewRepositories(db)
// Set a 5-second busy timeout to handle concurrent operations
_, err := db.conn.ExecContext(ctx, "PRAGMA busy_timeout = 5000")
if err != nil {
t.Fatalf("failed to set busy timeout: %v", err)
}
// Create a snapshot
snapshot := &Snapshot{
ID: "concurrent-test",
Hostname: internalTestHost,
StartedAt: time.Now(),
}
err = repos.Snapshots.Create(ctx, nil, snapshot)
if err != nil {
t.Fatal(err)
}
createConcurrentCleanupFiles(ctx, t, repos, snapshot.ID.String())
// Run multiple cleanup operations concurrently
// Note: SQLite has limited support for concurrent writes, so we expect some to fail
done := make(chan error, 3)
for i := 0; i < 3; i++ {
for range 3 {
go func() {
done <- repos.Files.DeleteOrphaned(ctx)
}()
}
// Wait for all to complete
for i := 0; i < 3; i++ {
for i := range 3 {
err := <-done
if err != nil {
t.Errorf("cleanup %d failed: %v", i, err)
@@ -850,10 +929,12 @@ func TestConcurrentOrphanedCleanup(t *testing.T) {
// Verify all remaining files are even-numbered
for _, file := range files {
var num int
_, err := fmt.Sscanf(file.Path.String(), "/concurrent-%d.txt", &num)
if err != nil {
t.Logf("failed to parse file number from %s: %v", file.Path, err)
}
if num%2 != 0 {
t.Errorf("odd-numbered file %s should have been deleted", file.Path)
}

View File

@@ -1,3 +1,4 @@
//nolint:testpackage // inspects the unexported database connection
package database
import (
@@ -6,15 +7,50 @@ import (
"time"
)
// TestOrphanedFileCleanupDebug tests orphaned file cleanup with debug output
func TestOrphanedFileCleanupDebug(t *testing.T) {
db, cleanup := setupTestDB(t)
defer cleanup()
// logSnapshotFileIDs logs every file_id present in snapshot_files.
func logSnapshotFileIDs(t *testing.T, db *DB) {
t.Helper()
ctx := context.Background()
repos := NewRepositories(db)
// Create files
rows, err := db.conn.QueryContext(ctx, "SELECT file_id FROM snapshot_files")
if err != nil {
t.Fatal(err)
}
defer func() {
err := rows.Close()
if err != nil {
t.Logf("failed to close rows: %v", err)
}
}()
t.Log("Files in snapshot_files:")
for rows.Next() {
var fileID string
err := rows.Scan(&fileID)
if err != nil {
t.Fatal(err)
}
t.Logf(" - %s", fileID)
}
err = rows.Err()
if err != nil {
t.Fatal(err)
}
}
// TestOrphanedFileCleanupDebug tests orphaned file cleanup with debug output
// createOrphanDebugFixtures creates one orphaned file, one referenced
// file, and the snapshot that will reference the latter.
func createOrphanDebugFixtures(
ctx context.Context, t *testing.T, repos *Repositories,
) (*File, *File, *Snapshot) {
t.Helper()
file1 := &File{
Path: "/orphaned.txt",
MTime: time.Now().Truncate(time.Second),
@@ -36,72 +72,65 @@ func TestOrphanedFileCleanupDebug(t *testing.T) {
if err != nil {
t.Fatalf("failed to create file1: %v", err)
}
t.Logf("Created file1 with ID: %s", file1.ID)
err = repos.Files.Create(ctx, nil, file2)
if err != nil {
t.Fatalf("failed to create file2: %v", err)
}
t.Logf("Created file2 with ID: %s", file2.ID)
// Create a snapshot and reference only file2
snapshot := &Snapshot{
ID: "test-snapshot",
Hostname: "test-host",
ID: internalTestSnapshotID,
Hostname: internalTestHost,
StartedAt: time.Now(),
}
err = repos.Snapshots.Create(ctx, nil, snapshot)
if err != nil {
t.Fatalf("failed to create snapshot: %v", err)
}
t.Logf("Created snapshot: %s", snapshot.ID)
return file1, file2, snapshot
}
func TestOrphanedFileCleanupDebug(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repos := NewRepositories(db)
file1, file2, snapshot := createOrphanDebugFixtures(ctx, t, repos)
// Check snapshot_files before adding
var count int
err = db.conn.QueryRow("SELECT COUNT(*) FROM snapshot_files").Scan(&count)
if err != nil {
t.Fatal(err)
}
count := countRow(t, db, "SELECT COUNT(*) FROM snapshot_files")
t.Logf("snapshot_files count before add: %d", count)
// Add file2 to snapshot
err = repos.Snapshots.AddFileByID(ctx, nil, snapshot.ID.String(), file2.ID)
err := repos.Snapshots.AddFileByID(ctx, nil, snapshot.ID.String(), file2.ID)
if err != nil {
t.Fatalf("failed to add file to snapshot: %v", err)
}
t.Logf("Added file2 to snapshot")
// Check snapshot_files after adding
err = db.conn.QueryRow("SELECT COUNT(*) FROM snapshot_files").Scan(&count)
if err != nil {
t.Fatal(err)
}
count = countRow(t, db, "SELECT COUNT(*) FROM snapshot_files")
t.Logf("snapshot_files count after add: %d", count)
// Check which files are referenced
rows, err := db.conn.Query("SELECT file_id FROM snapshot_files")
if err != nil {
t.Fatal(err)
}
defer func() {
if err := rows.Close(); err != nil {
t.Logf("failed to close rows: %v", err)
}
}()
t.Log("Files in snapshot_files:")
for rows.Next() {
var fileID string
if err := rows.Scan(&fileID); err != nil {
t.Fatal(err)
}
t.Logf(" - %s", fileID)
}
logSnapshotFileIDs(t, db)
// Check files before cleanup
err = db.conn.QueryRow("SELECT COUNT(*) FROM files").Scan(&count)
if err != nil {
t.Fatal(err)
}
count = countRow(t, db, countFilesQuery)
t.Logf("Files count before cleanup: %d", count)
// Run orphaned cleanup
@@ -109,13 +138,11 @@ func TestOrphanedFileCleanupDebug(t *testing.T) {
if err != nil {
t.Fatalf("failed to delete orphaned files: %v", err)
}
t.Log("Ran orphaned cleanup")
// Check files after cleanup
err = db.conn.QueryRow("SELECT COUNT(*) FROM files").Scan(&count)
if err != nil {
t.Fatal(err)
}
count = countRow(t, db, countFilesQuery)
t.Logf("Files count after cleanup: %d", count)
// List remaining files
@@ -123,7 +150,9 @@ func TestOrphanedFileCleanupDebug(t *testing.T) {
if err != nil {
t.Fatal(err)
}
t.Log("Remaining files:")
for _, f := range files {
t.Logf(" - ID: %s, Path: %s", f.ID, f.Path)
}
@@ -133,19 +162,16 @@ func TestOrphanedFileCleanupDebug(t *testing.T) {
if err != nil {
t.Fatalf("error getting file: %v", err)
}
if orphanedFile != nil {
t.Error("orphaned file should have been deleted")
// Let's check why it wasn't deleted
var exists bool
err = db.conn.QueryRow(`
stillReferenced := countRow(t, db, `
SELECT EXISTS(
SELECT 1 FROM snapshot_files
WHERE file_id = ?
)`, file1.ID).Scan(&exists)
if err != nil {
t.Fatal(err)
}
t.Logf("File1 exists in snapshot_files: %v", exists)
)`, file1.ID)
t.Logf("File1 exists in snapshot_files: %v", stillReferenced != 0)
} else {
t.Log("Orphaned file was correctly deleted")
}
@@ -155,6 +181,7 @@ func TestOrphanedFileCleanupDebug(t *testing.T) {
if err != nil {
t.Fatalf("error getting file: %v", err)
}
if referencedFile == nil {
t.Error("referenced file should not have been deleted")
} else {

View File

@@ -1,3 +1,4 @@
//nolint:testpackage // inspects the unexported database connection
package database
import (
@@ -10,20 +11,17 @@ import (
"sneak.berlin/go/vaultik/internal/types"
)
// TestFileRepositoryEdgeCases tests edge cases for file repository
func TestFileRepositoryEdgeCases(t *testing.T) {
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repo := NewFileRepository(db)
tests := []struct {
// fileEdgeCase describes one Create edge-case scenario.
type fileEdgeCase struct {
name string
file *File
wantErr bool
errMsg string
}{
}
// fileEdgeCases returns the Create edge-case table.
func fileEdgeCases() []fileEdgeCase {
return []fileEdgeCase{
{
name: "empty path",
file: &File{
@@ -51,6 +49,7 @@ func TestFileRepositoryEdgeCases(t *testing.T) {
{
name: "path with special characters",
file: &File{
//nolint:gosmopolitan // non-ASCII path is deliberate test data
Path: "/test/file with spaces and 特殊文字.txt",
MTime: time.Now(),
Size: 1024,
@@ -86,18 +85,33 @@ func TestFileRepositoryEdgeCases(t *testing.T) {
wantErr: false,
},
}
}
for i, tt := range tests {
// TestFileRepositoryEdgeCases tests edge cases for file repository
func TestFileRepositoryEdgeCases(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
t.Cleanup(cleanup)
ctx := context.Background()
repo := NewFileRepository(db)
for i, tt := range fileEdgeCases() {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
// Add a unique suffix to paths to avoid UNIQUE constraint violations
if tt.file.Path != "" {
tt.file.Path = types.FilePath(fmt.Sprintf("%s_%d_%d", tt.file.Path, i, time.Now().UnixNano()))
tt.file.Path = types.FilePath(fmt.Sprintf("%s_%d_%d",
tt.file.Path, i, time.Now().UnixNano()))
}
err := repo.Create(ctx, nil, tt.file)
if (err != nil) != tt.wantErr {
t.Errorf("Create() error = %v, wantErr %v", err, tt.wantErr)
}
if err != nil && tt.errMsg != "" && !strings.Contains(err.Error(), tt.errMsg) {
t.Errorf("Create() error = %v, want error containing %q", err, tt.errMsg)
}
@@ -105,16 +119,12 @@ func TestFileRepositoryEdgeCases(t *testing.T) {
}
}
// TestDuplicateHandling tests handling of duplicate entries
func TestDuplicateHandling(t *testing.T) {
db, cleanup := setupTestDB(t)
defer cleanup()
// testDuplicateFilePaths exercises the UPSERT behavior for duplicate paths.
func testDuplicateFilePaths(t *testing.T, repos *Repositories) {
t.Helper()
ctx := context.Background()
repos := NewRepositories(db)
// Test duplicate file paths - Create uses UPSERT logic
t.Run("duplicate file paths", func(t *testing.T) {
file1 := &File{
Path: "/duplicate.txt",
MTime: time.Now(),
@@ -136,6 +146,7 @@ func TestDuplicateHandling(t *testing.T) {
if err != nil {
t.Fatalf("failed to create file1: %v", err)
}
originalID := file1.ID
// Create with same path should update the existing record (UPSERT behavior)
@@ -157,31 +168,17 @@ func TestDuplicateHandling(t *testing.T) {
// ID might be different due to the UPSERT
if retrievedFile.ID != file2.ID {
t.Logf("File ID changed from %s to %s during upsert", originalID, retrievedFile.ID)
t.Logf("File ID changed from %s to %s during upsert",
originalID, retrievedFile.ID)
}
})
}
// Test duplicate chunk hashes
t.Run("duplicate chunk hashes", func(t *testing.T) {
chunk := &Chunk{
ChunkHash: types.ChunkHash("duplicate-chunk"),
Size: 1024,
}
// testDuplicateFileChunks exercises idempotent file-chunk mapping creation.
func testDuplicateFileChunks(t *testing.T, repos *Repositories) {
t.Helper()
err := repos.Chunks.Create(ctx, nil, chunk)
if err != nil {
t.Fatalf("failed to create chunk: %v", err)
}
ctx := context.Background()
// Creating the same chunk again should be idempotent (ON CONFLICT DO NOTHING)
err = repos.Chunks.Create(ctx, nil, chunk)
if err != nil {
t.Errorf("duplicate chunk creation should be idempotent, got error: %v", err)
}
})
// Test duplicate file-chunk mappings
t.Run("duplicate file-chunk mappings", func(t *testing.T) {
file := &File{
Path: "/test-dup-fc.txt",
MTime: time.Now(),
@@ -190,6 +187,7 @@ func TestDuplicateHandling(t *testing.T) {
UID: 1000,
GID: 1000,
}
err := repos.Files.Create(ctx, nil, file)
if err != nil {
t.Fatal(err)
@@ -199,6 +197,7 @@ func TestDuplicateHandling(t *testing.T) {
ChunkHash: types.ChunkHash("test-chunk-dup"),
Size: 1024,
}
err = repos.Chunks.Create(ctx, nil, chunk)
if err != nil {
t.Fatal(err)
@@ -220,19 +219,66 @@ func TestDuplicateHandling(t *testing.T) {
if err != nil {
t.Error("file-chunk creation should be idempotent")
}
}
// TestDuplicateHandling tests handling of duplicate entries
func TestDuplicateHandling(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
t.Cleanup(cleanup)
ctx := context.Background()
repos := NewRepositories(db)
// Test duplicate file paths - Create uses UPSERT logic
t.Run("duplicate file paths", func(t *testing.T) {
t.Parallel()
testDuplicateFilePaths(t, repos)
})
// Test duplicate chunk hashes
t.Run("duplicate chunk hashes", func(t *testing.T) {
t.Parallel()
chunk := &Chunk{
ChunkHash: types.ChunkHash("duplicate-chunk"),
Size: 1024,
}
err := repos.Chunks.Create(ctx, nil, chunk)
if err != nil {
t.Fatalf("failed to create chunk: %v", err)
}
// Creating the same chunk again should be idempotent (ON CONFLICT DO NOTHING)
err = repos.Chunks.Create(ctx, nil, chunk)
if err != nil {
t.Errorf("duplicate chunk creation should be idempotent, got error: %v", err)
}
})
// Test duplicate file-chunk mappings
t.Run("duplicate file-chunk mappings", func(t *testing.T) {
t.Parallel()
testDuplicateFileChunks(t, repos)
})
}
// TestNullHandling tests handling of NULL values
func TestNullHandling(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
t.Cleanup(cleanup)
ctx := context.Background()
repos := NewRepositories(db)
// Test file with no link target
t.Run("file without link target", func(t *testing.T) {
t.Parallel()
file := &File{
Path: "/regular.txt",
MTime: time.Now(),
@@ -260,9 +306,11 @@ func TestNullHandling(t *testing.T) {
// Test snapshot with NULL completed_at
t.Run("incomplete snapshot", func(t *testing.T) {
t.Parallel()
snapshot := &Snapshot{
ID: "incomplete-test",
Hostname: "test-host",
Hostname: internalTestHost,
StartedAt: time.Now(),
CompletedAt: nil, // Should remain NULL until completed
}
@@ -284,6 +332,18 @@ func TestNullHandling(t *testing.T) {
// Test blob with NULL uploaded_ts
t.Run("blob not uploaded", func(t *testing.T) {
t.Parallel()
verifyBlobNullUploadTS(ctx, t, repos)
})
}
// verifyBlobNullUploadTS checks that a blob created without an upload
// timestamp round-trips with UploadedTS nil.
func verifyBlobNullUploadTS(
ctx context.Context, t *testing.T, repos *Repositories,
) {
t.Helper()
blob := &Blob{
ID: types.NewBlobID(),
Hash: types.BlobHash("test-hash"),
@@ -304,11 +364,54 @@ func TestNullHandling(t *testing.T) {
if retrieved.UploadedTS != nil {
t.Error("expected nil UploadedTS for non-uploaded blob")
}
})
}
// createLargeDatasetFiles creates fileCount files and adds every other
// one to the snapshot.
func createLargeDatasetFiles(
t *testing.T,
repos *Repositories,
snapshotID string,
fileCount int,
) {
t.Helper()
ctx := context.Background()
start := time.Now()
for i := range fileCount {
file := &File{
Path: types.FilePath(fmt.Sprintf("/large/file%05d.txt", i)),
MTime: time.Now(),
Size: int64(i * 1024),
Mode: 0644,
UID: uint32(1000 + (i % 10)),
GID: uint32(1000 + (i % 10)),
}
err := repos.Files.Create(ctx, nil, file)
if err != nil {
t.Fatalf("failed to create file %d: %v", i, err)
}
// Add half to snapshot
if i%2 == 0 {
err = repos.Snapshots.AddFileByID(ctx, nil, snapshotID, file.ID)
if err != nil {
t.Fatal(err)
}
}
}
t.Logf("Created %d files in %v", fileCount, time.Since(start))
}
// TestLargeDatasets tests operations with large amounts of data
//
//nolint:tparallel // subtests share one database and are order-dependent
func TestLargeDatasets(t *testing.T) {
t.Parallel()
if testing.Short() {
t.Skip("skipping large dataset test in short mode")
}
@@ -322,9 +425,10 @@ func TestLargeDatasets(t *testing.T) {
// Create a snapshot
snapshot := &Snapshot{
ID: "large-dataset-test",
Hostname: "test-host",
Hostname: internalTestHost,
StartedAt: time.Now(),
}
err := repos.Snapshots.Create(ctx, nil, snapshot)
if err != nil {
t.Fatal(err)
@@ -332,56 +436,39 @@ func TestLargeDatasets(t *testing.T) {
// Create many files
const fileCount = 1000
fileIDs := make([]types.FileID, fileCount)
//nolint:paralleltest // phases share one database and are order-dependent
t.Run("create many files", func(t *testing.T) {
start := time.Now()
for i := 0; i < fileCount; i++ {
file := &File{
Path: types.FilePath(fmt.Sprintf("/large/file%05d.txt", i)),
MTime: time.Now(),
Size: int64(i * 1024),
Mode: 0644,
UID: uint32(1000 + (i % 10)),
GID: uint32(1000 + (i % 10)),
}
err := repos.Files.Create(ctx, nil, file)
if err != nil {
t.Fatalf("failed to create file %d: %v", i, err)
}
fileIDs[i] = file.ID
// Add half to snapshot
if i%2 == 0 {
err = repos.Snapshots.AddFileByID(ctx, nil, snapshot.ID.String(), file.ID)
if err != nil {
t.Fatal(err)
}
}
}
t.Logf("Created %d files in %v", fileCount, time.Since(start))
createLargeDatasetFiles(t, repos, snapshot.ID.String(), fileCount)
})
// Test ListByPrefix performance
//nolint:paralleltest // phases share one database and are order-dependent
t.Run("list by prefix performance", func(t *testing.T) {
start := time.Now()
files, err := repos.Files.ListByPrefix(ctx, "/large/")
if err != nil {
t.Fatal(err)
}
if len(files) != fileCount {
t.Errorf("expected %d files, got %d", fileCount, len(files))
}
t.Logf("Listed %d files in %v", len(files), time.Since(start))
})
// Test orphaned cleanup performance
//nolint:paralleltest // phases share one database and are order-dependent
t.Run("orphaned cleanup performance", func(t *testing.T) {
start := time.Now()
err := repos.Files.DeleteOrphaned(ctx)
if err != nil {
t.Fatal(err)
}
t.Logf("Cleaned up orphaned files in %v", time.Since(start))
// Verify correct number remain
@@ -389,26 +476,33 @@ func TestLargeDatasets(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if len(files) != fileCount/2 {
t.Errorf("expected %d files after cleanup, got %d", fileCount/2, len(files))
t.Errorf("expected %d files after cleanup, got %d",
fileCount/2, len(files))
}
})
}
// TestErrorPropagation tests that errors are properly propagated
func TestErrorPropagation(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
t.Cleanup(cleanup)
ctx := context.Background()
repos := NewRepositories(db)
// Test GetByID with non-existent ID
t.Run("GetByID non-existent", func(t *testing.T) {
t.Parallel()
file, err := repos.Files.GetByID(ctx, types.NewFileID())
if err != nil {
t.Errorf("GetByID should not return error for non-existent ID, got: %v", err)
}
if file != nil {
t.Error("expected nil file for non-existent ID")
}
@@ -416,10 +510,14 @@ func TestErrorPropagation(t *testing.T) {
// Test GetByPath with non-existent path
t.Run("GetByPath non-existent", func(t *testing.T) {
t.Parallel()
file, err := repos.Files.GetByPath(ctx, "/non/existent/path.txt")
if err != nil {
t.Errorf("GetByPath should not return error for non-existent path, got: %v", err)
t.Errorf("GetByPath should not return error for non-existent path, got: %v",
err)
}
if file != nil {
t.Error("expected nil file for non-existent path")
}
@@ -427,15 +525,19 @@ func TestErrorPropagation(t *testing.T) {
// Test invalid foreign key reference
t.Run("invalid foreign key", func(t *testing.T) {
t.Parallel()
fc := &FileChunk{
FileID: types.NewFileID(),
Idx: 0,
ChunkHash: types.ChunkHash("some-chunk"),
}
err := repos.FileChunks.Create(ctx, nil, fc)
if err == nil {
t.Error("expected error for invalid foreign key")
}
if !strings.Contains(err.Error(), "FOREIGN KEY") {
t.Errorf("expected foreign key error, got: %v", err)
}
@@ -444,8 +546,10 @@ func TestErrorPropagation(t *testing.T) {
// TestQueryInjection tests that the system is safe from SQL injection
func TestQueryInjection(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
t.Cleanup(cleanup)
ctx := context.Background()
repos := NewRepositories(db)
@@ -460,6 +564,8 @@ func TestQueryInjection(t *testing.T) {
for _, injection := range injectionTests {
t.Run("injection attempt", func(t *testing.T) {
t.Parallel()
// Try injection in file path
file := &File{
Path: types.FilePath(injection),
@@ -475,7 +581,8 @@ func TestQueryInjection(t *testing.T) {
// Verify tables still exist
var count int
err := db.conn.QueryRow("SELECT COUNT(*) FROM files").Scan(&count)
err := db.conn.QueryRowContext(ctx, countFilesQuery).Scan(&count)
if err != nil {
t.Fatal("files table was damaged by injection")
}
@@ -485,6 +592,8 @@ func TestQueryInjection(t *testing.T) {
// TestTimezoneHandling tests that times are properly handled in UTC
func TestTimezoneHandling(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()

View File

@@ -3,43 +3,60 @@ package database
import (
"context"
"database/sql"
"errors"
"fmt"
"strings"
"time"
"sneak.berlin/go/vaultik/internal/types"
)
// SnapshotRepository provides access to the snapshots table and its
// snapshot_files / snapshot_blobs association tables.
type SnapshotRepository struct {
db *DB
}
// NewSnapshotRepository creates a SnapshotRepository backed by db.
func NewSnapshotRepository(db *DB) *SnapshotRepository {
return &SnapshotRepository{db: db}
}
func (r *SnapshotRepository) Create(ctx context.Context, tx *sql.Tx, snapshot *Snapshot) error {
// Create inserts a snapshot row, using tx when non-nil.
func (r *SnapshotRepository) Create(
ctx context.Context, tx *sql.Tx, snapshot *Snapshot,
) error {
query := `
INSERT INTO snapshots (id, hostname, vaultik_version, vaultik_git_revision, started_at, completed_at,
file_count, chunk_count, blob_count, total_size, blob_size, blob_uncompressed_size,
compression_ratio, compression_level, upload_bytes, upload_duration_ms)
INSERT INTO snapshots (id, hostname, vaultik_version,
vaultik_git_revision, started_at, completed_at,
file_count, chunk_count, blob_count, total_size, blob_size,
blob_uncompressed_size, compression_ratio, compression_level,
upload_bytes, upload_duration_ms)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`
var completedAt *int64
if snapshot.CompletedAt != nil {
ts := snapshot.CompletedAt.Unix()
completedAt = &ts
}
args := []any{
snapshot.ID, snapshot.Hostname, snapshot.VaultikVersion,
snapshot.VaultikGitRevision, snapshot.StartedAt.Unix(),
completedAt, snapshot.FileCount, snapshot.ChunkCount,
snapshot.BlobCount, snapshot.TotalSize, snapshot.BlobSize,
snapshot.BlobUncompressedSize, snapshot.CompressionRatio,
snapshot.CompressionLevel, snapshot.UploadBytes,
snapshot.UploadDurationMs,
}
var err error
if tx != nil {
_, err = tx.ExecContext(ctx, query, snapshot.ID, snapshot.Hostname, snapshot.VaultikVersion, snapshot.VaultikGitRevision, snapshot.StartedAt.Unix(),
completedAt, snapshot.FileCount, snapshot.ChunkCount, snapshot.BlobCount, snapshot.TotalSize, snapshot.BlobSize, snapshot.BlobUncompressedSize,
snapshot.CompressionRatio, snapshot.CompressionLevel, snapshot.UploadBytes, snapshot.UploadDurationMs)
_, err = tx.ExecContext(ctx, query, args...)
} else {
_, err = r.db.ExecWithLog(ctx, query, snapshot.ID, snapshot.Hostname, snapshot.VaultikVersion, snapshot.VaultikGitRevision, snapshot.StartedAt.Unix(),
completedAt, snapshot.FileCount, snapshot.ChunkCount, snapshot.BlobCount, snapshot.TotalSize, snapshot.BlobSize, snapshot.BlobUncompressedSize,
snapshot.CompressionRatio, snapshot.CompressionLevel, snapshot.UploadBytes, snapshot.UploadDurationMs)
_, err = r.db.ExecWithLog(ctx, query, args...)
}
if err != nil {
@@ -49,7 +66,14 @@ func (r *SnapshotRepository) Create(ctx context.Context, tx *sql.Tx, snapshot *S
return nil
}
func (r *SnapshotRepository) UpdateCounts(ctx context.Context, tx *sql.Tx, snapshotID string, fileCount, chunkCount, blobCount, totalSize, blobSize int64) error {
// UpdateCounts updates a snapshot's file/chunk/blob counters and sizes,
// recomputing the compression ratio, using tx when non-nil.
func (r *SnapshotRepository) UpdateCounts(
ctx context.Context,
tx *sql.Tx,
snapshotID string,
fileCount, chunkCount, blobCount, totalSize, blobSize int64,
) error {
compressionRatio := 1.0
if totalSize > 0 {
compressionRatio = float64(blobSize) / float64(totalSize)
@@ -68,9 +92,13 @@ func (r *SnapshotRepository) UpdateCounts(ctx context.Context, tx *sql.Tx, snaps
var err error
if tx != nil {
_, err = tx.ExecContext(ctx, query, fileCount, chunkCount, blobCount, totalSize, blobSize, compressionRatio, snapshotID)
_, err = tx.ExecContext(ctx, query,
fileCount, chunkCount, blobCount, totalSize, blobSize,
compressionRatio, snapshotID)
} else {
_, err = r.db.ExecWithLog(ctx, query, fileCount, chunkCount, blobCount, totalSize, blobSize, compressionRatio, snapshotID)
_, err = r.db.ExecWithLog(ctx, query,
fileCount, chunkCount, blobCount, totalSize, blobSize,
compressionRatio, snapshotID)
}
if err != nil {
@@ -81,27 +109,19 @@ func (r *SnapshotRepository) UpdateCounts(ctx context.Context, tx *sql.Tx, snaps
}
// UpdateExtendedStats updates extended statistics for a snapshot
func (r *SnapshotRepository) UpdateExtendedStats(ctx context.Context, tx *sql.Tx, snapshotID string, blobUncompressedSize int64, compressionLevel int, uploadDurationMs int64) error {
// Calculate compression ratio based on uncompressed vs compressed sizes
var compressionRatio float64
if blobUncompressedSize > 0 {
// Get current blob_size from DB to calculate ratio
var blobSize int64
queryGet := `SELECT blob_size FROM snapshots WHERE id = ?`
if tx != nil {
err := tx.QueryRowContext(ctx, queryGet, snapshotID).Scan(&blobSize)
func (r *SnapshotRepository) UpdateExtendedStats(
ctx context.Context,
tx *sql.Tx,
snapshotID string,
blobUncompressedSize int64,
compressionLevel int,
uploadDurationMs int64,
) error {
compressionRatio, err := r.extendedCompressionRatio(
ctx, tx, snapshotID, blobUncompressedSize,
)
if err != nil {
return fmt.Errorf("getting blob size: %w", err)
}
} else {
err := r.db.conn.QueryRowContext(ctx, queryGet, snapshotID).Scan(&blobSize)
if err != nil {
return fmt.Errorf("getting blob size: %w", err)
}
}
compressionRatio = float64(blobSize) / float64(blobUncompressedSize)
} else {
compressionRatio = 1.0
return err
}
query := `
@@ -114,20 +134,28 @@ func (r *SnapshotRepository) UpdateExtendedStats(ctx context.Context, tx *sql.Tx
WHERE id = ?
`
var err error
if tx != nil {
_, err = tx.ExecContext(ctx, query, blobUncompressedSize, compressionRatio, compressionLevel, uploadDurationMs, snapshotID)
_, err = tx.ExecContext(ctx, query,
blobUncompressedSize, compressionRatio, compressionLevel,
uploadDurationMs, snapshotID)
} else {
_, err = r.db.ExecWithLog(ctx, query, blobUncompressedSize, compressionRatio, compressionLevel, uploadDurationMs, snapshotID)
_, err = r.db.ExecWithLog(ctx, query,
blobUncompressedSize, compressionRatio, compressionLevel,
uploadDurationMs, snapshotID)
}
if err != nil {
return fmt.Errorf("updating extended stats: %w", err)
}
return nil
}
func (r *SnapshotRepository) GetByID(ctx context.Context, snapshotID string) (*Snapshot, error) {
// GetByID returns the snapshot with the given ID, or nil if no such
// snapshot exists.
func (r *SnapshotRepository) GetByID(
ctx context.Context, snapshotID string,
) (*Snapshot, error) {
query := `
SELECT id, hostname, vaultik_version, vaultik_git_revision, started_at, completed_at,
file_count, chunk_count, blob_count, total_size, blob_size, blob_uncompressed_size,
@@ -136,9 +164,11 @@ func (r *SnapshotRepository) GetByID(ctx context.Context, snapshotID string) (*S
WHERE id = ?
`
var snapshot Snapshot
var startedAtUnix int64
var completedAtUnix *int64
var (
snapshot Snapshot
startedAtUnix int64
completedAtUnix *int64
)
err := r.db.conn.QueryRowContext(ctx, query, snapshotID).Scan(
&snapshot.ID,
@@ -159,9 +189,10 @@ func (r *SnapshotRepository) GetByID(ctx context.Context, snapshotID string) (*S
&snapshot.UploadDurationMs,
)
if err == sql.ErrNoRows {
return nil, nil
if errors.Is(err, sql.ErrNoRows) {
return nil, nil //nolint:nilnil // nil,nil signals not-found; callers check nil
}
if err != nil {
return nil, fmt.Errorf("querying snapshot: %w", err)
}
@@ -175,9 +206,14 @@ func (r *SnapshotRepository) GetByID(ctx context.Context, snapshotID string) (*S
return &snapshot, nil
}
func (r *SnapshotRepository) ListRecent(ctx context.Context, limit int) ([]*Snapshot, error) {
// ListRecent returns up to limit snapshots, most recently started first.
func (r *SnapshotRepository) ListRecent(
ctx context.Context, limit int,
) ([]*Snapshot, error) {
query := `
SELECT id, hostname, vaultik_version, vaultik_git_revision, started_at, completed_at, file_count, chunk_count, blob_count, total_size, blob_size, compression_ratio
SELECT id, hostname, vaultik_version, vaultik_git_revision,
started_at, completed_at, file_count, chunk_count, blob_count,
total_size, blob_size, compression_ratio
FROM snapshots
ORDER BY started_at DESC
LIMIT ?
@@ -189,44 +225,13 @@ func (r *SnapshotRepository) ListRecent(ctx context.Context, limit int) ([]*Snap
}
defer CloseRows(rows)
var snapshots []*Snapshot
for rows.Next() {
var snapshot Snapshot
var startedAtUnix int64
var completedAtUnix *int64
err := rows.Scan(
&snapshot.ID,
&snapshot.Hostname,
&snapshot.VaultikVersion,
&snapshot.VaultikGitRevision,
&startedAtUnix,
&completedAtUnix,
&snapshot.FileCount,
&snapshot.ChunkCount,
&snapshot.BlobCount,
&snapshot.TotalSize,
&snapshot.BlobSize,
&snapshot.CompressionRatio,
)
if err != nil {
return nil, fmt.Errorf("scanning snapshot: %w", err)
}
snapshot.StartedAt = time.Unix(startedAtUnix, 0)
if completedAtUnix != nil {
t := time.Unix(*completedAtUnix, 0)
snapshot.CompletedAt = &t
}
snapshots = append(snapshots, &snapshot)
}
return snapshots, rows.Err()
return r.scanSnapshotRows(rows)
}
// MarkComplete marks a snapshot as completed with the current timestamp
func (r *SnapshotRepository) MarkComplete(ctx context.Context, tx *sql.Tx, snapshotID string) error {
func (r *SnapshotRepository) MarkComplete(
ctx context.Context, tx *sql.Tx, snapshotID string,
) error {
query := `
UPDATE snapshots
SET completed_at = ?
@@ -250,7 +255,9 @@ func (r *SnapshotRepository) MarkComplete(ctx context.Context, tx *sql.Tx, snaps
}
// AddFile adds a file to a snapshot
func (r *SnapshotRepository) AddFile(ctx context.Context, tx *sql.Tx, snapshotID string, filePath string) error {
func (r *SnapshotRepository) AddFile(
ctx context.Context, tx *sql.Tx, snapshotID string, filePath string,
) error {
query := `
INSERT OR IGNORE INTO snapshot_files (snapshot_id, file_id)
SELECT ?, id FROM files WHERE path = ?
@@ -271,7 +278,9 @@ func (r *SnapshotRepository) AddFile(ctx context.Context, tx *sql.Tx, snapshotID
}
// AddFileByID adds a file to a snapshot by file ID
func (r *SnapshotRepository) AddFileByID(ctx context.Context, tx *sql.Tx, snapshotID string, fileID types.FileID) error {
func (r *SnapshotRepository) AddFileByID(
ctx context.Context, tx *sql.Tx, snapshotID string, fileID types.FileID,
) error {
query := `
INSERT OR IGNORE INTO snapshot_files (snapshot_id, file_id)
VALUES (?, ?)
@@ -292,37 +301,49 @@ func (r *SnapshotRepository) AddFileByID(ctx context.Context, tx *sql.Tx, snapsh
}
// AddFilesByIDBatch adds multiple files to a snapshot in batched inserts
func (r *SnapshotRepository) AddFilesByIDBatch(ctx context.Context, tx *sql.Tx, snapshotID string, fileIDs []types.FileID) error {
func (r *SnapshotRepository) AddFilesByIDBatch(
ctx context.Context, tx *sql.Tx, snapshotID string, fileIDs []types.FileID,
) error {
if len(fileIDs) == 0 {
return nil
}
// Each entry has 2 values, so batch at 400 to be safe
// Each snapshot_files row binds this many SQL variables.
const snapshotFileCols = 2
// Batch at 400 rows to be safe with SQLite's variable limit.
const batchSize = 400
for i := 0; i < len(fileIDs); i += batchSize {
end := i + batchSize
if end > len(fileIDs) {
end = len(fileIDs)
}
end := min(i+batchSize, len(fileIDs))
batch := fileIDs[i:end]
query := "INSERT OR IGNORE INTO snapshot_files (snapshot_id, file_id) VALUES "
args := make([]interface{}, 0, len(batch)*2)
args := make([]any, 0, len(batch)*snapshotFileCols)
var querySb312 strings.Builder
for j, fileID := range batch {
if j > 0 {
query += ", "
querySb312.WriteString(", ")
}
query += "(?, ?)"
querySb312.WriteString("(?, ?)")
args = append(args, snapshotID, fileID.String())
}
query += querySb312.String() //nolint:gosec // G202: appends "?" placeholders only
var err error
if tx != nil {
_, err = tx.ExecContext(ctx, query, args...)
} else {
_, err = r.db.ExecWithLog(ctx, query, args...)
}
if err != nil {
return fmt.Errorf("batch adding files to snapshot: %w", err)
}
@@ -341,7 +362,9 @@ func (r *SnapshotRepository) AddFilesByIDBatch(ctx context.Context, tx *sql.Tx,
// Returns the number of rows inserted (i.e. blobs that were previously
// referenced indirectly via file_chunks but not yet recorded in
// snapshot_blobs for this snapshot).
func (r *SnapshotRepository) PopulateReferencedBlobs(ctx context.Context, tx *sql.Tx, snapshotID string) (int64, error) {
func (r *SnapshotRepository) PopulateReferencedBlobs(
ctx context.Context, tx *sql.Tx, snapshotID string,
) (int64, error) {
query := `
INSERT OR IGNORE INTO snapshot_blobs (snapshot_id, blob_id, blob_hash)
SELECT DISTINCT ?, blobs.id, blobs.blob_hash
@@ -353,23 +376,33 @@ func (r *SnapshotRepository) PopulateReferencedBlobs(ctx context.Context, tx *sq
AND blobs.blob_hash IS NOT NULL
`
var result sql.Result
var err error
var (
result sql.Result
err error
)
if tx != nil {
result, err = tx.ExecContext(ctx, query, snapshotID, snapshotID)
} else {
result, err = r.db.ExecWithLog(ctx, query, snapshotID, snapshotID)
}
if err != nil {
return 0, fmt.Errorf("populating referenced blobs: %w", err)
}
n, _ := result.RowsAffected()
return n, nil
}
// AddBlob adds a blob to a snapshot
func (r *SnapshotRepository) AddBlob(ctx context.Context, tx *sql.Tx, snapshotID string, blobID types.BlobID, blobHash types.BlobHash) error {
func (r *SnapshotRepository) AddBlob(
ctx context.Context,
tx *sql.Tx,
snapshotID string,
blobID types.BlobID,
blobHash types.BlobHash,
) error {
query := `
INSERT OR IGNORE INTO snapshot_blobs (snapshot_id, blob_id, blob_hash)
VALUES (?, ?, ?)
@@ -390,7 +423,9 @@ func (r *SnapshotRepository) AddBlob(ctx context.Context, tx *sql.Tx, snapshotID
}
// GetBlobHashes returns all blob hashes for a snapshot
func (r *SnapshotRepository) GetBlobHashes(ctx context.Context, snapshotID string) ([]string, error) {
func (r *SnapshotRepository) GetBlobHashes(
ctx context.Context, snapshotID string,
) ([]string, error) {
query := `
SELECT sb.blob_hash
FROM snapshot_blobs sb
@@ -405,19 +440,26 @@ func (r *SnapshotRepository) GetBlobHashes(ctx context.Context, snapshotID strin
defer CloseRows(rows)
var blobs []string
for rows.Next() {
var blobHash string
if err := rows.Scan(&blobHash); err != nil {
err := rows.Scan(&blobHash)
if err != nil {
return nil, fmt.Errorf("scanning blob hash: %w", err)
}
blobs = append(blobs, blobHash)
}
return blobs, rows.Err()
}
// GetSnapshotTotalCompressedSize returns the total compressed size of all blobs referenced by a snapshot
func (r *SnapshotRepository) GetSnapshotTotalCompressedSize(ctx context.Context, snapshotID string) (int64, error) {
// GetSnapshotTotalCompressedSize returns the total compressed size of all
// blobs referenced by a snapshot.
func (r *SnapshotRepository) GetSnapshotTotalCompressedSize(
ctx context.Context, snapshotID string,
) (int64, error) {
query := `
SELECT COALESCE(SUM(b.compressed_size), 0)
FROM snapshot_blobs sb
@@ -426,6 +468,7 @@ func (r *SnapshotRepository) GetSnapshotTotalCompressedSize(ctx context.Context,
`
var totalSize int64
err := r.db.conn.QueryRowContext(ctx, query, snapshotID).Scan(&totalSize)
if err != nil {
return 0, fmt.Errorf("querying total compressed size: %w", err)
@@ -436,7 +479,9 @@ func (r *SnapshotRepository) GetSnapshotTotalCompressedSize(ctx context.Context,
// GetSnapshotUncompressedChunkSize returns the sum of plaintext sizes of all unique
// chunks referenced by a snapshot (via snapshot_files → file_chunks → chunks).
func (r *SnapshotRepository) GetSnapshotUncompressedChunkSize(ctx context.Context, snapshotID string) (int64, error) {
func (r *SnapshotRepository) GetSnapshotUncompressedChunkSize(
ctx context.Context, snapshotID string,
) (int64, error) {
query := `
SELECT COALESCE(SUM(c.size), 0)
FROM (
@@ -449,6 +494,7 @@ func (r *SnapshotRepository) GetSnapshotUncompressedChunkSize(ctx context.Contex
`
var totalSize int64
err := r.db.conn.QueryRowContext(ctx, query, snapshotID).Scan(&totalSize)
if err != nil {
return 0, fmt.Errorf("querying uncompressed chunk size: %w", err)
@@ -461,7 +507,9 @@ func (r *SnapshotRepository) GetSnapshotUncompressedChunkSize(ctx context.Contex
// referenced by this snapshot but not by any earlier completed snapshot known to
// the local database. The result is the marginal uncompressed data this snapshot
// added to the dedup pool — i.e., the delta from prior snapshots.
func (r *SnapshotRepository) GetSnapshotNewChunkSize(ctx context.Context, snapshotID string) (int64, error) {
func (r *SnapshotRepository) GetSnapshotNewChunkSize(
ctx context.Context, snapshotID string,
) (int64, error) {
query := `
WITH this_snap_chunks AS (
SELECT DISTINCT fc.chunk_hash
@@ -485,7 +533,10 @@ func (r *SnapshotRepository) GetSnapshotNewChunkSize(ctx context.Context, snapsh
`
var totalSize int64
err := r.db.conn.QueryRowContext(ctx, query, snapshotID, snapshotID, snapshotID).Scan(&totalSize)
err := r.db.conn.QueryRowContext(
ctx, query, snapshotID, snapshotID, snapshotID,
).Scan(&totalSize)
if err != nil {
return 0, fmt.Errorf("querying new chunk size: %w", err)
}
@@ -494,9 +545,13 @@ func (r *SnapshotRepository) GetSnapshotNewChunkSize(ctx context.Context, snapsh
}
// GetIncompleteSnapshots returns all snapshots that haven't been completed
func (r *SnapshotRepository) GetIncompleteSnapshots(ctx context.Context) ([]*Snapshot, error) {
func (r *SnapshotRepository) GetIncompleteSnapshots(
ctx context.Context,
) ([]*Snapshot, error) {
query := `
SELECT id, hostname, vaultik_version, vaultik_git_revision, started_at, completed_at, file_count, chunk_count, blob_count, total_size, blob_size, compression_ratio
SELECT id, hostname, vaultik_version, vaultik_git_revision,
started_at, completed_at, file_count, chunk_count, blob_count,
total_size, blob_size, compression_ratio
FROM snapshots
WHERE completed_at IS NULL
ORDER BY started_at DESC
@@ -508,46 +563,17 @@ func (r *SnapshotRepository) GetIncompleteSnapshots(ctx context.Context) ([]*Sna
}
defer CloseRows(rows)
var snapshots []*Snapshot
for rows.Next() {
var snapshot Snapshot
var startedAtUnix int64
var completedAtUnix *int64
err := rows.Scan(
&snapshot.ID,
&snapshot.Hostname,
&snapshot.VaultikVersion,
&snapshot.VaultikGitRevision,
&startedAtUnix,
&completedAtUnix,
&snapshot.FileCount,
&snapshot.ChunkCount,
&snapshot.BlobCount,
&snapshot.TotalSize,
&snapshot.BlobSize,
&snapshot.CompressionRatio,
)
if err != nil {
return nil, fmt.Errorf("scanning snapshot: %w", err)
}
snapshot.StartedAt = time.Unix(startedAtUnix, 0)
if completedAtUnix != nil {
t := time.Unix(*completedAtUnix, 0)
snapshot.CompletedAt = &t
}
snapshots = append(snapshots, &snapshot)
}
return snapshots, rows.Err()
return r.scanSnapshotRows(rows)
}
// GetIncompleteByHostname returns all incomplete snapshots for a specific hostname
func (r *SnapshotRepository) GetIncompleteByHostname(ctx context.Context, hostname string) ([]*Snapshot, error) {
func (r *SnapshotRepository) GetIncompleteByHostname(
ctx context.Context, hostname string,
) ([]*Snapshot, error) {
query := `
SELECT id, hostname, vaultik_version, vaultik_git_revision, started_at, completed_at, file_count, chunk_count, blob_count, total_size, blob_size, compression_ratio
SELECT id, hostname, vaultik_version, vaultik_git_revision,
started_at, completed_at, file_count, chunk_count, blob_count,
total_size, blob_size, compression_ratio
FROM snapshots
WHERE completed_at IS NULL AND hostname = ?
ORDER BY started_at DESC
@@ -560,10 +586,13 @@ func (r *SnapshotRepository) GetIncompleteByHostname(ctx context.Context, hostna
defer CloseRows(rows)
var snapshots []*Snapshot
for rows.Next() {
var snapshot Snapshot
var startedAtUnix int64
var completedAtUnix *int64
var (
snapshot Snapshot
startedAtUnix int64
completedAtUnix *int64
)
err := rows.Scan(
&snapshot.ID,
@@ -608,7 +637,9 @@ func (r *SnapshotRepository) Delete(ctx context.Context, snapshotID string) erro
}
// DeleteSnapshotFiles removes all snapshot_files entries for a snapshot
func (r *SnapshotRepository) DeleteSnapshotFiles(ctx context.Context, snapshotID string) error {
func (r *SnapshotRepository) DeleteSnapshotFiles(
ctx context.Context, snapshotID string,
) error {
query := `DELETE FROM snapshot_files WHERE snapshot_id = ?`
_, err := r.db.ExecWithLog(ctx, query, snapshotID)
@@ -620,7 +651,9 @@ func (r *SnapshotRepository) DeleteSnapshotFiles(ctx context.Context, snapshotID
}
// DeleteSnapshotBlobs removes all snapshot_blobs entries for a snapshot
func (r *SnapshotRepository) DeleteSnapshotBlobs(ctx context.Context, snapshotID string) error {
func (r *SnapshotRepository) DeleteSnapshotBlobs(
ctx context.Context, snapshotID string,
) error {
query := `DELETE FROM snapshot_blobs WHERE snapshot_id = ?`
_, err := r.db.ExecWithLog(ctx, query, snapshotID)
@@ -632,7 +665,9 @@ func (r *SnapshotRepository) DeleteSnapshotBlobs(ctx context.Context, snapshotID
}
// DeleteSnapshotUploads removes all uploads entries for a snapshot
func (r *SnapshotRepository) DeleteSnapshotUploads(ctx context.Context, snapshotID string) error {
func (r *SnapshotRepository) DeleteSnapshotUploads(
ctx context.Context, snapshotID string,
) error {
query := `DELETE FROM uploads WHERE snapshot_id = ?`
_, err := r.db.ExecWithLog(ctx, query, snapshotID)
@@ -642,3 +677,77 @@ func (r *SnapshotRepository) DeleteSnapshotUploads(ctx context.Context, snapshot
return nil
}
// extendedCompressionRatio computes the compression ratio for a snapshot
// from its stored blob_size and the given uncompressed size. Returns 1.0
// when the uncompressed size is zero.
func (r *SnapshotRepository) extendedCompressionRatio(
ctx context.Context,
tx *sql.Tx,
snapshotID string,
blobUncompressedSize int64,
) (float64, error) {
if blobUncompressedSize <= 0 {
return 1.0, nil
}
// Get current blob_size from DB to calculate ratio
var blobSize int64
queryGet := `SELECT blob_size FROM snapshots WHERE id = ?`
var err error
if tx != nil {
err = tx.QueryRowContext(ctx, queryGet, snapshotID).Scan(&blobSize)
} else {
err = r.db.conn.QueryRowContext(ctx, queryGet, snapshotID).Scan(&blobSize)
}
if err != nil {
return 0, fmt.Errorf("getting blob size: %w", err)
}
return float64(blobSize) / float64(blobUncompressedSize), nil
}
// scanSnapshotRows scans the standard snapshot column set from a rows
// iterator into Snapshot records.
func (r *SnapshotRepository) scanSnapshotRows(rows *sql.Rows) ([]*Snapshot, error) {
var snapshots []*Snapshot
for rows.Next() {
var (
snapshot Snapshot
startedAtUnix int64
completedAtUnix *int64
)
err := rows.Scan(
&snapshot.ID,
&snapshot.Hostname,
&snapshot.VaultikVersion,
&snapshot.VaultikGitRevision,
&startedAtUnix,
&completedAtUnix,
&snapshot.FileCount,
&snapshot.ChunkCount,
&snapshot.BlobCount,
&snapshot.TotalSize,
&snapshot.BlobSize,
&snapshot.CompressionRatio,
)
if err != nil {
return nil, fmt.Errorf("scanning snapshot: %w", err)
}
snapshot.StartedAt = time.Unix(startedAtUnix, 0)
if completedAtUnix != nil {
t := time.Unix(*completedAtUnix, 0)
snapshot.CompletedAt = &t
}
snapshots = append(snapshots, &snapshot)
}
return snapshots, rows.Err()
}

View File

@@ -1,4 +1,4 @@
package database
package database_test
import (
"context"
@@ -7,6 +7,7 @@ import (
"testing"
"time"
"sneak.berlin/go/vaultik/internal/database"
"sneak.berlin/go/vaultik/internal/types"
)
@@ -21,17 +22,19 @@ const (
)
func TestSnapshotRepository(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repo := NewSnapshotRepository(db)
repo := database.NewSnapshotRepository(db)
// Test Create
snapshot := &Snapshot{
snapshot := &database.Snapshot{
ID: "2024-01-01T12:00:00Z",
Hostname: "test-host",
VaultikVersion: "1.0.0",
Hostname: testHostname,
VaultikVersion: testVersion,
StartedAt: time.Now().Truncate(time.Second),
CompletedAt: nil,
FileCount: 100,
@@ -52,62 +55,118 @@ func TestSnapshotRepository(t *testing.T) {
if err != nil {
t.Fatalf("failed to get snapshot: %v", err)
}
if retrieved == nil {
t.Fatal("expected snapshot, got nil")
}
if retrieved.ID != snapshot.ID {
t.Errorf("ID mismatch: got %s, want %s", retrieved.ID, snapshot.ID)
}
if retrieved.Hostname != snapshot.Hostname {
t.Errorf("hostname mismatch: got %s, want %s", retrieved.Hostname, snapshot.Hostname)
t.Errorf("hostname mismatch: got %s, want %s",
retrieved.Hostname, snapshot.Hostname)
}
if retrieved.FileCount != snapshot.FileCount {
t.Errorf("file count mismatch: got %d, want %d", retrieved.FileCount, snapshot.FileCount)
t.Errorf("file count mismatch: got %d, want %d",
retrieved.FileCount, snapshot.FileCount)
}
}
func TestSnapshotRepositoryUpdateCounts(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repo := database.NewSnapshotRepository(db)
snapshot := &database.Snapshot{
ID: "2024-01-02T12:00:00Z",
Hostname: testHostname,
VaultikVersion: testVersion,
StartedAt: time.Now().Truncate(time.Second),
CompletedAt: nil,
FileCount: 100,
ChunkCount: 500,
BlobCount: 10,
TotalSize: oneHundredMebibytes,
BlobSize: fortyMebibytes,
CompressionRatio: compressionRatioPoint4,
}
err := repo.Create(ctx, nil, snapshot)
if err != nil {
t.Fatalf("failed to create snapshot: %v", err)
}
// Test UpdateCounts
err = repo.UpdateCounts(ctx, nil, snapshot.ID.String(), 200, 1000, 20, twoHundredMebibytes, sixtyMebibytes)
err = repo.UpdateCounts(ctx, nil, snapshot.ID.String(),
200, 1000, 20, twoHundredMebibytes, sixtyMebibytes)
if err != nil {
t.Fatalf("failed to update counts: %v", err)
}
retrieved, err = repo.GetByID(ctx, snapshot.ID.String())
retrieved, err := repo.GetByID(ctx, snapshot.ID.String())
if err != nil {
t.Fatalf("failed to get updated snapshot: %v", err)
}
if retrieved.FileCount != 200 {
t.Errorf("file count not updated: got %d, want %d", retrieved.FileCount, 200)
}
if retrieved.ChunkCount != 1000 {
t.Errorf("chunk count not updated: got %d, want %d", retrieved.ChunkCount, 1000)
t.Errorf("chunk count not updated: got %d, want %d",
retrieved.ChunkCount, 1000)
}
if retrieved.BlobCount != 20 {
t.Errorf("blob count not updated: got %d, want %d", retrieved.BlobCount, 20)
}
if retrieved.TotalSize != twoHundredMebibytes {
t.Errorf("total size not updated: got %d, want %d", retrieved.TotalSize, twoHundredMebibytes)
}
if retrieved.BlobSize != sixtyMebibytes {
t.Errorf("blob size not updated: got %d, want %d", retrieved.BlobSize, sixtyMebibytes)
}
expectedRatio := compressionRatioPoint3 // 0.3
if math.Abs(retrieved.CompressionRatio-expectedRatio) > 0.001 {
t.Errorf("compression ratio not updated: got %f, want %f", retrieved.CompressionRatio, expectedRatio)
t.Errorf("total size not updated: got %d, want %d",
retrieved.TotalSize, twoHundredMebibytes)
}
// Test ListRecent
// Add more snapshots
for i := 2; i <= 5; i++ {
s := &Snapshot{
if retrieved.BlobSize != sixtyMebibytes {
t.Errorf("blob size not updated: got %d, want %d",
retrieved.BlobSize, sixtyMebibytes)
}
expectedRatio := compressionRatioPoint3 // 0.3
if math.Abs(retrieved.CompressionRatio-expectedRatio) > 0.001 {
t.Errorf("compression ratio not updated: got %f, want %f",
retrieved.CompressionRatio, expectedRatio)
}
}
func TestSnapshotRepositoryListRecent(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repo := database.NewSnapshotRepository(db)
// Add snapshots
for i := 1; i <= 5; i++ {
s := &database.Snapshot{
ID: types.SnapshotID(fmt.Sprintf("2024-01-0%dT12:00:00Z", i)),
Hostname: "test-host",
VaultikVersion: "1.0.0",
Hostname: testHostname,
VaultikVersion: testVersion,
StartedAt: time.Now().Add(time.Duration(i) * time.Hour).Truncate(time.Second),
CompletedAt: nil,
FileCount: int64(100 * i),
ChunkCount: int64(500 * i),
BlobCount: int64(10 * i),
}
err := repo.Create(ctx, nil, s)
if err != nil {
t.Fatalf("failed to create snapshot %d: %v", i, err)
@@ -119,12 +178,13 @@ func TestSnapshotRepository(t *testing.T) {
if err != nil {
t.Fatalf("failed to list recent snapshots: %v", err)
}
if len(recent) != 3 {
t.Errorf("expected 3 recent snapshots, got %d", len(recent))
}
// Verify order (most recent first)
for i := 0; i < len(recent)-1; i++ {
for i := range len(recent) - 1 {
if recent[i].StartedAt.Before(recent[i+1].StartedAt) {
t.Error("snapshots not in descending order")
}
@@ -132,23 +192,27 @@ func TestSnapshotRepository(t *testing.T) {
}
func TestSnapshotRepositoryNotFound(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repo := NewSnapshotRepository(db)
repo := database.NewSnapshotRepository(db)
// Test GetByID with non-existent ID
snapshot, err := repo.GetByID(ctx, "nonexistent")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if snapshot != nil {
t.Error("expected nil for non-existent snapshot")
}
// Test UpdateCounts on non-existent snapshot
err = repo.UpdateCounts(ctx, nil, "nonexistent", 100, 200, 10, oneHundredMebibytes, fortyMebibytes)
err = repo.UpdateCounts(ctx, nil, "nonexistent",
100, 200, 10, oneHundredMebibytes, fortyMebibytes)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@@ -156,16 +220,18 @@ func TestSnapshotRepositoryNotFound(t *testing.T) {
}
func TestSnapshotRepositoryDuplicate(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repo := NewSnapshotRepository(db)
repo := database.NewSnapshotRepository(db)
snapshot := &Snapshot{
snapshot := &database.Snapshot{
ID: "2024-01-01T12:00:00Z",
Hostname: "test-host",
VaultikVersion: "1.0.0",
Hostname: testHostname,
VaultikVersion: testVersion,
StartedAt: time.Now().Truncate(time.Second),
CompletedAt: nil,
FileCount: 100,

View File

@@ -3,6 +3,7 @@ package database
import (
"context"
"database/sql"
"errors"
"time"
"sneak.berlin/go/vaultik/internal/log"
@@ -28,7 +29,9 @@ func NewUploadRepository(conn *sql.DB) *UploadRepository {
}
// Create inserts a new upload record
func (r *UploadRepository) Create(ctx context.Context, tx *sql.Tx, upload *Upload) error {
func (r *UploadRepository) Create(
ctx context.Context, tx *sql.Tx, upload *Upload,
) error {
query := `
INSERT INTO uploads (blob_hash, snapshot_id, uploaded_at, size, duration_ms)
VALUES (?, ?, ?, ?, ?)
@@ -36,16 +39,22 @@ func (r *UploadRepository) Create(ctx context.Context, tx *sql.Tx, upload *Uploa
var err error
if tx != nil {
_, err = tx.ExecContext(ctx, query, upload.BlobHash, upload.SnapshotID, upload.UploadedAt, upload.Size, upload.DurationMs)
_, err = tx.ExecContext(ctx, query,
upload.BlobHash, upload.SnapshotID, upload.UploadedAt,
upload.Size, upload.DurationMs)
} else {
_, err = r.conn.ExecContext(ctx, query, upload.BlobHash, upload.SnapshotID, upload.UploadedAt, upload.Size, upload.DurationMs)
_, err = r.conn.ExecContext(ctx, query,
upload.BlobHash, upload.SnapshotID, upload.UploadedAt,
upload.Size, upload.DurationMs)
}
return err
}
// GetByBlobHash retrieves an upload record by blob hash
func (r *UploadRepository) GetByBlobHash(ctx context.Context, blobHash string) (*Upload, error) {
func (r *UploadRepository) GetByBlobHash(
ctx context.Context, blobHash string,
) (*Upload, error) {
query := `
SELECT blob_hash, uploaded_at, size, duration_ms
FROM uploads
@@ -53,6 +62,7 @@ func (r *UploadRepository) GetByBlobHash(ctx context.Context, blobHash string) (
`
var upload Upload
err := r.conn.QueryRowContext(ctx, query, blobHash).Scan(
&upload.BlobHash,
&upload.UploadedAt,
@@ -60,9 +70,10 @@ func (r *UploadRepository) GetByBlobHash(ctx context.Context, blobHash string) (
&upload.DurationMs,
)
if err == sql.ErrNoRows {
return nil, nil
if errors.Is(err, sql.ErrNoRows) {
return nil, nil //nolint:nilnil // nil,nil signals not-found; callers check nil
}
if err != nil {
return nil, err
}
@@ -71,7 +82,9 @@ func (r *UploadRepository) GetByBlobHash(ctx context.Context, blobHash string) (
}
// GetRecentUploads retrieves recent uploads ordered by upload time
func (r *UploadRepository) GetRecentUploads(ctx context.Context, limit int) ([]*Upload, error) {
func (r *UploadRepository) GetRecentUploads(
ctx context.Context, limit int,
) ([]*Upload, error) {
query := `
SELECT blob_hash, uploaded_at, size, duration_ms
FROM uploads
@@ -84,17 +97,24 @@ func (r *UploadRepository) GetRecentUploads(ctx context.Context, limit int) ([]*
return nil, err
}
defer func() {
if err := rows.Close(); err != nil {
err := rows.Close()
if err != nil {
log.Error("failed to close rows", "error", err)
}
}()
var uploads []*Upload
for rows.Next() {
var upload Upload
if err := rows.Scan(&upload.BlobHash, &upload.UploadedAt, &upload.Size, &upload.DurationMs); err != nil {
err := rows.Scan(
&upload.BlobHash, &upload.UploadedAt, &upload.Size, &upload.DurationMs,
)
if err != nil {
return nil, err
}
uploads = append(uploads, &upload)
}
@@ -102,7 +122,9 @@ func (r *UploadRepository) GetRecentUploads(ctx context.Context, limit int) ([]*
}
// GetUploadStats returns aggregate statistics for uploads
func (r *UploadRepository) GetUploadStats(ctx context.Context, since time.Time) (*UploadStats, error) {
func (r *UploadRepository) GetUploadStats(
ctx context.Context, since time.Time,
) (*UploadStats, error) {
query := `
SELECT
COUNT(*) as count,
@@ -115,6 +137,7 @@ func (r *UploadRepository) GetUploadStats(ctx context.Context, since time.Time)
`
var stats UploadStats
err := r.conn.QueryRowContext(ctx, query, since).Scan(
&stats.Count,
&stats.TotalSize,
@@ -136,12 +159,17 @@ type UploadStats struct {
}
// GetCountBySnapshot returns the count of uploads for a specific snapshot
func (r *UploadRepository) GetCountBySnapshot(ctx context.Context, snapshotID string) (int64, error) {
func (r *UploadRepository) GetCountBySnapshot(
ctx context.Context, snapshotID string,
) (int64, error) {
query := `SELECT COUNT(*) FROM uploads WHERE snapshot_id = ?`
var count int64
err := r.conn.QueryRowContext(ctx, query, snapshotID).Scan(&count)
if err != nil {
return 0, err
}
return count, nil
}

View File

@@ -1,3 +1,5 @@
// Package globals holds application-wide metadata (name, version,
// commit) that is populated at build time via linker flags.
package globals
import (
@@ -5,16 +7,16 @@ import (
)
// Appname is the application name, populated from main().
var Appname string = "vaultik"
var Appname = "vaultik" //nolint:gochecknoglobals // set via -ldflags at build time
// Version is the application version, populated from main().
var Version string = "dev"
var Version = "dev" //nolint:gochecknoglobals // set via -ldflags at build time
// Commit is the git commit hash, populated from main().
var Commit string = "unknown"
var Commit = "unknown" //nolint:gochecknoglobals // set via -ldflags at build time
// CommitDate is the ISO-8601 date of the commit, populated from main().
var CommitDate string = "unknown"
var CommitDate = "unknown" //nolint:gochecknoglobals // set via -ldflags at build time
// Author identifies the upstream author of vaultik.
const Author = "Jeffrey Paul <sneak@sneak.berlin>"
@@ -34,7 +36,8 @@ type Globals struct {
StartTime time.Time
}
// New creates and returns a new Globals instance initialized with the package-level variables.
// New creates and returns a new Globals instance initialized with the
// package-level variables.
func New() (*Globals, error) {
return &Globals{
Appname: Appname,
@@ -44,11 +47,15 @@ func New() (*Globals, error) {
}, nil
}
// shortCommitLen is the number of commit-hash characters ShortCommit keeps.
const shortCommitLen = 12
// ShortCommit returns the first 12 chars of the commit hash, or the
// whole string if it's shorter (e.g. "unknown").
func (g *Globals) ShortCommit() string {
if len(g.Commit) > 12 {
return g.Commit[:12]
if len(g.Commit) > shortCommitLen {
return g.Commit[:shortCommitLen]
}
return g.Commit
}

View File

@@ -1,12 +1,16 @@
package globals
package globals_test
import (
"testing"
"sneak.berlin/go/vaultik/internal/globals"
)
// TestGlobalsNew ensures the globals package initializes correctly
func TestGlobalsNew(t *testing.T) {
g, err := New()
t.Parallel()
g, err := globals.New()
if err != nil {
t.Fatalf("Failed to create Globals: %v", err)
}

View File

@@ -1,3 +1,5 @@
// Package log provides the application-wide structured logger: slog
// with a colorized TTY handler on terminals and JSON output otherwise.
package log
import (
@@ -12,12 +14,12 @@ import (
"golang.org/x/term"
)
// LogLevel represents the logging level.
type LogLevel int
// Level represents the logging level.
type Level int
const (
// LevelFatal represents a fatal error level that will exit the program.
LevelFatal LogLevel = iota
LevelFatal Level = iota
// LevelError represents an error level.
LevelError
// LevelWarn represents a warning level.
@@ -38,6 +40,7 @@ type Config struct {
Quiet bool
}
//nolint:gochecknoglobals // package-level logger is the package's purpose
var logger *slog.Logger
// Initialize sets up the global logger based on the provided configuration.
@@ -45,18 +48,19 @@ func Initialize(cfg Config) {
// Determine log level based on configuration
var level slog.Level
if cfg.Cron || cfg.Quiet {
switch {
case cfg.Cron || cfg.Quiet:
// In cron/quiet mode keep warnings and errors visible — the
// whole point of --cron is to stay silent only on total
// success, so that anything cron emails to root is genuinely
// "something went wrong, look at it." A backup with stuck
// permission errors or skipped files should NOT be silent.
level = slog.LevelWarn
} else if cfg.Debug || strings.Contains(os.Getenv("GODEBUG"), "vaultik") {
case cfg.Debug || strings.Contains(os.Getenv("GODEBUG"), "vaultik"):
level = slog.LevelDebug
} else if cfg.Verbose {
case cfg.Verbose:
level = slog.LevelInfo
} else {
default:
level = slog.LevelWarn
}
@@ -78,12 +82,17 @@ func Initialize(cfg Config) {
slog.SetDefault(logger)
}
// callerSkipFrames is the number of stack frames between runtime.Caller
// and the code that invoked the package-level logging function.
const callerSkipFrames = 2
// getCaller returns the caller information as a string
func getCaller(skip int) string {
_, file, line, ok := runtime.Caller(skip)
func getCaller() string {
_, file, line, ok := runtime.Caller(callerSkipFrames)
if !ok {
return "unknown"
}
return fmt.Sprintf("%s:%d", filepath.Base(file), line)
}
@@ -91,9 +100,10 @@ func getCaller(skip int) string {
func Fatal(msg string, args ...any) {
if logger != nil {
// Add caller info to args
args = append(args, "caller", getCaller(2))
args = append(args, "caller", getCaller())
logger.Error(msg, args...)
}
os.Exit(1)
}
@@ -105,7 +115,7 @@ func Fatalf(format string, args ...any) {
// Error logs an error message.
func Error(msg string, args ...any) {
if logger != nil {
args = append(args, "caller", getCaller(2))
args = append(args, "caller", getCaller())
logger.Error(msg, args...)
}
}
@@ -118,7 +128,7 @@ func Errorf(format string, args ...any) {
// Warn logs a warning message.
func Warn(msg string, args ...any) {
if logger != nil {
args = append(args, "caller", getCaller(2))
args = append(args, "caller", getCaller())
logger.Warn(msg, args...)
}
}
@@ -131,7 +141,7 @@ func Warnf(format string, args ...any) {
// Notice logs a notice message (mapped to Info level).
func Notice(msg string, args ...any) {
if logger != nil {
args = append(args, "caller", getCaller(2))
args = append(args, "caller", getCaller())
logger.Info(msg, args...)
}
}
@@ -144,7 +154,7 @@ func Noticef(format string, args ...any) {
// Info logs an informational message.
func Info(msg string, args ...any) {
if logger != nil {
args = append(args, "caller", getCaller(2))
args = append(args, "caller", getCaller())
logger.Info(msg, args...)
}
}
@@ -157,7 +167,7 @@ func Infof(format string, args ...any) {
// Debug logs a debug message.
func Debug(msg string, args ...any) {
if logger != nil {
args = append(args, "caller", getCaller(2))
args = append(args, "caller", getCaller())
logger.Debug(msg, args...)
}
}
@@ -172,11 +182,12 @@ func With(args ...any) *slog.Logger {
if logger != nil {
return logger.With(args...)
}
return slog.Default()
}
// WithContext returns a logger with the provided context.
func WithContext(ctx context.Context) *slog.Logger {
func WithContext(_ context.Context) *slog.Logger {
return logger
}

View File

@@ -5,6 +5,8 @@ import (
)
// Module exports logging functionality for dependency injection.
//
//nolint:gochecknoglobals // fx module definitions are package globals
var Module = fx.Module("log",
fx.Invoke(func(cfg Config) {
Initialize(cfg)
@@ -12,12 +14,12 @@ var Module = fx.Module("log",
)
// New creates a new logger configuration from provided options.
func New(opts LogOptions) Config {
func New(opts Options) Config {
return Config(opts)
}
// LogOptions are provided by the CLI.
type LogOptions struct {
// Options are provided by the CLI.
type Options struct {
Verbose bool
Debug bool
Cron bool

View File

@@ -33,6 +33,7 @@ func NewTTYHandler(out io.Writer, opts *slog.HandlerOptions) *TTYHandler {
if opts == nil {
opts = &slog.HandlerOptions{}
}
return &TTYHandler{
out: out,
opts: *opts,
@@ -54,7 +55,9 @@ func (h *TTYHandler) Handle(_ context.Context, r slog.Record) error {
// Level and color
level := r.Level.String()
var levelColor string
switch r.Level {
case slog.LevelDebug:
levelColor = colorGray
@@ -91,38 +94,47 @@ func (h *TTYHandler) Handle(_ context.Context, r slog.Record) error {
if a.Key == "bytes" {
value = formatBytes(a.Value.Int64())
}
case slog.KindAny, slog.KindBool, slog.KindFloat64, slog.KindString,
slog.KindTime, slog.KindUint64, slog.KindGroup, slog.KindLogValuer:
// Plain string form above is already correct for these kinds.
default:
// Future kinds also use the plain string form.
}
_, _ = fmt.Fprintf(h.out, " %s%s%s=%s%s%s",
colorCyan, a.Key, colorReset,
colorBlue, value, colorReset)
return true
})
_, _ = fmt.Fprintln(h.out)
return nil
}
// WithAttrs returns a new handler with the given attributes.
func (h *TTYHandler) WithAttrs(attrs []slog.Attr) slog.Handler {
func (h *TTYHandler) WithAttrs(_ []slog.Attr) slog.Handler {
return h // Simplified for now
}
// WithGroup returns a new handler with the given group name.
func (h *TTYHandler) WithGroup(name string) slog.Handler {
func (h *TTYHandler) WithGroup(_ string) slog.Handler {
return h // Simplified for now
}
// formatDuration formats a duration in a human-readable way
func formatDuration(d time.Duration) string {
if d < time.Millisecond {
switch {
case d < time.Millisecond:
return fmt.Sprintf("%dµs", d.Microseconds())
} else if d < time.Second {
case d < time.Second:
return fmt.Sprintf("%dms", d.Milliseconds())
} else if d < time.Minute {
case d < time.Minute:
return fmt.Sprintf("%.1fs", d.Seconds())
}
default:
return d.String()
}
}
// formatBytes formats bytes in a human-readable way
@@ -131,10 +143,12 @@ func formatBytes(b int64) string {
if b < unit {
return fmt.Sprintf("%d B", b)
}
div, exp := int64(unit), 0
for n := b / unit; n >= unit; n /= unit {
div *= unit
exp++
}
return fmt.Sprintf("%.1f %cB", float64(b)/float64(div), "KMGTPE"[exp])
}

View File

@@ -1,3 +1,5 @@
// Package models defines shared value types describing files, chunks,
// blobs, and snapshots as they move through the backup pipeline.
package models
import (

View File

@@ -1,17 +1,21 @@
package models
package models_test
import (
"testing"
"time"
"sneak.berlin/go/vaultik/internal/models"
)
// TestModelsCompilation ensures all model types can be instantiated
func TestModelsCompilation(t *testing.T) {
t.Parallel()
// This test primarily serves as a compilation test
// to ensure all types are properly defined
// Test FileInfo
fi := &FileInfo{
fi := &models.FileInfo{
Path: "/test/file.txt",
MTime: time.Now(),
Size: 1024,
@@ -21,7 +25,7 @@ func TestModelsCompilation(t *testing.T) {
}
// Test ChunkInfo
ci := &ChunkInfo{
ci := &models.ChunkInfo{
Hash: "abc123",
Size: 512,
Offset: 0,
@@ -31,7 +35,7 @@ func TestModelsCompilation(t *testing.T) {
}
// Test BlobInfo
bi := &BlobInfo{
bi := &models.BlobInfo{
Hash: "blob123",
CreatedAt: time.Now(),
Size: 1024,
@@ -42,7 +46,7 @@ func TestModelsCompilation(t *testing.T) {
}
// Test Snapshot
s := &Snapshot{
s := &models.Snapshot{
ID: "2024-01-01T00:00:00Z",
Hostname: "test-host",
Version: "1.0.0",

View File

@@ -21,6 +21,13 @@ type Lock struct {
path string
}
const (
// lockDirPerm is the mode for the lock directory (owner-only).
lockDirPerm = 0o700
// pidFilePerm is the mode for the PID file (owner-only).
pidFilePerm = 0o600
)
// Acquire attempts to acquire a PID lock in the specified directory.
// If the lock file exists and the process is still running, it returns
// ErrAlreadyRunning with details about the existing process.
@@ -28,7 +35,8 @@ type Lock struct {
// a Lock that must be released with Release().
func Acquire(lockDir string) (*Lock, error) {
// Ensure lock directory exists
if err := os.MkdirAll(lockDir, 0700); err != nil {
err := os.MkdirAll(lockDir, lockDirPerm)
if err != nil {
return nil, fmt.Errorf("creating lock directory: %w", err)
}
@@ -46,7 +54,9 @@ func Acquire(lockDir string) (*Lock, error) {
// Write our PID
pid := os.Getpid()
if err := os.WriteFile(lockPath, []byte(strconv.Itoa(pid)), 0600); err != nil {
err = os.WriteFile(lockPath, []byte(strconv.Itoa(pid)), pidFilePerm)
if err != nil {
return nil, fmt.Errorf("writing PID file: %w", err)
}
@@ -64,7 +74,7 @@ func (l *Lock) Release() error {
existingPID, err := readPIDFile(l.path)
if err != nil {
// File already gone or unreadable - that's fine
return nil
return nil //nolint:nilerr // unreadable lock file means nothing to release
}
if existingPID != os.Getpid() {
@@ -72,17 +82,19 @@ func (l *Lock) Release() error {
return nil
}
if err := os.Remove(l.path); err != nil && !os.IsNotExist(err) {
err = os.Remove(l.path)
if err != nil && !os.IsNotExist(err) {
return fmt.Errorf("removing PID file: %w", err)
}
l.path = "" // Prevent double-release
return nil
}
// readPIDFile reads and parses the PID from a lock file.
func readPIDFile(path string) (int, error) {
data, err := os.ReadFile(path)
data, err := os.ReadFile(path) //nolint:gosec // G304: path is our own lock file
if err != nil {
return 0, err
}
@@ -104,5 +116,6 @@ func isProcessRunning(pid int) bool {
// On Unix, FindProcess always succeeds. We need to send signal 0 to check.
err = process.Signal(syscall.Signal(0))
return err == nil
}

View File

@@ -1,4 +1,4 @@
package pidlock
package pidlock_test
import (
"os"
@@ -8,18 +8,22 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"sneak.berlin/go/vaultik/internal/pidlock"
)
func TestAcquireAndRelease(t *testing.T) {
t.Parallel()
tmpDir := t.TempDir()
// Acquire lock
lock, err := Acquire(tmpDir)
lock, err := pidlock.Acquire(tmpDir)
require.NoError(t, err)
require.NotNil(t, lock)
// Verify PID file exists with our PID
data, err := os.ReadFile(filepath.Join(tmpDir, "vaultik.pid"))
pidPath := filepath.Join(tmpDir, "vaultik.pid")
data, err := os.ReadFile(pidPath) //nolint:gosec // G304: test's own temp file
require.NoError(t, err)
pid, err := strconv.Atoi(string(data))
require.NoError(t, err)
@@ -30,26 +34,31 @@ func TestAcquireAndRelease(t *testing.T) {
require.NoError(t, err)
// Verify PID file is gone
_, err = os.Stat(filepath.Join(tmpDir, "vaultik.pid"))
_, err = os.Stat(pidPath)
assert.True(t, os.IsNotExist(err))
}
func TestAcquireBlocksSecondInstance(t *testing.T) {
t.Parallel()
tmpDir := t.TempDir()
// Acquire first lock
lock1, err := Acquire(tmpDir)
lock1, err := pidlock.Acquire(tmpDir)
require.NoError(t, err)
require.NotNil(t, lock1)
defer func() { _ = lock1.Release() }()
// Try to acquire second lock - should fail
lock2, err := Acquire(tmpDir)
assert.ErrorIs(t, err, ErrAlreadyRunning)
lock2, err := pidlock.Acquire(tmpDir)
require.ErrorIs(t, err, pidlock.ErrAlreadyRunning)
assert.Nil(t, lock2)
}
func TestAcquireWithStaleLock(t *testing.T) {
t.Parallel()
tmpDir := t.TempDir()
// Write a stale PID file (PID that doesn't exist)
@@ -59,13 +68,14 @@ func TestAcquireWithStaleLock(t *testing.T) {
require.NoError(t, err)
// Should be able to acquire lock (stale lock is cleaned up)
lock, err := Acquire(tmpDir)
lock, err := pidlock.Acquire(tmpDir)
require.NoError(t, err)
require.NotNil(t, lock)
defer func() { _ = lock.Release() }()
// Verify our PID is now in the file
data, err := os.ReadFile(pidPath)
data, err := os.ReadFile(pidPath) //nolint:gosec // G304: test's own temp file
require.NoError(t, err)
pid, err := strconv.Atoi(string(data))
require.NoError(t, err)
@@ -73,9 +83,11 @@ func TestAcquireWithStaleLock(t *testing.T) {
}
func TestReleaseIsIdempotent(t *testing.T) {
t.Parallel()
tmpDir := t.TempDir()
lock, err := Acquire(tmpDir)
lock, err := pidlock.Acquire(tmpDir)
require.NoError(t, err)
// Release multiple times - should not error
@@ -87,17 +99,23 @@ func TestReleaseIsIdempotent(t *testing.T) {
}
func TestReleaseNilLock(t *testing.T) {
var lock *Lock
t.Parallel()
var lock *pidlock.Lock
err := lock.Release()
assert.NoError(t, err)
require.NoError(t, err)
}
func TestAcquireCreatesDirectory(t *testing.T) {
t.Parallel()
tmpDir := t.TempDir()
nestedDir := filepath.Join(tmpDir, "nested", "dir")
lock, err := Acquire(nestedDir)
lock, err := pidlock.Acquire(nestedDir)
require.NoError(t, err)
require.NotNil(t, lock)
defer func() { _ = lock.Release() }()

View File

@@ -1,3 +1,5 @@
// Package s3 wraps the AWS S3 SDK with a simplified client for vaultik's
// bucket-and-prefix scoped object operations.
package s3
import (
@@ -42,7 +44,7 @@ type Config struct {
// Used to suppress SDK warnings about checksums.
type nopLogger struct{}
func (nopLogger) Logf(classification logging.Classification, format string, v ...interface{}) {}
func (nopLogger) Logf(_ logging.Classification, _ string, _ ...any) {}
// NewClient creates a new S3 client with the provided configuration.
// It establishes a connection to the S3-compatible storage service and
@@ -92,6 +94,7 @@ func (c *Client) PutObject(ctx context.Context, key string, data io.Reader) erro
Key: aws.String(fullKey),
Body: data,
})
return err
}
@@ -104,13 +107,18 @@ type ProgressCallback func(bytesUploaded int64) error
// The size parameter must be the exact size of the data to upload.
// The progress callback is called periodically with the number of bytes uploaded.
// Returns an error if the upload fails.
func (c *Client) PutObjectWithProgress(ctx context.Context, key string, data io.Reader, size int64, progress ProgressCallback) error {
func (c *Client) PutObjectWithProgress(
ctx context.Context, key string, data io.Reader,
size int64, progress ProgressCallback,
) error {
fullKey := c.prefix + key
// uploadPartSize is 10MB for better progress granularity.
const uploadPartSize = 10 * 1024 * 1024
// Create an uploader with the S3 client
uploader := manager.NewUploader(c.s3Client, func(u *manager.Uploader) {
// Set part size to 10MB for better progress granularity
u.PartSize = 10 * 1024 * 1024
u.PartSize = uploadPartSize
})
// Create a progress reader that tracks upload progress
@@ -137,6 +145,7 @@ func (c *Client) PutObjectWithProgress(ctx context.Context, key string, data io.
// close the returned reader when done to avoid resource leaks.
func (c *Client) GetObject(ctx context.Context, key string) (io.ReadCloser, error) {
fullKey := c.prefix + key
result, err := c.s3Client.GetObject(ctx, &s3.GetObjectInput{
Bucket: aws.String(c.bucket),
Key: aws.String(fullKey),
@@ -144,6 +153,7 @@ func (c *Client) GetObject(ctx context.Context, key string) (io.ReadCloser, erro
if err != nil {
return nil, err
}
return result.Body, nil
}
@@ -156,6 +166,7 @@ func (c *Client) DeleteObject(ctx context.Context, key string) error {
Bucket: aws.String(c.bucket),
Key: aws.String(fullKey),
})
return err
}
@@ -168,6 +179,7 @@ func (c *Client) ListObjects(ctx context.Context, prefix string) ([]string, erro
fullPrefix := c.prefix + prefix
var keys []string
paginator := s3.NewListObjectsV2Paginator(c.s3Client, &s3.ListObjectsV2Input{
Bucket: aws.String(c.bucket),
Prefix: aws.String(fullPrefix),
@@ -186,6 +198,7 @@ func (c *Client) ListObjects(ctx context.Context, prefix string) ([]string, erro
if len(key) > len(c.prefix) {
key = key[len(c.prefix):]
}
keys = append(keys, key)
}
}
@@ -200,18 +213,23 @@ func (c *Client) ListObjects(ctx context.Context, prefix string) ([]string, erro
// Note: This method returns false for any error, not just "not found".
func (c *Client) HeadObject(ctx context.Context, key string) (bool, error) {
fullKey := c.prefix + key
_, err := c.s3Client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: aws.String(c.bucket),
Key: aws.String(fullKey),
})
if err != nil {
var notFound *s3types.NotFound
var noSuchKey *s3types.NoSuchKey
var (
notFound *s3types.NotFound
noSuchKey *s3types.NoSuchKey
)
if errors.As(err, &notFound) || errors.As(err, &noSuchKey) {
return false, nil
}
return false, err
}
return true, nil
}
@@ -230,7 +248,9 @@ type ObjectInfo struct {
// listing is complete or an error occurs. If an error occurs, it will be
// sent as the last item with the Err field set. The recursive parameter
// is currently unused but reserved for future use.
func (c *Client) ListObjectsStream(ctx context.Context, prefix string, recursive bool) <-chan ObjectInfo {
func (c *Client) ListObjectsStream(
ctx context.Context, prefix string, _ bool,
) <-chan ObjectInfo {
ch := make(chan ObjectInfo)
go func() {
@@ -247,6 +267,7 @@ func (c *Client) ListObjectsStream(ctx context.Context, prefix string, recursive
page, err := paginator.NextPage(ctx)
if err != nil {
ch <- ObjectInfo{Err: err}
return
}
@@ -257,6 +278,7 @@ func (c *Client) ListObjectsStream(ctx context.Context, prefix string, recursive
if len(key) > len(c.prefix) {
key = key[len(c.prefix):]
}
ch <- ObjectInfo{
Key: key,
Size: *obj.Size,
@@ -275,6 +297,7 @@ func (c *Client) ListObjectsStream(ctx context.Context, prefix string, recursive
// Returns an error if the object doesn't exist or if the operation fails.
func (c *Client) StatObject(ctx context.Context, key string) (*ObjectInfo, error) {
fullKey := c.prefix + key
result, err := c.s3Client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: aws.String(c.bucket),
Key: aws.String(fullKey),
@@ -313,6 +336,7 @@ func (c *Client) Endpoint() string {
if c.endpoint == "" {
return "s3.amazonaws.com"
}
return c.endpoint
}
@@ -329,11 +353,14 @@ func (pr *progressReader) Read(p []byte) (int, error) {
n, err := pr.reader.Read(p)
if n > 0 {
atomic.AddInt64(&pr.read, int64(n))
if pr.callback != nil {
if callbackErr := pr.callback(atomic.LoadInt64(&pr.read)); callbackErr != nil {
callbackErr := pr.callback(atomic.LoadInt64(&pr.read))
if callbackErr != nil {
return n, callbackErr
}
}
}
return n, err
}

View File

@@ -9,10 +9,12 @@ import (
"sneak.berlin/go/vaultik/internal/s3"
)
//nolint:paralleltest // test servers share a fixed localhost port
func TestClient(t *testing.T) {
ts := NewTestServer(t)
defer func() {
if err := ts.Cleanup(); err != nil {
err := ts.Cleanup()
if err != nil {
t.Errorf("cleanup failed: %v", err)
}
}()
@@ -32,10 +34,21 @@ func TestClient(t *testing.T) {
t.Fatalf("failed to create client: %v", err)
}
// Test PutObject
testKey := "foo/bar.txt"
testData := []byte("test data")
err = client.PutObject(ctx, testKey, bytes.NewReader(testData))
verifyPutGetHead(ctx, t, client, testKey, testData)
verifyListAndDelete(ctx, t, client, testKey)
}
// verifyPutGetHead uploads an object, reads it back, and checks existence.
func verifyPutGetHead(
ctx context.Context, t *testing.T, client *s3.Client,
testKey string, testData []byte,
) {
t.Helper()
err := client.PutObject(ctx, testKey, bytes.NewReader(testData))
if err != nil {
t.Fatalf("failed to put object: %v", err)
}
@@ -46,7 +59,8 @@ func TestClient(t *testing.T) {
t.Fatalf("failed to get object: %v", err)
}
defer func() {
if err := reader.Close(); err != nil {
err := reader.Close()
if err != nil {
t.Errorf("failed to close reader: %v", err)
}
}()
@@ -65,18 +79,28 @@ func TestClient(t *testing.T) {
if err != nil {
t.Fatalf("failed to head object: %v", err)
}
if !exists {
t.Error("expected object to exist")
}
}
// verifyListAndDelete lists the object's prefix, deletes it, and checks
// it is gone.
func verifyListAndDelete(
ctx context.Context, t *testing.T, client *s3.Client, testKey string,
) {
t.Helper()
// Test ListObjects
keys, err := client.ListObjects(ctx, "foo/")
if err != nil {
t.Fatalf("failed to list objects: %v", err)
}
if len(keys) != 1 {
t.Errorf("expected 1 key, got %d", len(keys))
}
if keys[0] != testKey {
t.Errorf("unexpected key: got %s, want %s", keys[0], testKey)
}
@@ -88,10 +112,11 @@ func TestClient(t *testing.T) {
}
// Verify deletion
exists, err = client.HeadObject(ctx, testKey)
exists, err := client.HeadObject(ctx, testKey)
if err != nil {
t.Fatalf("failed to head object after deletion: %v", err)
}
if exists {
t.Error("expected object to not exist after deletion")
}

View File

@@ -10,6 +10,8 @@ import (
// Module exports S3 functionality as an fx module.
// It provides automatic dependency injection for the S3 client,
// configuring it based on the application's configuration settings.
//
//nolint:gochecknoglobals // fx module definitions are package globals
var Module = fx.Module("s3",
fx.Provide(
provideClient,
@@ -32,7 +34,7 @@ func provideClient(lc fx.Lifecycle, cfg *config.Config) (*Client, error) {
}
lc.Append(fx.Hook{
OnStop: func(ctx context.Context) error {
OnStop: func(_ context.Context) error {
// S3 client doesn't need explicit cleanup
return nil
},

View File

@@ -3,10 +3,10 @@ package s3_test
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"testing"
"time"
@@ -37,13 +37,16 @@ type TestServer struct {
logBuf *bytes.Buffer
}
// testServerReadHeaderTimeout bounds header reads on the in-process
// test server (gosec G112).
const testServerReadHeaderTimeout = 5 * time.Second
// NewTestServer creates and starts a new test server
func NewTestServer(t *testing.T) *TestServer {
t.Helper()
// Create temp directory for any file operations
tempDir, err := os.MkdirTemp("", "vaultik-s3-test-*")
if err != nil {
t.Fatalf("failed to create temp dir: %v", err)
}
tempDir := t.TempDir()
// Create in-memory backend
backend := s3mem.New()
@@ -53,11 +56,13 @@ func NewTestServer(t *testing.T) *TestServer {
server := &http.Server{
Addr: "localhost:9999",
Handler: faker.Server(),
ReadHeaderTimeout: testServerReadHeaderTimeout,
}
// Start server in background
go func() {
if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
err := server.ListenAndServe()
if err != nil && !errors.Is(err, http.ErrServerClosed) {
t.Logf("test server error: %v", err)
}
}()
@@ -69,6 +74,14 @@ func NewTestServer(t *testing.T) *TestServer {
logBuf := &bytes.Buffer{}
// Create S3 client with custom logger
logFn := func(classification logging.Classification, format string, v ...any) {
// Capture logs to buffer instead of stdout
fmt.Fprintf(logBuf, "SDK %s %s %s\n",
time.Now().Format("2006/01/02 15:04:05"),
string(classification),
fmt.Sprintf(format, v...))
}
cfg, err := config.LoadDefaultConfig(context.Background(),
config.WithRegion(testRegion),
config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(
@@ -76,14 +89,9 @@ func NewTestServer(t *testing.T) *TestServer {
testSecretKey,
"",
)),
config.WithClientLogMode(aws.LogRetries|aws.LogRequestWithBody|aws.LogResponseWithBody),
config.WithLogger(logging.LoggerFunc(func(classification logging.Classification, format string, v ...interface{}) {
// Capture logs to buffer instead of stdout
fmt.Fprintf(logBuf, "SDK %s %s %s\n",
time.Now().Format("2006/01/02 15:04:05"),
string(classification),
fmt.Sprintf(format, v...))
})),
config.WithClientLogMode(
aws.LogRetries|aws.LogRequestWithBody|aws.LogResponseWithBody),
config.WithLogger(logging.LoggerFunc(logFn)),
)
if err != nil {
t.Fatalf("failed to create AWS config: %v", err)
@@ -120,16 +128,13 @@ func NewTestServer(t *testing.T) *TestServer {
return ts
}
// Cleanup shuts down the server and removes temp directory
// Cleanup shuts down the server. The temp directory is removed
// automatically by t.TempDir.
func (ts *TestServer) Cleanup() error {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if err := ts.server.Shutdown(ctx); err != nil {
return err
}
return os.RemoveAll(ts.tempDir)
return ts.server.Shutdown(ctx)
}
// Client returns the S3 client configured for the test server
@@ -138,10 +143,13 @@ func (ts *TestServer) Client() *s3.Client {
}
// TestBasicS3Operations tests basic store and retrieve operations
//
//nolint:paralleltest // test servers share a fixed localhost port
func TestBasicS3Operations(t *testing.T) {
ts := NewTestServer(t)
defer func() {
if err := ts.Cleanup(); err != nil {
err := ts.Cleanup()
if err != nil {
t.Errorf("cleanup failed: %v", err)
}
}()
@@ -172,7 +180,8 @@ func TestBasicS3Operations(t *testing.T) {
t.Fatalf("failed to get object: %v", err)
}
defer func() {
if err := result.Body.Close(); err != nil {
err := result.Body.Close()
if err != nil {
t.Errorf("failed to close body: %v", err)
}
}()
@@ -189,10 +198,13 @@ func TestBasicS3Operations(t *testing.T) {
}
// TestBlobOperations tests blob storage patterns for vaultik
//
//nolint:paralleltest // test servers share a fixed localhost port
func TestBlobOperations(t *testing.T) {
ts := NewTestServer(t)
defer func() {
if err := ts.Cleanup(); err != nil {
err := ts.Cleanup()
if err != nil {
t.Errorf("cleanup failed: %v", err)
}
}()
@@ -252,10 +264,13 @@ func TestBlobOperations(t *testing.T) {
}
// TestMetadataOperations tests metadata storage patterns
//
//nolint:paralleltest // test servers share a fixed localhost port
func TestMetadataOperations(t *testing.T) {
ts := NewTestServer(t)
defer func() {
if err := ts.Cleanup(); err != nil {
err := ts.Cleanup()
if err != nil {
t.Errorf("cleanup failed: %v", err)
}
}()
@@ -280,7 +295,8 @@ func TestMetadataOperations(t *testing.T) {
// Store manifest
manifestKey := filepath.Join("metadata", snapshotID+".manifest.json.zst")
manifestData := []byte(`{"snapshot_id":"2024-01-01T12:00:00Z","blob_hashes":["hash1","hash2"]}`)
manifestData := []byte(`{"snapshot_id":"2024-01-01T12:00:00Z",` +
`"blob_hashes":["hash1","hash2"]}`)
_, err = client.PutObject(ctx, &s3.PutObjectInput{
Bucket: aws.String(testBucket),

View File

@@ -1,9 +1,11 @@
package snapshot
package snapshot_test
import (
"context"
"crypto/sha256"
"database/sql"
"encoding/hex"
"errors"
"fmt"
"io"
"io/fs"
@@ -17,6 +19,12 @@ import (
"sneak.berlin/go/vaultik/internal/types"
)
// errBlobNotFound is returned by the mock S3 client for unknown hashes.
var errBlobNotFound = errors.New("blob not found")
// testFile1Name is the shared fixture filename used across backup tests.
const testFile1Name = "file1.txt"
// MockS3Client is a mock implementation of S3 operations for testing
type MockS3Client struct {
storage map[string][]byte
@@ -28,36 +36,149 @@ func NewMockS3Client() *MockS3Client {
}
}
func (m *MockS3Client) PutBlob(ctx context.Context, hash string, data []byte) error {
func (m *MockS3Client) PutBlob(_ context.Context, hash string, data []byte) error {
m.storage[hash] = data
return nil
}
func (m *MockS3Client) GetBlob(ctx context.Context, hash string) ([]byte, error) {
func (m *MockS3Client) GetBlob(_ context.Context, hash string) ([]byte, error) {
data, ok := m.storage[hash]
if !ok {
return nil, fmt.Errorf("blob not found: %s", hash)
return nil, fmt.Errorf("%w: %s", errBlobNotFound, hash)
}
return data, nil
}
func (m *MockS3Client) BlobExists(ctx context.Context, hash string) (bool, error) {
func (m *MockS3Client) BlobExists(_ context.Context, hash string) (bool, error) {
_, ok := m.storage[hash]
return ok, nil
}
func (m *MockS3Client) CreateBucket(ctx context.Context, bucket string) error {
func (m *MockS3Client) CreateBucket(_ context.Context, _ string) error {
return nil
}
// verifyBackupFiles checks the file records created by a backup against
// the fixture filesystem.
func verifyBackupFiles(
ctx context.Context,
t *testing.T,
repos *database.Repositories,
testFS fstest.MapFS,
) {
t.Helper()
files, err := repos.Files.ListByPrefix(ctx, "")
if err != nil {
t.Fatalf("Failed to list files: %v", err)
}
expectedFiles := map[string]bool{
testFile1Name: true,
"dir1/file2.txt": true,
"dir1/subdir/file3.txt": true,
"largefile.bin": true,
}
if len(files) != len(expectedFiles) {
t.Errorf("Expected %d files, got %d", len(expectedFiles), len(files))
}
for _, file := range files {
if !expectedFiles[file.Path.String()] {
t.Errorf("Unexpected file in database: %s", file.Path)
}
delete(expectedFiles, file.Path.String())
// Verify file metadata
fsFile := testFS[file.Path.String()]
if fsFile == nil {
t.Errorf("File %s not found in test filesystem", file.Path)
continue
}
if file.Size != int64(len(fsFile.Data)) {
t.Errorf("File %s: expected size %d, got %d",
file.Path, len(fsFile.Data), file.Size)
}
if file.Mode != uint32(fsFile.Mode) {
t.Errorf("File %s: expected mode %o, got %o",
file.Path, fsFile.Mode, file.Mode)
}
}
if len(expectedFiles) > 0 {
t.Errorf("Files not found in database: %v", expectedFiles)
}
}
// verifyBackupChunksAndBlobs checks that chunking produced the expected
// records and every referenced blob exists in the mock S3 store.
func verifyBackupChunksAndBlobs(
ctx context.Context,
t *testing.T,
repos *database.Repositories,
s3Client *MockS3Client,
snapshotID string,
) {
t.Helper()
chunks, err := repos.Chunks.List(ctx)
if err != nil {
t.Fatalf("Failed to list chunks: %v", err)
}
if len(chunks) == 0 {
t.Error("No chunks found in database")
}
// The large file should create 10 chunks (10MB / 1MB chunk size)
// Plus the small files
minExpectedChunks := 10 + 3
if len(chunks) < minExpectedChunks {
t.Errorf("Expected at least %d chunks, got %d", minExpectedChunks, len(chunks))
}
// Verify at least one blob was created and uploaded
// We can't list blobs directly, but we can check via snapshot blobs
blobHashes, err := repos.Snapshots.GetBlobHashes(ctx, snapshotID)
if err != nil {
t.Fatalf("Failed to get blob hashes: %v", err)
}
if len(blobHashes) == 0 {
t.Error("Expected at least one blob to be created")
}
for _, blobHash := range blobHashes {
// Check blob exists in mock S3
exists, err := s3Client.BlobExists(ctx, blobHash)
if err != nil {
t.Errorf("Failed to check blob %s: %v", blobHash, err)
}
if !exists {
t.Errorf("Blob %s not found in S3", blobHash)
}
}
}
func TestBackupWithInMemoryFS(t *testing.T) {
t.Parallel()
// Create a temporary directory for the database
tempDir := t.TempDir()
dbPath := filepath.Join(tempDir, "test.db")
// Create test filesystem
testFS := fstest.MapFS{
"file1.txt": &fstest.MapFile{
testFile1Name: &fstest.MapFile{
Data: []byte("Hello, World!"),
Mode: 0644,
ModTime: time.Now(),
@@ -81,12 +202,15 @@ func TestBackupWithInMemoryFS(t *testing.T) {
// Initialize the database
ctx := context.Background()
db, err := database.New(ctx, dbPath)
if err != nil {
t.Fatalf("Failed to create database: %v", err)
}
defer func() {
if err := db.Close(); err != nil {
err := db.Close()
if err != nil {
t.Logf("Failed to close database: %v", err)
}
}()
@@ -121,96 +245,21 @@ func TestBackupWithInMemoryFS(t *testing.T) {
t.Error("Expected snapshot to have files")
}
// Verify files in database
files, err := repos.Files.ListByPrefix(ctx, "")
if err != nil {
t.Fatalf("Failed to list files: %v", err)
}
expectedFiles := map[string]bool{
"file1.txt": true,
"dir1/file2.txt": true,
"dir1/subdir/file3.txt": true,
"largefile.bin": true,
}
if len(files) != len(expectedFiles) {
t.Errorf("Expected %d files, got %d", len(expectedFiles), len(files))
}
for _, file := range files {
if !expectedFiles[file.Path.String()] {
t.Errorf("Unexpected file in database: %s", file.Path)
}
delete(expectedFiles, file.Path.String())
// Verify file metadata
fsFile := testFS[file.Path.String()]
if fsFile == nil {
t.Errorf("File %s not found in test filesystem", file.Path)
continue
}
if file.Size != int64(len(fsFile.Data)) {
t.Errorf("File %s: expected size %d, got %d", file.Path, len(fsFile.Data), file.Size)
}
if file.Mode != uint32(fsFile.Mode) {
t.Errorf("File %s: expected mode %o, got %o", file.Path, fsFile.Mode, file.Mode)
}
}
if len(expectedFiles) > 0 {
t.Errorf("Files not found in database: %v", expectedFiles)
}
// Verify chunks
chunks, err := repos.Chunks.List(ctx)
if err != nil {
t.Fatalf("Failed to list chunks: %v", err)
}
if len(chunks) == 0 {
t.Error("No chunks found in database")
}
// The large file should create 10 chunks (10MB / 1MB chunk size)
// Plus the small files
minExpectedChunks := 10 + 3
if len(chunks) < minExpectedChunks {
t.Errorf("Expected at least %d chunks, got %d", minExpectedChunks, len(chunks))
}
// Verify at least one blob was created and uploaded
// We can't list blobs directly, but we can check via snapshot blobs
blobHashes, err := repos.Snapshots.GetBlobHashes(ctx, snapshotID)
if err != nil {
t.Fatalf("Failed to get blob hashes: %v", err)
}
if len(blobHashes) == 0 {
t.Error("Expected at least one blob to be created")
}
for _, blobHash := range blobHashes {
// Check blob exists in mock S3
exists, err := s3Client.BlobExists(ctx, blobHash)
if err != nil {
t.Errorf("Failed to check blob %s: %v", blobHash, err)
}
if !exists {
t.Errorf("Blob %s not found in S3", blobHash)
}
}
// Verify files, chunks, and blob records
verifyBackupFiles(ctx, t, repos, testFS)
verifyBackupChunksAndBlobs(ctx, t, repos, s3Client, snapshotID)
}
func TestBackupDeduplication(t *testing.T) {
t.Parallel()
// Create a temporary directory for the database
tempDir := t.TempDir()
dbPath := filepath.Join(tempDir, "test.db")
// Create test filesystem with duplicate content
testFS := fstest.MapFS{
"file1.txt": &fstest.MapFile{
testFile1Name: &fstest.MapFile{
Data: []byte("Duplicate content"),
Mode: 0644,
ModTime: time.Now(),
@@ -229,12 +278,15 @@ func TestBackupDeduplication(t *testing.T) {
// Initialize the database
ctx := context.Background()
db, err := database.New(ctx, dbPath)
if err != nil {
t.Fatalf("Failed to create database: %v", err)
}
defer func() {
if err := db.Close(); err != nil {
err := db.Close()
if err != nil {
t.Logf("Failed to close database: %v", err)
}
}()
@@ -275,7 +327,8 @@ func TestBackupDeduplication(t *testing.T) {
// The duplicate content chunk should be referenced by 2 files
if chunk.Size == int64(len("Duplicate content")) && len(files) != 2 {
t.Errorf("Expected duplicate chunk to be referenced by 2 files, got %d", len(files))
t.Errorf("Expected duplicate chunk to be referenced by 2 files, got %d",
len(files))
}
}
}
@@ -289,8 +342,19 @@ type BackupEngine struct {
}
}
// backupCounters accumulates statistics across a test backup run.
type backupCounters struct {
fileCount int64
chunkCount int64
blobCount int64
totalSize int64
blobSize int64
}
// Backup performs a backup of the given filesystem
func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (string, error) {
func (b *BackupEngine) Backup(
ctx context.Context, fsys fs.FS, root string,
) (string, error) {
// Create a new snapshot
hostname, _ := os.Hostname()
snapshotID := time.Now().Format(time.RFC3339)
@@ -310,8 +374,7 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
return "", err
}
// Track counters
var fileCount, chunkCount, blobCount, totalSize, blobSize int64
counters := &backupCounters{}
// Track which chunks we've seen to handle deduplication
processedChunks := make(map[string]bool)
@@ -339,6 +402,40 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
return nil
}
return b.backupOneFile(ctx, fsys, path, info, processedChunks, counters)
})
if err != nil {
return "", err
}
// After all files are processed, create blobs for new chunks
err = b.createBlobsForChunks(ctx, snapshotID, processedChunks, counters)
if err != nil {
return "", err
}
// Update snapshot with final counts
err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
return b.repos.Snapshots.UpdateCounts(ctx, tx, snapshotID,
counters.fileCount, counters.chunkCount, counters.blobCount,
counters.totalSize, counters.blobSize)
})
if err != nil {
return "", err
}
return snapshotID, nil
}
// backupOneFile records a single regular file and its chunks.
func (b *BackupEngine) backupOneFile(
ctx context.Context,
fsys fs.FS,
path string,
info fs.FileInfo,
processedChunks map[string]bool,
counters *backupCounters,
) error {
// Create file record in a short transaction
file := &database.File{
Path: types.FilePath(path),
@@ -348,15 +445,16 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
UID: 1000, // Default UID for test
GID: 1000, // Default GID for test
}
err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
err := b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
return b.repos.Files.Create(ctx, tx, file)
})
if err != nil {
return err
}
fileCount++
totalSize += info.Size()
counters.fileCount++
counters.totalSize += info.Size()
// Read and process file in chunks
f, err := fsys.Open(path)
@@ -364,7 +462,8 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
return err
}
defer func() {
if err := f.Close(); err != nil {
err := f.Close()
if err != nil {
// Log but don't fail since we're already in an error path potentially
fmt.Fprintf(os.Stderr, "Failed to close file: %v\n", err)
}
@@ -376,56 +475,15 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
for {
n, err := f.Read(buffer)
if err != nil && err != io.EOF {
if err != nil && !errors.Is(err, io.EOF) {
return err
}
if n == 0 {
break
}
chunkData := buffer[:n]
chunkHash := calculateHash(chunkData)
// Check if chunk already exists (outside of transaction)
existingChunk, _ := b.repos.Chunks.GetByHash(ctx, chunkHash)
if existingChunk == nil {
// Create new chunk in a short transaction
err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
chunk := &database.Chunk{
ChunkHash: types.ChunkHash(chunkHash),
Size: int64(n),
}
return b.repos.Chunks.Create(ctx, tx, chunk)
})
if err != nil {
return err
}
processedChunks[chunkHash] = true
}
// Create file-chunk mapping in a short transaction
err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
fileChunk := &database.FileChunk{
FileID: file.ID,
Idx: chunkIndex,
ChunkHash: types.ChunkHash(chunkHash),
}
return b.repos.FileChunks.Create(ctx, tx, fileChunk)
})
if err != nil {
return err
}
// Create chunk-file mapping in a short transaction
err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
chunkFile := &database.ChunkFile{
ChunkHash: types.ChunkHash(chunkHash),
FileID: file.ID,
FileOffset: int64(chunkIndex * defaultChunkSize),
Length: int64(n),
}
return b.repos.ChunkFiles.Create(ctx, tx, chunkFile)
})
err = b.recordChunk(ctx, file, chunkIndex, buffer[:n], processedChunks)
if err != nil {
return err
}
@@ -434,21 +492,80 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
}
return nil
})
}
if err != nil {
return "", err
// recordChunk creates the chunk record (if new) and its file associations.
func (b *BackupEngine) recordChunk(
ctx context.Context,
file *database.File,
chunkIndex int,
chunkData []byte,
processedChunks map[string]bool,
) error {
chunkHash := calculateHash(chunkData)
// Check if chunk already exists (outside of transaction)
existingChunk, _ := b.repos.Chunks.GetByHash(ctx, chunkHash)
if existingChunk == nil {
// Create new chunk in a short transaction
err := b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
chunk := &database.Chunk{
ChunkHash: types.ChunkHash(chunkHash),
Size: int64(len(chunkData)),
}
// After all files are processed, create blobs for new chunks
return b.repos.Chunks.Create(ctx, tx, chunk)
})
if err != nil {
return err
}
processedChunks[chunkHash] = true
}
// Create file-chunk mapping in a short transaction
err := b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
fileChunk := &database.FileChunk{
FileID: file.ID,
Idx: chunkIndex,
ChunkHash: types.ChunkHash(chunkHash),
}
return b.repos.FileChunks.Create(ctx, tx, fileChunk)
})
if err != nil {
return err
}
// Create chunk-file mapping in a short transaction
return b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
chunkFile := &database.ChunkFile{
ChunkHash: types.ChunkHash(chunkHash),
FileID: file.ID,
FileOffset: int64(chunkIndex * defaultChunkSize),
Length: int64(len(chunkData)),
}
return b.repos.ChunkFiles.Create(ctx, tx, chunkFile)
})
}
// createBlobsForChunks uploads one blob per new chunk and records the blob
// metadata and snapshot association.
func (b *BackupEngine) createBlobsForChunks(
ctx context.Context,
snapshotID string,
processedChunks map[string]bool,
counters *backupCounters,
) error {
for chunkHash := range processedChunks {
// Get chunk data (outside of transaction)
chunk, err := b.repos.Chunks.GetByHash(ctx, chunkHash)
if err != nil {
return "", err
return err
}
chunkCount++
counters.chunkCount++
// In a real system, blobs would contain multiple chunks and be encrypted
// For testing, we'll create a blob with a "blob-" prefix to differentiate
@@ -458,26 +575,29 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
dummyData := []byte(chunkHash)
// Upload to S3 as a blob
if err := b.s3Client.PutBlob(ctx, blobHash, dummyData); err != nil {
return "", err
err = b.s3Client.PutBlob(ctx, blobHash, dummyData)
if err != nil {
return err
}
// Create blob entry in a short transaction
blobID := types.NewBlobID()
err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
blob := &database.Blob{
ID: blobID,
Hash: types.BlobHash(blobHash),
CreatedTS: time.Now(),
}
return b.repos.Blobs.Create(ctx, tx, blob)
})
if err != nil {
return "", err
return err
}
blobCount++
blobSize += chunk.Size
counters.blobCount++
counters.blobSize += chunk.Size
// Create blob-chunk mapping in a short transaction
err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
@@ -487,46 +607,41 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
Offset: 0,
Length: chunk.Size,
}
return b.repos.BlobChunks.Create(ctx, tx, blobChunk)
})
if err != nil {
return "", err
return err
}
// Add blob to snapshot in a short transaction
err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
return b.repos.Snapshots.AddBlob(ctx, tx, snapshotID, blobID, types.BlobHash(blobHash))
return b.repos.Snapshots.AddBlob(ctx, tx, snapshotID, blobID,
types.BlobHash(blobHash))
})
if err != nil {
return "", err
return err
}
}
// Update snapshot with final counts
err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
return b.repos.Snapshots.UpdateCounts(ctx, tx, snapshotID, fileCount, chunkCount, blobCount, totalSize, blobSize)
})
if err != nil {
return "", err
}
return snapshotID, nil
return nil
}
func calculateHash(data []byte) string {
h := sha256.New()
h.Write(data)
return fmt.Sprintf("%x", h.Sum(nil))
return hex.EncodeToString(h.Sum(nil))
}
func generateLargeFileContent(size int) []byte {
data := make([]byte, size)
// Fill with pattern that changes every chunk to avoid deduplication
for i := 0; i < size; i++ {
for i := range size {
chunkNum := i / defaultChunkSize
data[i] = byte((i + chunkNum) % 256)
}
return data
}

View File

@@ -10,16 +10,15 @@ import (
"github.com/spf13/afero"
"github.com/stretchr/testify/require"
"sneak.berlin/go/vaultik/internal/database"
"sneak.berlin/go/vaultik/internal/log"
"sneak.berlin/go/vaultik/internal/snapshot"
"sneak.berlin/go/vaultik/internal/types"
)
func setupExcludeTestFS(t *testing.T) afero.Fs {
func setupExcludeTestFS(t *testing.T) *afero.MemMapFs {
t.Helper()
// Create in-memory filesystem
fs := afero.NewMemMapFs()
fs := &afero.MemMapFs{}
// Create test directory structure:
// /backup/
@@ -63,6 +62,7 @@ func setupExcludeTestFS(t *testing.T) afero.Fs {
}
testTime := time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC)
for path, content := range files {
dir := filepath.Dir(path)
err := fs.MkdirAll(dir, 0755)
@@ -76,12 +76,11 @@ func setupExcludeTestFS(t *testing.T) afero.Fs {
return fs
}
func createTestScanner(t *testing.T, fs afero.Fs, excludePatterns []string) (*snapshot.Scanner, *database.Repositories, func()) {
func createTestScanner(
t *testing.T, fs afero.Fs, excludePatterns []string,
) (*snapshot.Scanner, *database.Repositories, func()) {
t.Helper()
// Initialize logger
log.Initialize(log.Config{})
// Create test database
db, err := database.NewTestDB()
require.NoError(t, err)
@@ -94,7 +93,8 @@ func createTestScanner(t *testing.T, fs afero.Fs, excludePatterns []string) (*sn
Repositories: repos,
MaxBlobSize: 1024 * 1024,
CompressionLevel: 3,
AgeRecipients: []string{"age1ql3z7hjy54pw3hyww5ayyfg7zqgvc7w3j2elw8zmrj2kg5sfn9aqmcac8p"},
AgeRecipients: []string{
"age1ql3z7hjy54pw3hyww5ayyfg7zqgvc7w3j2elw8zmrj2kg5sfn9aqmcac8p"},
Exclude: excludePatterns,
})
@@ -105,13 +105,16 @@ func createTestScanner(t *testing.T, fs afero.Fs, excludePatterns []string) (*sn
return scanner, repos, cleanup
}
func createSnapshotRecord(t *testing.T, ctx context.Context, repos *database.Repositories, snapshotID string) {
func createSnapshotRecord(
ctx context.Context, t *testing.T, repos *database.Repositories, snapshotID string,
) {
t.Helper()
err := repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
snap := &database.Snapshot{
ID: types.SnapshotID(snapshotID),
Hostname: "test-host",
VaultikVersion: "test",
Hostname: testHost,
VaultikVersion: testVersion,
StartedAt: time.Now(),
CompletedAt: nil,
FileCount: 0,
@@ -121,25 +124,31 @@ func createSnapshotRecord(t *testing.T, ctx context.Context, repos *database.Rep
BlobSize: 0,
CompressionRatio: 1.0,
}
return repos.Snapshots.Create(ctx, tx, snap)
})
require.NoError(t, err)
}
func TestExcludePatterns_ExcludeGitDirectory(t *testing.T) {
t.Parallel()
fs := setupExcludeTestFS(t)
scanner, repos, cleanup := createTestScanner(t, fs, []string{".git"})
defer cleanup()
require.NotNil(t, scanner)
ctx := context.Background()
createSnapshotRecord(t, ctx, repos, "test-snapshot")
createSnapshotRecord(ctx, t, repos, "test-snapshot")
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
require.NoError(t, err)
// Should have scanned files but NOT .git directory contents
// Expected: file1.txt, file2.log, src/main.go, src/test.go, node_modules/package/index.js,
// Expected: file1.txt, file2.log, src/main.go, src/test.go,
// node_modules/package/index.js,
// cache/temp.dat, build/output.bin, docs/readme.md, .DS_Store, thumbs.db,
// src/.hidden, important.log.bak
// Excluded: .git/config, .git/objects/pack/data.pack
@@ -147,13 +156,17 @@ func TestExcludePatterns_ExcludeGitDirectory(t *testing.T) {
}
func TestExcludePatterns_ExcludeByExtension(t *testing.T) {
t.Parallel()
fs := setupExcludeTestFS(t)
scanner, repos, cleanup := createTestScanner(t, fs, []string{"*.log"})
defer cleanup()
require.NotNil(t, scanner)
ctx := context.Background()
createSnapshotRecord(t, ctx, repos, "test-snapshot")
createSnapshotRecord(ctx, t, repos, "test-snapshot")
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
require.NoError(t, err)
@@ -164,13 +177,17 @@ func TestExcludePatterns_ExcludeByExtension(t *testing.T) {
}
func TestExcludePatterns_ExcludeNodeModules(t *testing.T) {
t.Parallel()
fs := setupExcludeTestFS(t)
scanner, repos, cleanup := createTestScanner(t, fs, []string{"node_modules"})
defer cleanup()
require.NotNil(t, scanner)
ctx := context.Background()
createSnapshotRecord(t, ctx, repos, "test-snapshot")
createSnapshotRecord(ctx, t, repos, "test-snapshot")
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
require.NoError(t, err)
@@ -181,30 +198,41 @@ func TestExcludePatterns_ExcludeNodeModules(t *testing.T) {
}
func TestExcludePatterns_MultiplePatterns(t *testing.T) {
t.Parallel()
fs := setupExcludeTestFS(t)
scanner, repos, cleanup := createTestScanner(t, fs, []string{".git", "node_modules", "*.log", ".DS_Store", "thumbs.db", "cache", "build"})
scanner, repos, cleanup := createTestScanner(t, fs, []string{
".git", "node_modules", "*.log", ".DS_Store", "thumbs.db", "cache", "build"})
defer cleanup()
require.NotNil(t, scanner)
ctx := context.Background()
createSnapshotRecord(t, ctx, repos, "test-snapshot")
createSnapshotRecord(ctx, t, repos, "test-snapshot")
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
require.NoError(t, err)
// Should only have: file1.txt, src/main.go, src/test.go, docs/readme.md, src/.hidden, important.log.bak
// Excluded: .git/*, node_modules/*, *.log (file2.log), .DS_Store, thumbs.db, cache/*, build/*
// Should only have: file1.txt, src/main.go, src/test.go, docs/readme.md,
// src/.hidden, important.log.bak
// Excluded: .git/*, node_modules/*, *.log (file2.log), .DS_Store,
// thumbs.db, cache/*, build/*
require.Equal(t, 6, result.FilesScanned, "Should exclude multiple patterns")
}
func TestExcludePatterns_NoExclusions(t *testing.T) {
t.Parallel()
fs := setupExcludeTestFS(t)
scanner, repos, cleanup := createTestScanner(t, fs, []string{})
defer cleanup()
require.NotNil(t, scanner)
ctx := context.Background()
createSnapshotRecord(t, ctx, repos, "test-snapshot")
createSnapshotRecord(ctx, t, repos, "test-snapshot")
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
require.NoError(t, err)
@@ -214,30 +242,40 @@ func TestExcludePatterns_NoExclusions(t *testing.T) {
}
func TestExcludePatterns_ExcludeHiddenFiles(t *testing.T) {
t.Parallel()
fs := setupExcludeTestFS(t)
scanner, repos, cleanup := createTestScanner(t, fs, []string{".*"})
defer cleanup()
require.NotNil(t, scanner)
ctx := context.Background()
createSnapshotRecord(t, ctx, repos, "test-snapshot")
createSnapshotRecord(ctx, t, repos, "test-snapshot")
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
require.NoError(t, err)
// Should exclude: .git/*, .DS_Store, src/.hidden
// Total files: 14, excluded: 4 (.git/config, .git/objects/pack/data.pack, .DS_Store, src/.hidden)
require.Equal(t, 10, result.FilesScanned, "Should exclude hidden files and directories")
// Total files: 14, excluded: 4 (.git/config,
// .git/objects/pack/data.pack, .DS_Store, src/.hidden)
require.Equal(t, 10, result.FilesScanned,
"Should exclude hidden files and directories")
}
func TestExcludePatterns_DoubleStarGlob(t *testing.T) {
t.Parallel()
fs := setupExcludeTestFS(t)
scanner, repos, cleanup := createTestScanner(t, fs, []string{"**/*.pack"})
defer cleanup()
require.NotNil(t, scanner)
ctx := context.Background()
createSnapshotRecord(t, ctx, repos, "test-snapshot")
createSnapshotRecord(ctx, t, repos, "test-snapshot")
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
require.NoError(t, err)
@@ -248,13 +286,17 @@ func TestExcludePatterns_DoubleStarGlob(t *testing.T) {
}
func TestExcludePatterns_ExactFileName(t *testing.T) {
t.Parallel()
fs := setupExcludeTestFS(t)
scanner, repos, cleanup := createTestScanner(t, fs, []string{"thumbs.db", ".DS_Store"})
defer cleanup()
require.NotNil(t, scanner)
ctx := context.Background()
createSnapshotRecord(t, ctx, repos, "test-snapshot")
createSnapshotRecord(ctx, t, repos, "test-snapshot")
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
require.NoError(t, err)
@@ -265,14 +307,18 @@ func TestExcludePatterns_ExactFileName(t *testing.T) {
}
func TestExcludePatterns_CaseSensitive(t *testing.T) {
t.Parallel()
// Pattern matching should be case-sensitive
fs := setupExcludeTestFS(t)
scanner, repos, cleanup := createTestScanner(t, fs, []string{"THUMBS.DB"})
defer cleanup()
require.NotNil(t, scanner)
ctx := context.Background()
createSnapshotRecord(t, ctx, repos, "test-snapshot")
createSnapshotRecord(ctx, t, repos, "test-snapshot")
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
require.NoError(t, err)
@@ -283,32 +329,39 @@ func TestExcludePatterns_CaseSensitive(t *testing.T) {
}
func TestExcludePatterns_DirectoryWithTrailingSlash(t *testing.T) {
t.Parallel()
fs := setupExcludeTestFS(t)
// Some users might add trailing slashes to directory patterns
scanner, repos, cleanup := createTestScanner(t, fs, []string{"cache/", "build/"})
defer cleanup()
require.NotNil(t, scanner)
ctx := context.Background()
createSnapshotRecord(t, ctx, repos, "test-snapshot")
createSnapshotRecord(ctx, t, repos, "test-snapshot")
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
require.NoError(t, err)
// Should exclude cache/temp.dat and build/output.bin
// Total files: 14, excluded: 2
require.Equal(t, 12, result.FilesScanned, "Should handle directory patterns with trailing slashes")
require.Equal(t, 12, result.FilesScanned,
"Should handle directory patterns with trailing slashes")
}
func TestExcludePatterns_PatternInSubdirectory(t *testing.T) {
t.Parallel()
fs := setupExcludeTestFS(t)
// Exclude .hidden file specifically in src directory
scanner, repos, cleanup := createTestScanner(t, fs, []string{"src/.hidden"})
defer cleanup()
require.NotNil(t, scanner)
ctx := context.Background()
createSnapshotRecord(t, ctx, repos, "test-snapshot")
createSnapshotRecord(ctx, t, repos, "test-snapshot")
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
require.NoError(t, err)
@@ -327,13 +380,14 @@ func TestExcludePatterns_PatternInSubdirectory(t *testing.T) {
// file.txt (should be excluded with /projectname)
// otherproject/
// projectname/
// file.txt (should NOT be excluded with /projectname, only with projectname)
// file.txt (should NOT be excluded with /projectname,
// only with projectname)
// src/
// file.go
func setupAnchoredTestFS(t *testing.T) afero.Fs {
func setupAnchoredTestFS(t *testing.T) *afero.MemMapFs {
t.Helper()
fs := afero.NewMemMapFs()
fs := &afero.MemMapFs{}
files := map[string]string{
"/backup/projectname/file.txt": "root project file",
@@ -343,6 +397,7 @@ func setupAnchoredTestFS(t *testing.T) afero.Fs {
}
testTime := time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC)
for path, content := range files {
dir := filepath.Dir(path)
err := fs.MkdirAll(dir, 0755)
@@ -357,14 +412,18 @@ func setupAnchoredTestFS(t *testing.T) afero.Fs {
}
func TestExcludePatterns_AnchoredPattern(t *testing.T) {
t.Parallel()
// Pattern starting with / should only match from root of source dir
fs := setupAnchoredTestFS(t)
scanner, repos, cleanup := createTestScanner(t, fs, []string{"/projectname"})
defer cleanup()
require.NotNil(t, scanner)
ctx := context.Background()
createSnapshotRecord(t, ctx, repos, "test-snapshot")
createSnapshotRecord(ctx, t, repos, "test-snapshot")
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
require.NoError(t, err)
@@ -372,18 +431,23 @@ func TestExcludePatterns_AnchoredPattern(t *testing.T) {
// /projectname should ONLY exclude /backup/projectname/file.txt (1 file)
// /backup/otherproject/projectname/file.txt should NOT be excluded
// Total files: 4, excluded: 1
require.Equal(t, 3, result.FilesScanned, "Anchored pattern /projectname should only match at root of source dir")
require.Equal(t, 3, result.FilesScanned,
"Anchored pattern /projectname should only match at root of source dir")
}
func TestExcludePatterns_UnanchoredPattern(t *testing.T) {
t.Parallel()
// Pattern without leading / should match anywhere in path
fs := setupAnchoredTestFS(t)
scanner, repos, cleanup := createTestScanner(t, fs, []string{"projectname"})
defer cleanup()
require.NotNil(t, scanner)
ctx := context.Background()
createSnapshotRecord(t, ctx, repos, "test-snapshot")
createSnapshotRecord(ctx, t, repos, "test-snapshot")
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
require.NoError(t, err)
@@ -392,18 +456,23 @@ func TestExcludePatterns_UnanchoredPattern(t *testing.T) {
// - /backup/projectname/file.txt
// - /backup/otherproject/projectname/file.txt
// Total files: 4, excluded: 2
require.Equal(t, 2, result.FilesScanned, "Unanchored pattern should match anywhere in path")
require.Equal(t, 2, result.FilesScanned,
"Unanchored pattern should match anywhere in path")
}
func TestExcludePatterns_AnchoredPatternWithGlob(t *testing.T) {
t.Parallel()
// Anchored pattern with glob
fs := setupAnchoredTestFS(t)
scanner, repos, cleanup := createTestScanner(t, fs, []string{"/src/*.go"})
defer cleanup()
require.NotNil(t, scanner)
ctx := context.Background()
createSnapshotRecord(t, ctx, repos, "test-snapshot")
createSnapshotRecord(ctx, t, repos, "test-snapshot")
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
require.NoError(t, err)
@@ -414,14 +483,18 @@ func TestExcludePatterns_AnchoredPatternWithGlob(t *testing.T) {
}
func TestExcludePatterns_AnchoredPatternFile(t *testing.T) {
t.Parallel()
// Anchored pattern for exact file at root
fs := setupAnchoredTestFS(t)
scanner, repos, cleanup := createTestScanner(t, fs, []string{"/file.txt"})
defer cleanup()
require.NotNil(t, scanner)
ctx := context.Background()
createSnapshotRecord(t, ctx, repos, "test-snapshot")
createSnapshotRecord(ctx, t, repos, "test-snapshot")
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
require.NoError(t, err)
@@ -429,18 +502,23 @@ func TestExcludePatterns_AnchoredPatternFile(t *testing.T) {
// /file.txt should ONLY exclude /backup/file.txt
// NOT /backup/projectname/file.txt or /backup/otherproject/projectname/file.txt
// Total files: 4, excluded: 1
require.Equal(t, 3, result.FilesScanned, "Anchored pattern for file should only match at root")
require.Equal(t, 3, result.FilesScanned,
"Anchored pattern for file should only match at root")
}
func TestExcludePatterns_UnanchoredPatternFile(t *testing.T) {
t.Parallel()
// Unanchored pattern for file should match anywhere
fs := setupAnchoredTestFS(t)
scanner, repos, cleanup := createTestScanner(t, fs, []string{"file.txt"})
defer cleanup()
require.NotNil(t, scanner)
ctx := context.Background()
createSnapshotRecord(t, ctx, repos, "test-snapshot")
createSnapshotRecord(ctx, t, repos, "test-snapshot")
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
require.NoError(t, err)
@@ -450,5 +528,6 @@ func TestExcludePatterns_UnanchoredPatternFile(t *testing.T) {
// - /backup/projectname/file.txt
// - /backup/otherproject/projectname/file.txt
// Total files: 4, excluded: 3
require.Equal(t, 1, result.FilesScanned, "Unanchored pattern for file should match anywhere")
require.Equal(t, 1, result.FilesScanned,
"Unanchored pattern for file should match anywhere")
}

View File

@@ -2,7 +2,6 @@ package snapshot_test
import (
"context"
"database/sql"
"testing"
"time"
@@ -15,101 +14,20 @@ import (
"sneak.berlin/go/vaultik/internal/types"
)
// TestFileContentChange verifies that when a file's content changes,
// the old chunks are properly disassociated
func TestFileContentChange(t *testing.T) {
// Initialize logger for tests
log.Initialize(log.Config{})
// Create in-memory filesystem
fs := afero.NewMemMapFs()
// Create initial file
err := afero.WriteFile(fs, "/test.txt", []byte("Initial content"), 0644)
require.NoError(t, err)
// Create test database
db, err := database.NewTestDB()
require.NoError(t, err)
defer func() {
if err := db.Close(); err != nil {
t.Errorf("failed to close database: %v", err)
}
}()
repos := database.NewRepositories(db)
// Create scanner
scanner := snapshot.NewScanner(snapshot.ScannerConfig{
FS: fs,
ChunkSize: int64(1024 * 16), // 16KB chunks for testing
Repositories: repos,
MaxBlobSize: int64(1024 * 1024), // 1MB blobs
CompressionLevel: 3,
AgeRecipients: []string{"age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05gl0sjq9q9wjg"}, // Test public key
})
// Create first snapshot
ctx := context.Background()
snapshotID1 := "snapshot1"
err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
snapshot := &database.Snapshot{
ID: types.SnapshotID(snapshotID1),
Hostname: "test-host",
VaultikVersion: "test",
StartedAt: time.Now(),
}
return repos.Snapshots.Create(ctx, tx, snapshot)
})
require.NoError(t, err)
// First scan - should create chunks for initial content
result1, err := scanner.Scan(ctx, "/", snapshotID1)
require.NoError(t, err)
t.Logf("First scan: %d files scanned", result1.FilesScanned)
// Get file chunks from first scan
fileChunks1, err := repos.FileChunks.GetByPath(ctx, "/test.txt")
require.NoError(t, err)
assert.Len(t, fileChunks1, 1) // Small file = 1 chunk
oldChunkHash := fileChunks1[0].ChunkHash
// Get chunk files from first scan
chunkFiles1, err := repos.ChunkFiles.GetByFilePath(ctx, "/test.txt")
require.NoError(t, err)
assert.Len(t, chunkFiles1, 1)
// Modify the file
time.Sleep(10 * time.Millisecond) // Ensure mtime changes
err = afero.WriteFile(fs, "/test.txt", []byte("Modified content with different data"), 0644)
require.NoError(t, err)
// Create second snapshot
snapshotID2 := "snapshot2"
err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
snapshot := &database.Snapshot{
ID: types.SnapshotID(snapshotID2),
Hostname: "test-host",
VaultikVersion: "test",
StartedAt: time.Now(),
}
return repos.Snapshots.Create(ctx, tx, snapshot)
})
require.NoError(t, err)
// Second scan - should create new chunks and remove old associations
result2, err := scanner.Scan(ctx, "/", snapshotID2)
require.NoError(t, err)
t.Logf("Second scan: %d files scanned", result2.FilesScanned)
// Get file chunks from second scan
fileChunks2, err := repos.FileChunks.GetByPath(ctx, "/test.txt")
require.NoError(t, err)
assert.Len(t, fileChunks2, 1) // Still 1 chunk but different hash
newChunkHash := fileChunks2[0].ChunkHash
// verifyChunkChange checks that after a content change the file references
// the new chunk, the old chunk still exists, and the old chunk no longer
// maps to the modified file.
func verifyChunkChange(
ctx context.Context,
t *testing.T,
repos *database.Repositories,
oldChunkHash, newChunkHash types.ChunkHash,
) {
t.Helper()
// Verify the chunk hashes are different
assert.NotEqual(t, oldChunkHash, newChunkHash, "Chunk hash should change when content changes")
assert.NotEqual(t, oldChunkHash, newChunkHash,
"Chunk hash should change when content changes")
// Get chunk files from second scan
chunkFiles2, err := repos.ChunkFiles.GetByFilePath(ctx, "/test.txt")
@@ -130,17 +48,103 @@ func TestFileContentChange(t *testing.T) {
// Verify that chunk_files for old chunk no longer references this file
oldChunkFiles, err := repos.ChunkFiles.GetByChunkHash(ctx, oldChunkHash)
require.NoError(t, err)
for _, cf := range oldChunkFiles {
file, err := repos.Files.GetByID(ctx, cf.FileID)
require.NoError(t, err)
assert.NotEqual(t, "/data/test.txt", file.Path, "Old chunk should not be associated with the modified file")
assert.NotEqual(t, "/data/test.txt", file.Path,
"Old chunk should not be associated with the modified file")
}
}
// TestFileContentChange verifies that when a file's content changes,
// the old chunks are properly disassociated
func TestFileContentChange(t *testing.T) {
// Initialize logger for tests
log.Initialize(log.Config{})
t.Parallel()
// Create in-memory filesystem
fs := afero.NewMemMapFs()
// Create initial file
err := afero.WriteFile(fs, "/test.txt", []byte("Initial content"), 0644)
require.NoError(t, err)
// Create test database
db, err := database.NewTestDB()
require.NoError(t, err)
defer func() {
err := db.Close()
if err != nil {
t.Errorf("failed to close database: %v", err)
}
}()
repos := database.NewRepositories(db)
// Create scanner
scanner := snapshot.NewScanner(snapshot.ScannerConfig{
FS: fs,
ChunkSize: int64(1024 * 16), // 16KB chunks for testing
Repositories: repos,
MaxBlobSize: int64(1024 * 1024), // 1MB blobs
CompressionLevel: 3,
AgeRecipients: []string{testAgePublicKey},
})
// Create first snapshot
ctx := context.Background()
snapshotID1 := "snapshot1"
createSnapshotRecord(ctx, t, repos, snapshotID1)
// First scan - should create chunks for initial content
result1, err := scanner.Scan(ctx, "/", snapshotID1)
require.NoError(t, err)
t.Logf("First scan: %d files scanned", result1.FilesScanned)
// Get file chunks from first scan
fileChunks1, err := repos.FileChunks.GetByPath(ctx, "/test.txt")
require.NoError(t, err)
assert.Len(t, fileChunks1, 1) // Small file = 1 chunk
oldChunkHash := fileChunks1[0].ChunkHash
// Get chunk files from first scan
chunkFiles1, err := repos.ChunkFiles.GetByFilePath(ctx, "/test.txt")
require.NoError(t, err)
assert.Len(t, chunkFiles1, 1)
// Modify the file
time.Sleep(10 * time.Millisecond) // Ensure mtime changes
err = afero.WriteFile(fs, "/test.txt",
[]byte("Modified content with different data"), 0644)
require.NoError(t, err)
// Create second snapshot
snapshotID2 := "snapshot2"
createSnapshotRecord(ctx, t, repos, snapshotID2)
// Second scan - should create new chunks and remove old associations
result2, err := scanner.Scan(ctx, "/", snapshotID2)
require.NoError(t, err)
t.Logf("Second scan: %d files scanned", result2.FilesScanned)
// Get file chunks from second scan
fileChunks2, err := repos.FileChunks.GetByPath(ctx, "/test.txt")
require.NoError(t, err)
assert.Len(t, fileChunks2, 1) // Still 1 chunk but different hash
newChunkHash := fileChunks2[0].ChunkHash
verifyChunkChange(ctx, t, repos, oldChunkHash, newChunkHash)
}
// TestMultipleFileChanges verifies handling of multiple file changes in one scan
func TestMultipleFileChanges(t *testing.T) {
// Initialize logger for tests
log.Initialize(log.Config{})
t.Parallel()
// Create in-memory filesystem
fs := afero.NewMemMapFs()
@@ -159,9 +163,11 @@ func TestMultipleFileChanges(t *testing.T) {
// Create test database
db, err := database.NewTestDB()
require.NoError(t, err)
defer func() {
if err := db.Close(); err != nil {
err := db.Close()
if err != nil {
t.Errorf("failed to close database: %v", err)
}
}()
@@ -175,22 +181,13 @@ func TestMultipleFileChanges(t *testing.T) {
Repositories: repos,
MaxBlobSize: int64(1024 * 1024), // 1MB blobs
CompressionLevel: 3,
AgeRecipients: []string{"age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05gl0sjq9q9wjg"}, // Test public key
AgeRecipients: []string{testAgePublicKey},
})
// Create first snapshot
ctx := context.Background()
snapshotID1 := "snapshot1"
err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
snapshot := &database.Snapshot{
ID: types.SnapshotID(snapshotID1),
Hostname: "test-host",
VaultikVersion: "test",
StartedAt: time.Now(),
}
return repos.Snapshots.Create(ctx, tx, snapshot)
})
require.NoError(t, err)
createSnapshotRecord(ctx, t, repos, snapshotID1)
// First scan
result1, err := scanner.Scan(ctx, "/", snapshotID1)
@@ -200,6 +197,7 @@ func TestMultipleFileChanges(t *testing.T) {
// Modify two files
time.Sleep(10 * time.Millisecond) // Ensure mtime changes
err = afero.WriteFile(fs, "/file1.txt", []byte("Modified content 1"), 0644)
require.NoError(t, err)
err = afero.WriteFile(fs, "/file3.txt", []byte("Modified content 3"), 0644)
@@ -207,16 +205,7 @@ func TestMultipleFileChanges(t *testing.T) {
// Create second snapshot
snapshotID2 := "snapshot2"
err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
snapshot := &database.Snapshot{
ID: types.SnapshotID(snapshotID2),
Hostname: "test-host",
VaultikVersion: "test",
StartedAt: time.Now(),
}
return repos.Snapshots.Create(ctx, tx, snapshot)
})
require.NoError(t, err)
createSnapshotRecord(ctx, t, repos, snapshotID2)
// Second scan
result2, err := scanner.Scan(ctx, "/", snapshotID2)
@@ -229,10 +218,12 @@ func TestMultipleFileChanges(t *testing.T) {
for path := range files {
fileChunks, err := repos.FileChunks.GetByPath(ctx, path)
require.NoError(t, err)
assert.Len(t, fileChunks, 1, "File %s should have exactly 1 chunk association", path)
assert.Len(t, fileChunks, 1,
"File %s should have exactly 1 chunk association", path)
chunkFiles, err := repos.ChunkFiles.GetByFilePath(ctx, path)
require.NoError(t, err)
assert.Len(t, chunkFiles, 1, "File %s should have exactly 1 chunk-file association", path)
assert.Len(t, chunkFiles, 1,
"File %s should have exactly 1 chunk-file association", path)
}
}

View File

@@ -10,6 +10,8 @@ import (
)
// Manifest represents the structure of a snapshot's blob manifest
//
//nolint:tagliatelle // snake_case is the established on-disk manifest format
type Manifest struct {
SnapshotID string `json:"snapshot_id"`
Timestamp string `json:"timestamp"`
@@ -19,6 +21,8 @@ type Manifest struct {
}
// BlobInfo represents information about a single blob in the manifest
//
//nolint:tagliatelle // snake_case is the established on-disk manifest format
type BlobInfo struct {
Hash string `json:"hash"`
CompressedSize int64 `json:"compressed_size"`
@@ -35,7 +39,9 @@ func DecodeManifest(r io.Reader) (*Manifest, error) {
// Decode JSON manifest
var manifest Manifest
if err := json.NewDecoder(zr).Decode(&manifest); err != nil {
err = json.NewDecoder(zr).Decode(&manifest)
if err != nil {
return nil, fmt.Errorf("decoding manifest: %w", err)
}
@@ -52,17 +58,22 @@ func EncodeManifest(manifest *Manifest, compressionLevel int) ([]byte, error) {
// Compress using zstd
var compressedBuf bytes.Buffer
writer, err := zstd.NewWriter(&compressedBuf, zstd.WithEncoderLevel(zstd.EncoderLevelFromZstd(compressionLevel)))
writer, err := zstd.NewWriter(&compressedBuf,
zstd.WithEncoderLevel(zstd.EncoderLevelFromZstd(compressionLevel)))
if err != nil {
return nil, fmt.Errorf("creating zstd writer: %w", err)
}
if _, err := writer.Write(jsonData); err != nil {
_, err = writer.Write(jsonData)
if err != nil {
_ = writer.Close()
return nil, fmt.Errorf("writing compressed data: %w", err)
}
if err := writer.Close(); err != nil {
err = writer.Close()
if err != nil {
return nil, fmt.Errorf("closing zstd writer: %w", err)
}

View File

@@ -21,6 +21,8 @@ type ScannerParams struct {
// Module exports backup functionality as an fx module.
// It provides a ScannerFactory that can create Scanner instances
// with custom parameters while sharing common dependencies.
//
//nolint:gochecknoglobals // fx module definitions are conventionally globals
var Module = fx.Module("backup",
fx.Provide(
provideScannerFactory,
@@ -31,7 +33,9 @@ var Module = fx.Module("backup",
// ScannerFactory creates scanners with custom parameters
type ScannerFactory func(params ScannerParams) *Scanner
func provideScannerFactory(cfg *config.Config, repos *database.Repositories, storer storage.Storer) ScannerFactory {
func provideScannerFactory(
cfg *config.Config, repos *database.Repositories, storer storage.Storer,
) ScannerFactory {
return func(params ScannerParams) *Scanner {
// Use provided excludes, or fall back to global config excludes
excludes := params.Exclude

View File

@@ -1,3 +1,4 @@
//nolint:testpackage // needs access to unexported wrapPermissionError
package snapshot
import (
@@ -9,10 +10,15 @@ import (
"testing"
)
// errDiskOnFire is a non-permission sentinel used to verify pass-through.
var errDiskOnFire = errors.New("disk on fire")
func TestWrapPermissionError(t *testing.T) {
t.Parallel()
// Non-permission errors pass through unchanged.
plain := errors.New("disk on fire")
if got := wrapPermissionError("/some/path", plain); got != plain {
got := wrapPermissionError("/some/path", errDiskOnFire)
if !errors.Is(got, errDiskOnFire) {
t.Errorf("non-permission error should pass through, got %v", got)
}
@@ -23,20 +29,23 @@ func TestWrapPermissionError(t *testing.T) {
if !errors.Is(wrapped, os.ErrPermission) {
t.Error("wrapped error should still match os.ErrPermission")
}
if !strings.Contains(wrapped.Error(), "/Users/u/Library/Calendars") {
t.Error("wrapped error should name the offending path")
}
if runtime.GOOS == "darwin" {
if !strings.Contains(wrapped.Error(), "Full Disk Access") {
t.Errorf("macOS permission error should mention Full Disk Access:\n%s", wrapped.Error())
t.Errorf("macOS permission error should mention Full Disk Access:\n%s",
wrapped.Error())
}
if !strings.Contains(wrapped.Error(), "System Settings") {
t.Errorf("macOS permission error should point at System Settings:\n%s", wrapped.Error())
}
} else {
if !strings.Contains(wrapped.Error(), "--skip-errors") {
t.Errorf("non-macOS permission error should mention --skip-errors:\n%s", wrapped.Error())
t.Errorf("macOS permission error should point at System Settings:\n%s",
wrapped.Error())
}
} else if !strings.Contains(wrapped.Error(), "--skip-errors") {
t.Errorf("non-macOS permission error should mention --skip-errors:\n%s",
wrapped.Error())
}
}

View File

@@ -19,14 +19,41 @@ const (
// These updates show current progress, ETA, and the file being processed.
SummaryInterval = 10 * time.Second
// DetailInterval defines how often multi-line detailed status reports are printed.
// These reports include comprehensive statistics about files, chunks, blobs, and uploads.
// DetailInterval defines how often multi-line detailed status reports are
// printed. These reports include comprehensive statistics about files,
// chunks, blobs, and uploads.
DetailInterval = 60 * time.Second
// UploadProgressInterval defines how often upload progress messages are logged.
UploadProgressInterval = 15 * time.Second
)
const (
// bitsPerByte converts byte counts to bit counts for speed display.
bitsPerByte = 8
// percentScale converts a ratio to a percentage.
percentScale = 100
// currentFileMaxLen is the display width used for current-file paths.
currentFileMaxLen = 40
// secondsPerMinute and minutesPerHour are used for duration formatting.
secondsPerMinute = 60
minutesPerHour = 60
// Bit-rate thresholds for human-readable upload speed formatting.
bitsPerGbit = 1e9
bitsPerMbit = 1e6
bitsPerKbit = 1e3
// ellipsis prefixes truncated paths and suffixes shortened hashes.
ellipsis = "..."
// hashPrefixLen is how many hex characters of a blob hash to show in logs.
hashPrefixLen = 8
)
// ProgressStats holds atomic counters for progress tracking
type ProgressStats struct {
FilesScanned atomic.Int64 // Total files seen during scan (includes skipped)
@@ -64,7 +91,7 @@ type UploadInfo struct {
// ProgressReporter handles periodic progress reporting
type ProgressReporter struct {
stats *ProgressStats
ctx context.Context
ctx context.Context //nolint:containedctx // bound at construction
cancel context.CancelFunc
wg sync.WaitGroup
detailTicker *time.Ticker
@@ -127,6 +154,161 @@ func (pr *ProgressReporter) SetTotalSize(size int64) {
pr.stats.ProcessStartTime.Store(time.Now().UTC())
}
// Helper functions
func formatDuration(d time.Duration) string {
if d < 0 {
return "unknown"
}
if d < time.Minute {
return fmt.Sprintf("%ds", int(d.Seconds()))
}
if d < time.Hour {
return fmt.Sprintf("%dm%ds", int(d.Minutes()), int(d.Seconds())%secondsPerMinute)
}
return fmt.Sprintf("%dh%dm", int(d.Hours()), int(d.Minutes())%minutesPerHour)
}
func formatPercent(numerator, denominator int64) string {
if denominator == 0 {
return "0.0%"
}
return fmt.Sprintf("%.1f%%", float64(numerator)/float64(denominator)*percentScale)
}
func formatRatio(compressed, uncompressed int64) string {
if uncompressed == 0 {
return "1.00"
}
ratio := float64(compressed) / float64(uncompressed)
return fmt.Sprintf("%.2f", ratio)
}
func truncatePath(path string, maxLen int) string {
if len(path) <= maxLen {
return path
}
// Keep the last maxLen-len(ellipsis) characters and prepend the ellipsis.
return ellipsis + path[len(path)-(maxLen-len(ellipsis)):]
}
// safeUint64 converts a non-negative int64 counter to uint64 for display,
// clamping negative values to zero.
func safeUint64(n int64) uint64 {
if n < 0 {
return 0
}
return uint64(n)
}
// ReportUploadStart marks the beginning of a blob upload
func (pr *ProgressReporter) ReportUploadStart(blobHash string, size int64) {
info := &UploadInfo{
BlobHash: blobHash,
Size: size,
StartTime: time.Now().UTC(),
}
pr.stats.CurrentUpload.Store(info)
// Log the start of upload
log.Info("Starting blob upload",
"hash", blobHash[:hashPrefixLen]+ellipsis,
"size", humanize.Bytes(safeUint64(size)))
}
// ReportUploadComplete marks the completion of a blob upload
func (pr *ProgressReporter) ReportUploadComplete(
blobHash string, size int64, duration time.Duration,
) {
// Clear current upload
pr.stats.CurrentUpload.Store((*UploadInfo)(nil))
// Add to total upload duration
pr.stats.UploadDurationMs.Add(duration.Milliseconds())
// Calculate speed
if duration < time.Millisecond {
duration = time.Millisecond
}
bytesPerSec := float64(size) / duration.Seconds()
bitsPerSec := bytesPerSec * bitsPerByte
// Format speed
var speedStr string
switch {
case bitsPerSec >= bitsPerGbit:
speedStr = fmt.Sprintf("%.1fGbit/sec", bitsPerSec/bitsPerGbit)
case bitsPerSec >= bitsPerMbit:
speedStr = fmt.Sprintf("%.0fMbit/sec", bitsPerSec/bitsPerMbit)
case bitsPerSec >= bitsPerKbit:
speedStr = fmt.Sprintf("%.0fKbit/sec", bitsPerSec/bitsPerKbit)
default:
speedStr = fmt.Sprintf("%.0fbit/sec", bitsPerSec)
}
log.Info("Blob upload completed",
"hash", blobHash[:hashPrefixLen]+ellipsis,
"size", humanize.Bytes(safeUint64(size)),
"duration", formatDuration(duration),
"speed", speedStr)
}
// UpdateChunkingActivity updates the last chunking time
func (pr *ProgressReporter) UpdateChunkingActivity() {
pr.stats.mu.Lock()
pr.stats.lastChunkingTime = time.Now().UTC()
pr.stats.mu.Unlock()
}
// ReportUploadProgress reports current upload progress with instantaneous speed
func (pr *ProgressReporter) ReportUploadProgress(
blobHash string, bytesUploaded, totalSize int64, instantSpeed float64,
) {
// Update the current upload info with progress
uploadInfo, ok := pr.stats.CurrentUpload.Load().(*UploadInfo)
if ok && uploadInfo != nil {
now := time.Now()
// Only log at the configured interval
if now.Sub(uploadInfo.LastLogTime) >= UploadProgressInterval {
// Format speed in bits/second using humanize
bitsPerSec := instantSpeed * bitsPerByte
speedStr := humanize.SI(bitsPerSec, "bit/sec")
percent := float64(bytesUploaded) / float64(totalSize) * percentScale
// Calculate ETA based on current speed
etaStr := "unknown"
if instantSpeed > 0 && bytesUploaded < totalSize {
remainingBytes := totalSize - bytesUploaded
remainingSeconds := float64(remainingBytes) / instantSpeed
eta := time.Duration(remainingSeconds * float64(time.Second))
etaStr = formatDuration(eta)
}
log.Info("Blob upload progress",
"hash", blobHash[:hashPrefixLen]+ellipsis,
"progress", fmt.Sprintf("%.1f%%", percent),
"uploaded", humanize.Bytes(safeUint64(bytesUploaded)),
"total", humanize.Bytes(safeUint64(totalSize)),
"speed", speedStr,
"eta", etaStr)
uploadInfo.LastLogTime = now
}
}
}
// run is the main progress reporting loop
func (pr *ProgressReporter) run() {
defer pr.wg.Done()
@@ -150,9 +332,11 @@ func (pr *ProgressReporter) run() {
// printSummaryStatus prints a one-line status update
func (pr *ProgressReporter) printSummaryStatus() {
// Check if we're currently uploading
if uploadInfo, ok := pr.stats.CurrentUpload.Load().(*UploadInfo); ok && uploadInfo != nil {
uploadInfo, ok := pr.stats.CurrentUpload.Load().(*UploadInfo)
if ok && uploadInfo != nil {
// Show upload progress instead
pr.printUploadProgress(uploadInfo)
return
}
@@ -171,20 +355,22 @@ func (pr *ProgressReporter) printSummaryStatus() {
bytesSkipped := pr.stats.BytesSkipped.Load()
bytesProcessed := pr.stats.BytesProcessed.Load()
totalSize := pr.stats.TotalSize.Load()
currentFile := pr.stats.CurrentFile.Load().(string)
currentFile, _ := pr.stats.CurrentFile.Load().(string)
// Calculate ETA if we have total size and are processing
etaStr := ""
if totalSize > 0 && bytesProcessed > 0 {
processStart, ok := pr.stats.ProcessStartTime.Load().(time.Time)
if ok && !processStart.IsZero() {
processElapsed := time.Since(processStart)
rate := float64(bytesProcessed) / processElapsed.Seconds()
if rate > 0 {
remainingBytes := totalSize - bytesProcessed
remainingSeconds := float64(remainingBytes) / rate
eta := time.Duration(remainingSeconds * float64(time.Second))
etaStr = fmt.Sprintf(" | ETA: %s", formatDuration(eta))
etaStr = " | ETA: " + formatDuration(eta)
}
}
}
@@ -198,15 +384,15 @@ func (pr *ProgressReporter) printSummaryStatus() {
status := fmt.Sprintf("Snapshot progress: %d/%d files, %s/%s (%.1f%%), %s/s%s",
filesProcessed,
totalFiles,
humanize.Bytes(uint64(bytesProcessed)),
humanize.Bytes(uint64(totalSize)),
float64(bytesProcessed)/float64(totalSize)*100,
humanize.Bytes(safeUint64(bytesProcessed)),
humanize.Bytes(safeUint64(totalSize)),
float64(bytesProcessed)/float64(totalSize)*percentScale,
humanize.Bytes(uint64(rate)),
etaStr,
)
if currentFile != "" {
status += fmt.Sprintf(" | Current: %s", truncatePath(currentFile, 40))
status += " | Current: " + truncatePath(currentFile, currentFileMaxLen)
}
log.Info(status)
@@ -229,7 +415,7 @@ func (pr *ProgressReporter) printDetailedStatus() {
blobsCreated := pr.stats.BlobsCreated.Load()
blobsUploaded := pr.stats.BlobsUploaded.Load()
bytesUploaded := pr.stats.BytesUploaded.Load()
currentFile := pr.stats.CurrentFile.Load().(string)
currentFile, _ := pr.stats.CurrentFile.Load().(string)
totalBytes := bytesScanned + bytesSkipped
rate := float64(totalBytes) / elapsed.Seconds()
@@ -242,16 +428,17 @@ func (pr *ProgressReporter) printDetailedStatus() {
processStart, ok := pr.stats.ProcessStartTime.Load().(time.Time)
if ok && !processStart.IsZero() {
processElapsed := time.Since(processStart)
processRate := float64(bytesProcessed) / processElapsed.Seconds()
if processRate > 0 {
remainingBytes := totalSize - bytesProcessed
remainingSeconds := float64(remainingBytes) / processRate
eta := time.Duration(remainingSeconds * float64(time.Second))
percentComplete := float64(bytesProcessed) / float64(totalSize) * 100
percentComplete := float64(bytesProcessed) / float64(totalSize) * percentScale
log.Info("Overall progress",
"percent", fmt.Sprintf("%.1f%%", percentComplete),
"processed", humanize.Bytes(uint64(bytesProcessed)),
"total", humanize.Bytes(uint64(totalSize)),
"processed", humanize.Bytes(safeUint64(bytesProcessed)),
"total", humanize.Bytes(safeUint64(totalSize)),
"rate", humanize.Bytes(uint64(processRate))+"/s",
"eta", formatDuration(eta))
}
@@ -264,9 +451,9 @@ func (pr *ProgressReporter) printDetailedStatus() {
"total", filesScanned,
"skip_rate", formatPercent(filesSkipped, filesScanned))
log.Info("Data scanned",
"new", humanize.Bytes(uint64(bytesScanned)),
"skipped", humanize.Bytes(uint64(bytesSkipped)),
"total", humanize.Bytes(uint64(totalBytes)),
"new", humanize.Bytes(safeUint64(bytesScanned)),
"skipped", humanize.Bytes(safeUint64(bytesSkipped)),
"total", humanize.Bytes(safeUint64(totalBytes)),
"scan_rate", humanize.Bytes(uint64(rate))+"/s")
log.Info("Chunks created", "count", chunksCreated)
log.Info("Blobs status",
@@ -274,146 +461,18 @@ func (pr *ProgressReporter) printDetailedStatus() {
"uploaded", blobsUploaded,
"pending", blobsCreated-blobsUploaded)
log.Info("Total uploaded to remote",
"uploaded", humanize.Bytes(uint64(bytesUploaded)),
"uploaded", humanize.Bytes(safeUint64(bytesUploaded)),
"compression_ratio", formatRatio(bytesUploaded, bytesScanned))
if currentFile != "" {
log.Info("Current file", "path", currentFile)
}
log.Notice("=============================")
}
// Helper functions
func formatDuration(d time.Duration) string {
if d < 0 {
return "unknown"
}
if d < time.Minute {
return fmt.Sprintf("%ds", int(d.Seconds()))
}
if d < time.Hour {
return fmt.Sprintf("%dm%ds", int(d.Minutes()), int(d.Seconds())%60)
}
return fmt.Sprintf("%dh%dm", int(d.Hours()), int(d.Minutes())%60)
}
func formatPercent(numerator, denominator int64) string {
if denominator == 0 {
return "0.0%"
}
return fmt.Sprintf("%.1f%%", float64(numerator)/float64(denominator)*100)
}
func formatRatio(compressed, uncompressed int64) string {
if uncompressed == 0 {
return "1.00"
}
ratio := float64(compressed) / float64(uncompressed)
return fmt.Sprintf("%.2f", ratio)
}
func truncatePath(path string, maxLen int) string {
if len(path) <= maxLen {
return path
}
// Keep the last maxLen-3 characters and prepend "..."
return "..." + path[len(path)-(maxLen-3):]
}
// printUploadProgress prints upload progress
func (pr *ProgressReporter) printUploadProgress(info *UploadInfo) {
func (pr *ProgressReporter) printUploadProgress(_ *UploadInfo) {
// This function is called repeatedly during upload, not just at start
// Don't print anything here - the actual progress is shown by ReportUploadProgress
}
// ReportUploadStart marks the beginning of a blob upload
func (pr *ProgressReporter) ReportUploadStart(blobHash string, size int64) {
info := &UploadInfo{
BlobHash: blobHash,
Size: size,
StartTime: time.Now().UTC(),
}
pr.stats.CurrentUpload.Store(info)
// Log the start of upload
log.Info("Starting blob upload",
"hash", blobHash[:8]+"...",
"size", humanize.Bytes(uint64(size)))
}
// ReportUploadComplete marks the completion of a blob upload
func (pr *ProgressReporter) ReportUploadComplete(blobHash string, size int64, duration time.Duration) {
// Clear current upload
pr.stats.CurrentUpload.Store((*UploadInfo)(nil))
// Add to total upload duration
pr.stats.UploadDurationMs.Add(duration.Milliseconds())
// Calculate speed
if duration < time.Millisecond {
duration = time.Millisecond
}
bytesPerSec := float64(size) / duration.Seconds()
bitsPerSec := bytesPerSec * 8
// Format speed
var speedStr string
if bitsPerSec >= 1e9 {
speedStr = fmt.Sprintf("%.1fGbit/sec", bitsPerSec/1e9)
} else if bitsPerSec >= 1e6 {
speedStr = fmt.Sprintf("%.0fMbit/sec", bitsPerSec/1e6)
} else if bitsPerSec >= 1e3 {
speedStr = fmt.Sprintf("%.0fKbit/sec", bitsPerSec/1e3)
} else {
speedStr = fmt.Sprintf("%.0fbit/sec", bitsPerSec)
}
log.Info("Blob upload completed",
"hash", blobHash[:8]+"...",
"size", humanize.Bytes(uint64(size)),
"duration", formatDuration(duration),
"speed", speedStr)
}
// UpdateChunkingActivity updates the last chunking time
func (pr *ProgressReporter) UpdateChunkingActivity() {
pr.stats.mu.Lock()
pr.stats.lastChunkingTime = time.Now().UTC()
pr.stats.mu.Unlock()
}
// ReportUploadProgress reports current upload progress with instantaneous speed
func (pr *ProgressReporter) ReportUploadProgress(blobHash string, bytesUploaded, totalSize int64, instantSpeed float64) {
// Update the current upload info with progress
if uploadInfo, ok := pr.stats.CurrentUpload.Load().(*UploadInfo); ok && uploadInfo != nil {
now := time.Now()
// Only log at the configured interval
if now.Sub(uploadInfo.LastLogTime) >= UploadProgressInterval {
// Format speed in bits/second using humanize
bitsPerSec := instantSpeed * 8
speedStr := humanize.SI(bitsPerSec, "bit/sec")
percent := float64(bytesUploaded) / float64(totalSize) * 100
// Calculate ETA based on current speed
etaStr := "unknown"
if instantSpeed > 0 && bytesUploaded < totalSize {
remainingBytes := totalSize - bytesUploaded
remainingSeconds := float64(remainingBytes) / instantSpeed
eta := time.Duration(remainingSeconds * float64(time.Second))
etaStr = formatDuration(eta)
}
log.Info("Blob upload progress",
"hash", blobHash[:8]+"...",
"progress", fmt.Sprintf("%.1f%%", percent),
"uploaded", humanize.Bytes(uint64(bytesUploaded)),
"total", humanize.Bytes(uint64(totalSize)),
"speed", speedStr,
"eta", etaStr)
uploadInfo.LastLogTime = now
}
}
}

View File

@@ -36,5 +36,6 @@ const remoteKeyPrefix = "vaultik|"
func RemoteSnapshotKey(snapshotID string) string {
first := sha256.Sum256([]byte(remoteKeyPrefix + snapshotID))
second := sha256.Sum256(first[:])
return hex.EncodeToString(second[:])
}

File diff suppressed because it is too large Load Diff

View File

@@ -3,6 +3,7 @@ package snapshot_test
import (
"context"
"database/sql"
"os"
"path/filepath"
"testing"
"time"
@@ -14,70 +15,37 @@ import (
"sneak.berlin/go/vaultik/internal/types"
)
func TestScannerSimpleDirectory(t *testing.T) {
// Initialize logger for tests
// Shared test fixture values for the snapshot_test package.
const (
// testHost is the hostname recorded on test snapshot rows.
testHost = "test-host"
// testVersion is the vaultik version recorded on test snapshot rows.
testVersion = "test"
// testAgePublicKey is the fixed age public key used for test encryption.
testAgePublicKey = "age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05gl0sjq9q9wjg"
)
// TestMain initializes the shared logger once, before any tests run, so
// parallel tests never race on the logger's global state.
func TestMain(m *testing.M) {
log.Initialize(log.Config{})
os.Exit(m.Run())
}
// Create in-memory filesystem
fs := afero.NewMemMapFs()
// createTestSnapshotRecord inserts an empty snapshot row used as the
// association target for scan tests.
func createTestSnapshotRecord(
ctx context.Context, t *testing.T, repos *database.Repositories, snapshotID string,
) {
t.Helper()
// Create test directory structure
testFiles := map[string]string{
"/source/file1.txt": "Hello, world!", // 13 bytes
"/source/file2.txt": "This is another file", // 20 bytes
"/source/subdir/file3.txt": "File in subdirectory", // 20 bytes
"/source/subdir/file4.txt": "Another file in subdirectory", // 28 bytes
"/source/empty.txt": "", // 0 bytes
"/source/subdir2/file5.txt": "Yet another file", // 16 bytes
}
// Create files with specific times
testTime := time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC)
for path, content := range testFiles {
dir := filepath.Dir(path)
if err := fs.MkdirAll(dir, 0755); err != nil {
t.Fatalf("failed to create directory %s: %v", dir, err)
}
if err := afero.WriteFile(fs, path, []byte(content), 0644); err != nil {
t.Fatalf("failed to write file %s: %v", path, err)
}
// Set times
if err := fs.Chtimes(path, testTime, testTime); err != nil {
t.Fatalf("failed to set times for %s: %v", path, err)
}
}
// Create test database
db, err := database.NewTestDB()
if err != nil {
t.Fatalf("failed to create test database: %v", err)
}
defer func() {
if err := db.Close(); err != nil {
t.Errorf("failed to close database: %v", err)
}
}()
repos := database.NewRepositories(db)
// Create scanner
scanner := snapshot.NewScanner(snapshot.ScannerConfig{
FS: fs,
ChunkSize: int64(1024 * 16), // 16KB chunks for testing
Repositories: repos,
MaxBlobSize: int64(1024 * 1024), // 1MB blobs
CompressionLevel: 3,
AgeRecipients: []string{"age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05gl0sjq9q9wjg"}, // Test public key
})
// Create a snapshot record for testing
ctx := context.Background()
snapshotID := "test-snapshot-001"
err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
err := repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
snapshot := &database.Snapshot{
ID: types.SnapshotID(snapshotID),
Hostname: "test-host",
VaultikVersion: "test",
Hostname: testHost,
VaultikVersion: testVersion,
StartedAt: time.Now(),
CompletedAt: nil,
FileCount: 0,
@@ -87,28 +55,20 @@ func TestScannerSimpleDirectory(t *testing.T) {
BlobSize: 0,
CompressionRatio: 1.0,
}
return repos.Snapshots.Create(ctx, tx, snapshot)
})
if err != nil {
t.Fatalf("failed to create snapshot: %v", err)
}
}
// Scan the directory
var result *snapshot.ScanResult
result, err = scanner.Scan(ctx, "/source", snapshotID)
if err != nil {
t.Fatalf("scan failed: %v", err)
}
// Verify results - we only scan regular files, not directories
if result.FilesScanned != 6 {
t.Errorf("expected 6 files scanned, got %d", result.FilesScanned)
}
// Total bytes should be the sum of all file contents
if result.BytesScanned < 97 { // At minimum we have 97 bytes of file content
t.Errorf("expected at least 97 bytes scanned, got %d", result.BytesScanned)
}
// verifySimpleScanDatabase checks the database contents produced by
// TestScannerSimpleDirectory's scan.
func verifySimpleScanDatabase(
ctx context.Context, t *testing.T, repos *database.Repositories,
) {
t.Helper()
// Verify files in database - includes regular files and directories
files, err := repos.Files.ListByPrefix(ctx, "/source")
@@ -159,9 +119,99 @@ func TestScannerSimpleDirectory(t *testing.T) {
}
}
func TestScannerSimpleDirectory(t *testing.T) {
// Initialize logger for tests
log.Initialize(log.Config{})
t.Parallel()
// Create in-memory filesystem
fs := afero.NewMemMapFs()
// Create test directory structure
testFiles := map[string]string{
"/source/file1.txt": "Hello, world!", // 13 bytes
"/source/file2.txt": "This is another file", // 20 bytes
"/source/subdir/file3.txt": "File in subdirectory", // 20 bytes
"/source/subdir/file4.txt": "Another file in subdirectory", // 28 bytes
"/source/empty.txt": "", // 0 bytes
"/source/subdir2/file5.txt": "Yet another file", // 16 bytes
}
// Create files with specific times
testTime := time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC)
for path, content := range testFiles {
dir := filepath.Dir(path)
err := fs.MkdirAll(dir, 0755)
if err != nil {
t.Fatalf("failed to create directory %s: %v", dir, err)
}
err = afero.WriteFile(fs, path, []byte(content), 0644)
if err != nil {
t.Fatalf("failed to write file %s: %v", path, err)
}
// Set times
err = fs.Chtimes(path, testTime, testTime)
if err != nil {
t.Fatalf("failed to set times for %s: %v", path, err)
}
}
// Create test database
db, err := database.NewTestDB()
if err != nil {
t.Fatalf("failed to create test database: %v", err)
}
defer func() {
err := db.Close()
if err != nil {
t.Errorf("failed to close database: %v", err)
}
}()
repos := database.NewRepositories(db)
// Create scanner
scanner := snapshot.NewScanner(snapshot.ScannerConfig{
FS: fs,
ChunkSize: int64(1024 * 16), // 16KB chunks for testing
Repositories: repos,
MaxBlobSize: int64(1024 * 1024), // 1MB blobs
CompressionLevel: 3,
AgeRecipients: []string{testAgePublicKey},
})
// Create a snapshot record for testing
ctx := context.Background()
snapshotID := "test-snapshot-001"
createTestSnapshotRecord(ctx, t, repos, snapshotID)
// Scan the directory
result, err := scanner.Scan(ctx, "/source", snapshotID)
if err != nil {
t.Fatalf("scan failed: %v", err)
}
// Verify results - we only scan regular files, not directories
if result.FilesScanned != 6 {
t.Errorf("expected 6 files scanned, got %d", result.FilesScanned)
}
// Total bytes should be the sum of all file contents
if result.BytesScanned < 97 { // At minimum we have 97 bytes of file content
t.Errorf("expected at least 97 bytes scanned, got %d", result.BytesScanned)
}
verifySimpleScanDatabase(ctx, t, repos)
}
func TestScannerLargeFile(t *testing.T) {
// Initialize logger for tests
log.Initialize(log.Config{})
t.Parallel()
// Create in-memory filesystem
fs := afero.NewMemMapFs()
@@ -170,15 +220,19 @@ func TestScannerLargeFile(t *testing.T) {
// Use random content to ensure good chunk boundaries
largeContent := make([]byte, 1024*1024) // 1MB
// Fill with pseudo-random data to ensure chunk boundaries
for i := 0; i < len(largeContent); i++ {
for i := range largeContent {
// Simple pseudo-random generator for deterministic tests
//nolint:gosec // G115: intentional byte truncation of test data
largeContent[i] = byte((i * 7919) ^ (i >> 3))
}
if err := fs.MkdirAll("/source", 0755); err != nil {
err := fs.MkdirAll("/source", 0755)
if err != nil {
t.Fatal(err)
}
if err := afero.WriteFile(fs, "/source/large.bin", largeContent, 0644); err != nil {
err = afero.WriteFile(fs, "/source/large.bin", largeContent, 0644)
if err != nil {
t.Fatal(err)
}
@@ -188,7 +242,8 @@ func TestScannerLargeFile(t *testing.T) {
t.Fatalf("failed to create test database: %v", err)
}
defer func() {
if err := db.Close(); err != nil {
err := db.Close()
if err != nil {
t.Errorf("failed to close database: %v", err)
}
}()
@@ -202,35 +257,17 @@ func TestScannerLargeFile(t *testing.T) {
Repositories: repos,
MaxBlobSize: int64(1024 * 1024),
CompressionLevel: 3,
AgeRecipients: []string{"age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05gl0sjq9q9wjg"}, // Test public key
AgeRecipients: []string{testAgePublicKey},
})
// Create a snapshot record for testing
ctx := context.Background()
snapshotID := "test-snapshot-001"
err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
snapshot := &database.Snapshot{
ID: types.SnapshotID(snapshotID),
Hostname: "test-host",
VaultikVersion: "test",
StartedAt: time.Now(),
CompletedAt: nil,
FileCount: 0,
ChunkCount: 0,
BlobCount: 0,
TotalSize: 0,
BlobSize: 0,
CompressionRatio: 1.0,
}
return repos.Snapshots.Create(ctx, tx, snapshot)
})
if err != nil {
t.Fatalf("failed to create snapshot: %v", err)
}
createTestSnapshotRecord(ctx, t, repos, snapshotID)
// Scan the directory
var result *snapshot.ScanResult
result, err = scanner.Scan(ctx, "/source", snapshotID)
result, err := scanner.Scan(ctx, "/source", snapshotID)
if err != nil {
t.Fatalf("scan failed: %v", err)
}
@@ -242,7 +279,8 @@ func TestScannerLargeFile(t *testing.T) {
// The file size should be at least 1MB
if result.BytesScanned < 1024*1024 {
t.Errorf("expected at least %d bytes scanned, got %d", 1024*1024, result.BytesScanned)
t.Errorf("expected at least %d bytes scanned, got %d",
1024*1024, result.BytesScanned)
}
// Verify chunks

File diff suppressed because it is too large Load Diff

View File

@@ -1,3 +1,4 @@
//nolint:testpackage // exercises unexported SnapshotManager internals
package snapshot
import (
@@ -33,12 +34,69 @@ func copyFile(fs afero.Fs, src, dst string) error {
defer func() { _ = destFile.Close() }()
_, err = io.Copy(destFile, sourceFile)
return err
}
// verifyCleanedDB opens the cleaned database and checks that the kept
// snapshot survived while the orphan file and chunk were removed.
func verifyCleanedDB(
ctx context.Context,
t *testing.T,
tempDBPath, snapshotID string,
file *database.File,
chunk *database.Chunk,
) {
t.Helper()
cleanedDB, err := database.New(ctx, tempDBPath)
if err != nil {
t.Fatalf("failed to open cleaned database: %v", err)
}
defer func() {
err := cleanedDB.Close()
if err != nil {
t.Errorf("failed to close database: %v", err)
}
}()
cleanedRepos := database.NewRepositories(cleanedDB)
// Verify snapshot exists
verifySnapshot, err := cleanedRepos.Snapshots.GetByID(ctx, snapshotID)
if err != nil {
t.Fatalf("failed to get snapshot: %v", err)
}
if verifySnapshot == nil {
t.Error("snapshot should exist")
}
// Verify orphan file is gone
f, err := cleanedRepos.Files.GetByPath(ctx, file.Path.String())
if err != nil {
t.Fatalf("failed to check file: %v", err)
}
if f != nil {
t.Error("orphan file should not exist")
}
// Verify orphan chunk is gone
c, err := cleanedRepos.Chunks.GetByHash(ctx, chunk.ChunkHash.String())
if err != nil {
t.Fatalf("failed to check chunk: %v", err)
}
if c != nil {
t.Error("orphan chunk should not exist")
}
}
func TestCleanSnapshotDBEmptySnapshot(t *testing.T) {
// Initialize logger
log.Initialize(log.Config{})
t.Parallel()
ctx := context.Background()
fs := afero.NewOsFs()
@@ -46,6 +104,7 @@ func TestCleanSnapshotDBEmptySnapshot(t *testing.T) {
// Create a test database
tempDir := t.TempDir()
dbPath := filepath.Join(tempDir, "test.db")
db, err := database.New(ctx, dbPath)
if err != nil {
t.Fatalf("failed to create database: %v", err)
@@ -71,9 +130,11 @@ func TestCleanSnapshotDBEmptySnapshot(t *testing.T) {
chunk := &database.Chunk{ChunkHash: "orphan-chunk", Size: 500}
err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
if err := repos.Files.Create(ctx, tx, file); err != nil {
err := repos.Files.Create(ctx, tx, file)
if err != nil {
return err
}
return repos.Chunks.Create(ctx, tx, chunk)
})
if err != nil {
@@ -81,13 +142,16 @@ func TestCleanSnapshotDBEmptySnapshot(t *testing.T) {
}
// Close the database
if err := db.Close(); err != nil {
err = db.Close()
if err != nil {
t.Fatalf("failed to close database: %v", err)
}
// Copy database
tempDBPath := filepath.Join(tempDir, "temp.db")
if err := copyFile(fs, dbPath, tempDBPath); err != nil {
err = copyFile(fs, dbPath, tempDBPath)
if err != nil {
t.Fatalf("failed to copy database: %v", err)
}
@@ -101,54 +165,20 @@ func TestCleanSnapshotDBEmptySnapshot(t *testing.T) {
config: cfg,
fs: fs,
}
if _, err := sm.cleanSnapshotDB(ctx, tempDBPath, snapshot.ID.String()); err != nil {
_, err = sm.cleanSnapshotDB(ctx, tempDBPath, snapshot.ID.String())
if err != nil {
t.Fatalf("failed to clean snapshot database: %v", err)
}
// Verify the cleaned database
cleanedDB, err := database.New(ctx, tempDBPath)
if err != nil {
t.Fatalf("failed to open cleaned database: %v", err)
}
defer func() {
if err := cleanedDB.Close(); err != nil {
t.Errorf("failed to close database: %v", err)
}
}()
cleanedRepos := database.NewRepositories(cleanedDB)
// Verify snapshot exists
verifySnapshot, err := cleanedRepos.Snapshots.GetByID(ctx, snapshot.ID.String())
if err != nil {
t.Fatalf("failed to get snapshot: %v", err)
}
if verifySnapshot == nil {
t.Error("snapshot should exist")
}
// Verify orphan file is gone
f, err := cleanedRepos.Files.GetByPath(ctx, file.Path.String())
if err != nil {
t.Fatalf("failed to check file: %v", err)
}
if f != nil {
t.Error("orphan file should not exist")
}
// Verify orphan chunk is gone
c, err := cleanedRepos.Chunks.GetByHash(ctx, chunk.ChunkHash.String())
if err != nil {
t.Fatalf("failed to check chunk: %v", err)
}
if c != nil {
t.Error("orphan chunk should not exist")
}
verifyCleanedDB(ctx, t, tempDBPath, snapshot.ID.String(), file, chunk)
}
func TestCleanSnapshotDBNonExistentSnapshot(t *testing.T) {
// Initialize logger
log.Initialize(log.Config{})
t.Parallel()
ctx := context.Background()
fs := afero.NewOsFs()
@@ -156,19 +186,23 @@ func TestCleanSnapshotDBNonExistentSnapshot(t *testing.T) {
// Create a test database
tempDir := t.TempDir()
dbPath := filepath.Join(tempDir, "test.db")
db, err := database.New(ctx, dbPath)
if err != nil {
t.Fatalf("failed to create database: %v", err)
}
// Close immediately
if err := db.Close(); err != nil {
err = db.Close()
if err != nil {
t.Fatalf("failed to close database: %v", err)
}
// Copy database
tempDBPath := filepath.Join(tempDir, "temp.db")
if err := copyFile(fs, dbPath, tempDBPath); err != nil {
err = copyFile(fs, dbPath, tempDBPath)
if err != nil {
t.Fatalf("failed to copy database: %v", err)
}

View File

@@ -42,18 +42,19 @@ func (f *FileStorer) SetFilesystem(fs afero.Fs) {
f.fs = fs
}
// fullPath returns the full filesystem path for a key.
func (f *FileStorer) fullPath(key string) string {
return filepath.Join(f.basePath, key)
}
// storageDirPerm is the mode used for directories created under the
// storage base path.
const storageDirPerm = 0o755
// Put stores data at the specified key.
func (f *FileStorer) Put(ctx context.Context, key string, data io.Reader) error {
func (f *FileStorer) Put(_ context.Context, key string, data io.Reader) error {
path := f.fullPath(key)
// Create parent directories
dir := filepath.Dir(path)
if err := f.fs.MkdirAll(dir, 0755); err != nil {
err := f.fs.MkdirAll(dir, storageDirPerm)
if err != nil {
return fmt.Errorf("creating directories: %w", err)
}
@@ -63,7 +64,8 @@ func (f *FileStorer) Put(ctx context.Context, key string, data io.Reader) error
}
defer func() { _ = file.Close() }()
if _, err := io.Copy(file, data); err != nil {
_, err = io.Copy(file, data)
if err != nil {
return fmt.Errorf("writing file: %w", err)
}
@@ -71,12 +73,17 @@ func (f *FileStorer) Put(ctx context.Context, key string, data io.Reader) error
}
// PutWithProgress stores data with progress reporting.
func (f *FileStorer) PutWithProgress(ctx context.Context, key string, data io.Reader, size int64, progress ProgressCallback) error {
func (f *FileStorer) PutWithProgress(
_ context.Context, key string, data io.Reader,
_ int64, progress ProgressCallback,
) error {
path := f.fullPath(key)
// Create parent directories
dir := filepath.Dir(path)
if err := f.fs.MkdirAll(dir, 0755); err != nil {
err := f.fs.MkdirAll(dir, storageDirPerm)
if err != nil {
return fmt.Errorf("creating directories: %w", err)
}
@@ -92,7 +99,8 @@ func (f *FileStorer) PutWithProgress(ctx context.Context, key string, data io.Re
callback: progress,
}
if _, err := io.Copy(pw, data); err != nil {
_, err = io.Copy(pw, data)
if err != nil {
return fmt.Errorf("writing file: %w", err)
}
@@ -100,28 +108,34 @@ func (f *FileStorer) PutWithProgress(ctx context.Context, key string, data io.Re
}
// Get retrieves data from the specified key.
func (f *FileStorer) Get(ctx context.Context, key string) (io.ReadCloser, error) {
func (f *FileStorer) Get(_ context.Context, key string) (io.ReadCloser, error) {
path := f.fullPath(key)
file, err := f.fs.Open(path)
if err != nil {
if os.IsNotExist(err) {
return nil, ErrNotFound
}
return nil, fmt.Errorf("opening file: %w", err)
}
return file, nil
}
// Stat returns metadata about an object without retrieving its contents.
func (f *FileStorer) Stat(ctx context.Context, key string) (*ObjectInfo, error) {
func (f *FileStorer) Stat(_ context.Context, key string) (*ObjectInfo, error) {
path := f.fullPath(key)
info, err := f.fs.Stat(path)
if err != nil {
if os.IsNotExist(err) {
return nil, ErrNotFound
}
return nil, fmt.Errorf("stat file: %w", err)
}
return &ObjectInfo{
Key: key,
Size: info.Size(),
@@ -129,21 +143,25 @@ func (f *FileStorer) Stat(ctx context.Context, key string) (*ObjectInfo, error)
}
// Delete removes an object.
func (f *FileStorer) Delete(ctx context.Context, key string) error {
func (f *FileStorer) Delete(_ context.Context, key string) error {
path := f.fullPath(key)
err := f.fs.Remove(path)
if os.IsNotExist(err) {
return nil // Match S3 behavior: no error if doesn't exist
}
if err != nil {
return fmt.Errorf("removing file: %w", err)
}
return nil
}
// List returns all keys with the given prefix.
func (f *FileStorer) List(ctx context.Context, prefix string) ([]string, error) {
var keys []string
basePath := f.fullPath(prefix)
// Check if base path exists
@@ -151,6 +169,7 @@ func (f *FileStorer) List(ctx context.Context, prefix string) ([]string, error)
if err != nil {
return nil, fmt.Errorf("checking path: %w", err)
}
if !exists {
return keys, nil // Empty list for non-existent prefix
}
@@ -177,9 +196,9 @@ func (f *FileStorer) List(ctx context.Context, prefix string) ([]string, error)
relPath = strings.ReplaceAll(relPath, string(filepath.Separator), "/")
keys = append(keys, relPath)
}
return nil
})
if err != nil {
return nil, fmt.Errorf("walking directory: %w", err)
}
@@ -192,14 +211,17 @@ func (f *FileStorer) ListStream(ctx context.Context, prefix string) <-chan Objec
ch := make(chan ObjectInfo)
go func() {
defer close(ch)
basePath := f.fullPath(prefix)
// Check if base path exists
exists, err := afero.Exists(f.fs, basePath)
if err != nil {
ch <- ObjectInfo{Err: fmt.Errorf("checking path: %w", err)}
return
}
if !exists {
return // Empty channel for non-existent prefix
}
@@ -209,19 +231,22 @@ func (f *FileStorer) ListStream(ctx context.Context, prefix string) <-chan Objec
select {
case <-ctx.Done():
ch <- ObjectInfo{Err: ctx.Err()}
return ctx.Err()
default:
}
if err != nil {
ch <- ObjectInfo{Err: err}
return nil // Continue walking despite errors
return nil //nolint:nilerr // continue walking despite errors
}
if !info.IsDir() {
relPath, err := filepath.Rel(f.basePath, path)
if err != nil {
ch <- ObjectInfo{Err: fmt.Errorf("computing relative path: %w", err)}
return nil
}
// Normalize path separators
@@ -231,20 +256,27 @@ func (f *FileStorer) ListStream(ctx context.Context, prefix string) <-chan Objec
Size: info.Size(),
}
}
return nil
})
}()
return ch
}
// Info returns human-readable storage location information.
func (f *FileStorer) Info() StorageInfo {
return StorageInfo{
Type: "file",
func (f *FileStorer) Info() Info {
return Info{
Type: schemeFile,
Location: f.basePath,
}
}
// fullPath returns the full filesystem path for a key.
func (f *FileStorer) fullPath(key string) string {
return filepath.Join(f.basePath, key)
}
// progressWriter wraps an io.Writer to track write progress.
type progressWriter struct {
writer io.Writer
@@ -257,10 +289,12 @@ func (pw *progressWriter) Write(p []byte) (int, error) {
if n > 0 {
pw.written += int64(n)
if pw.callback != nil {
if callbackErr := pw.callback(pw.written); callbackErr != nil {
callbackErr := pw.callback(pw.written)
if callbackErr != nil {
return n, callbackErr
}
}
}
return n, err
}

View File

@@ -10,9 +10,17 @@ import (
"sneak.berlin/go/vaultik/internal/s3"
)
// defaultS3Region is used when neither the URL nor the config specify one.
const defaultS3Region = "us-east-1"
// defaultS3Endpoint is the AWS endpoint used when none is configured.
const defaultS3Endpoint = "s3.amazonaws.com"
// Module exports storage functionality as an fx module.
// It provides a Storer implementation based on the configured storage URL
// or falls back to legacy S3 configuration.
//
//nolint:gochecknoglobals // fx module definitions are package globals
var Module = fx.Module("storage",
fx.Provide(NewStorer),
)
@@ -20,13 +28,17 @@ var Module = fx.Module("storage",
// NewStorer creates a Storer based on configuration.
// If StorageURL is set, it uses URL-based configuration.
// Otherwise, it falls back to legacy S3 configuration.
//
//nolint:ireturn // fx provider intentionally returns the Storer interface
func NewStorer(cfg *config.Config) (Storer, error) {
if cfg.StorageURL != "" {
return storerFromURL(cfg.StorageURL, cfg)
}
return storerFromLegacyS3Config(cfg)
}
//nolint:ireturn // factory intentionally returns the Storer interface
func storerFromURL(rawURL string, cfg *config.Config) (Storer, error) {
parsed, err := ParseStorageURL(rawURL)
if err != nil {
@@ -34,28 +46,48 @@ func storerFromURL(rawURL string, cfg *config.Config) (Storer, error) {
}
switch parsed.Scheme {
case "file":
case schemeFile:
return NewFileStorer(parsed.Prefix)
case "s3":
case schemeS3:
return storerFromParsedS3URL(parsed, cfg)
case schemeRclone:
return NewRcloneStorer(
context.Background(), parsed.RcloneRemote, parsed.Prefix)
default:
return nil, fmt.Errorf("%w: %s", ErrUnsupportedStorage, parsed.Scheme)
}
}
// storerFromParsedS3URL builds an S3 storer from a parsed s3:// URL,
// filling endpoint protocol and region defaults from the config.
//
//nolint:ireturn // factory intentionally returns the Storer interface
func storerFromParsedS3URL(parsed *URL, cfg *config.Config) (Storer, error) {
// Build endpoint URL
endpoint := parsed.Endpoint
if endpoint == "" {
endpoint = "s3.amazonaws.com"
endpoint = defaultS3Endpoint
}
// Add protocol if not present
if parsed.UseSSL && !strings.HasPrefix(endpoint, "https://") && !strings.HasPrefix(endpoint, "http://") {
hasProtocol := strings.HasPrefix(endpoint, "https://") ||
strings.HasPrefix(endpoint, "http://")
if !hasProtocol {
if parsed.UseSSL {
endpoint = "https://" + endpoint
} else if !parsed.UseSSL && !strings.HasPrefix(endpoint, "http://") && !strings.HasPrefix(endpoint, "https://") {
} else {
endpoint = "http://" + endpoint
}
}
region := parsed.Region
if region == "" {
region = cfg.S3.Region
if region == "" {
region = "us-east-1"
region = defaultS3Region
}
}
@@ -71,21 +103,17 @@ func storerFromURL(rawURL string, cfg *config.Config) (Storer, error) {
if err != nil {
return nil, fmt.Errorf("creating S3 client: %w", err)
}
return NewS3Storer(client), nil
case "rclone":
return NewRcloneStorer(context.Background(), parsed.RcloneRemote, parsed.Prefix)
default:
return nil, fmt.Errorf("unsupported storage scheme: %s", parsed.Scheme)
}
}
//nolint:ireturn // factory intentionally returns the Storer interface
func storerFromLegacyS3Config(cfg *config.Config) (Storer, error) {
endpoint := cfg.S3.Endpoint
// Ensure protocol is present
if !strings.HasPrefix(endpoint, "http://") && !strings.HasPrefix(endpoint, "https://") {
if !strings.HasPrefix(endpoint, "http://") &&
!strings.HasPrefix(endpoint, "https://") {
if cfg.S3.UseSSL {
endpoint = "https://" + endpoint
} else {
@@ -95,7 +123,7 @@ func storerFromLegacyS3Config(cfg *config.Config) (Storer, error) {
region := cfg.S3.Region
if region == "" {
region = "us-east-1"
region = defaultS3Region
}
client, err := s3.NewClient(context.Background(), s3.Config{
@@ -109,5 +137,6 @@ func storerFromLegacyS3Config(cfg *config.Config) (Storer, error) {
if err != nil {
return nil, fmt.Errorf("creating S3 client: %w", err)
}
return NewS3Storer(client), nil
}

View File

@@ -49,6 +49,7 @@ func NewRcloneStorer(ctx context.Context, remote, path string) (*RcloneStorer, e
strings.Contains(err.Error(), "failed to find remote") {
return nil, fmt.Errorf("%w: %s", ErrRemoteNotFound, remote)
}
return nil, fmt.Errorf("creating rclone filesystem: %w", err)
}
@@ -68,7 +69,8 @@ func (r *RcloneStorer) Put(ctx context.Context, key string, data io.Reader) erro
}
// Upload the object
_, err = operations.Rcat(ctx, r.fsys, key, io.NopCloser(bytes.NewReader(buf)), time.Now(), nil)
_, err = operations.Rcat(ctx, r.fsys, key,
io.NopCloser(bytes.NewReader(buf)), time.Now(), nil)
if err != nil {
return fmt.Errorf("uploading object: %w", err)
}
@@ -77,7 +79,10 @@ func (r *RcloneStorer) Put(ctx context.Context, key string, data io.Reader) erro
}
// PutWithProgress stores data with progress reporting.
func (r *RcloneStorer) PutWithProgress(ctx context.Context, key string, data io.Reader, size int64, progress ProgressCallback) error {
func (r *RcloneStorer) PutWithProgress(
ctx context.Context, key string, data io.Reader,
_ int64, progress ProgressCallback,
) error {
// Wrap reader with progress tracking
pr := &progressReader{
reader: data,
@@ -101,9 +106,11 @@ func (r *RcloneStorer) Get(ctx context.Context, key string) (io.ReadCloser, erro
if errors.Is(err, fs.ErrorObjectNotFound) {
return nil, ErrNotFound
}
if errors.Is(err, fs.ErrorDirNotFound) {
return nil, ErrNotFound
}
return nil, fmt.Errorf("getting object: %w", err)
}
@@ -123,9 +130,11 @@ func (r *RcloneStorer) Stat(ctx context.Context, key string) (*ObjectInfo, error
if errors.Is(err, fs.ErrorObjectNotFound) {
return nil, ErrNotFound
}
if errors.Is(err, fs.ErrorDirNotFound) {
return nil, ErrNotFound
}
return nil, fmt.Errorf("getting object: %w", err)
}
@@ -142,13 +151,16 @@ func (r *RcloneStorer) Delete(ctx context.Context, key string) error {
if errors.Is(err, fs.ErrorObjectNotFound) {
return nil // Match S3 behavior: no error if doesn't exist
}
if errors.Is(err, fs.ErrorDirNotFound) {
return nil
}
return fmt.Errorf("getting object: %w", err)
}
if err := obj.Remove(ctx); err != nil {
err = obj.Remove(ctx)
if err != nil {
return fmt.Errorf("removing object: %w", err)
}
@@ -173,7 +185,9 @@ func (r *RcloneStorer) List(ctx context.Context, prefix string) ([]string, error
}
// ListStream returns a channel of ObjectInfo for large result sets.
func (r *RcloneStorer) ListStream(ctx context.Context, prefix string) <-chan ObjectInfo {
func (r *RcloneStorer) ListStream(
ctx context.Context, prefix string,
) <-chan ObjectInfo {
ch := make(chan ObjectInfo)
go func() {
@@ -204,13 +218,14 @@ func (r *RcloneStorer) ListStream(ctx context.Context, prefix string) <-chan Obj
}
// Info returns human-readable storage location information.
func (r *RcloneStorer) Info() StorageInfo {
func (r *RcloneStorer) Info() Info {
location := r.remote
if r.path != "" {
location += ":" + r.path
}
return StorageInfo{
Type: "rclone",
return Info{
Type: schemeRclone,
Location: location,
}
}
@@ -227,10 +242,12 @@ func (pr *progressReader) Read(p []byte) (int, error) {
if n > 0 {
pr.read += int64(n)
if pr.callback != nil {
if callbackErr := pr.callback(pr.read); callbackErr != nil {
callbackErr := pr.callback(pr.read)
if callbackErr != nil {
return n, callbackErr
}
}
}
return n, err
}

View File

@@ -24,12 +24,16 @@ func (s *S3Storer) Put(ctx context.Context, key string, data io.Reader) error {
}
// PutWithProgress stores data with progress reporting.
func (s *S3Storer) PutWithProgress(ctx context.Context, key string, data io.Reader, size int64, progress ProgressCallback) error {
func (s *S3Storer) PutWithProgress(
ctx context.Context, key string, data io.Reader,
size int64, progress ProgressCallback,
) error {
// Convert storage.ProgressCallback to s3.ProgressCallback
var s3Progress s3.ProgressCallback
if progress != nil {
s3Progress = s3.ProgressCallback(progress)
}
return s.client.PutObjectWithProgress(ctx, key, data, size, s3Progress)
}
@@ -44,6 +48,7 @@ func (s *S3Storer) Stat(ctx context.Context, key string) (*ObjectInfo, error) {
if err != nil {
return nil, err
}
return &ObjectInfo{
Key: info.Key,
Size: info.Size,
@@ -65,6 +70,7 @@ func (s *S3Storer) ListStream(ctx context.Context, prefix string) <-chan ObjectI
ch := make(chan ObjectInfo)
go func() {
defer close(ch)
for info := range s.client.ListObjectsStream(ctx, prefix, false) {
ch <- ObjectInfo{
Key: info.Key,
@@ -73,12 +79,13 @@ func (s *S3Storer) ListStream(ctx context.Context, prefix string) <-chan ObjectI
}
}
}()
return ch
}
// Info returns human-readable storage location information.
func (s *S3Storer) Info() StorageInfo {
return StorageInfo{
func (s *S3Storer) Info() Info {
return Info{
Type: "s3",
Location: fmt.Sprintf("%s/%s", s.client.Endpoint(), s.client.BucketName()),
}

View File

@@ -30,14 +30,15 @@ type ObjectInfo struct {
Err error // Error for streaming results (nil on success)
}
// StorageInfo provides human-readable storage configuration.
type StorageInfo struct {
// Info provides human-readable storage configuration.
type Info struct {
Type string // "s3" or "file"
Location string // endpoint/bucket for S3, base path for filesystem
}
// Storer defines the interface for storage backends.
// All paths are relative to the storage root (bucket/prefix for S3, base directory for filesystem).
// All paths are relative to the storage root (bucket/prefix for S3, base
// directory for filesystem).
type Storer interface {
// Put stores data at the specified key.
// Parent directories are created automatically for filesystem backends.
@@ -46,7 +47,8 @@ type Storer interface {
// PutWithProgress stores data with progress reporting.
// Size must be the exact size of the data to store.
// The progress callback is called periodically with bytes transferred.
PutWithProgress(ctx context.Context, key string, data io.Reader, size int64, progress ProgressCallback) error
PutWithProgress(ctx context.Context, key string, data io.Reader,
size int64, progress ProgressCallback) error
// Get retrieves data from the specified key.
// The caller must close the returned ReadCloser.
@@ -70,5 +72,5 @@ type Storer interface {
ListStream(ctx context.Context, prefix string) <-chan ObjectInfo
// Info returns human-readable storage location information.
Info() StorageInfo
Info() Info
}

Some files were not shown because too many files have changed in this diff Show More