sneak/vaultik
1
0
Fork 0
Code Issues 19 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Update golangci-lint to v2.12.2 with canonical config (#62)
All checks were successful
check / check (push) Successful in 5s
Details

Browse Source 
Updates golangci-lint to v2.12.2 everywhere it is pinned and installs the canonical `.golangci.yml`, then remediates every finding the new linter/config surfaces so `make check` is green.

## Version bump

- `Dockerfile` lint stage: `golangci/golangci-lint:v2.11.3-alpine` -> `v2.12.2-alpine` (digest-pinned, date comment updated)
- `Makefile` `deps` target: `go install` moved from the old v1 module path at `@latest` to the pinned `github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.12.2`
- `.golangci.yml` replaced with the canonical config (v2 schema; settings under `linters.settings` so the thresholds actually apply; `default: all` with the standard six disables)
- `script/bootstrap` installs golangci-lint via the system package manager and carries no version pin, so it is unchanged
- CI (`.gitea/workflows/check.yml`) only runs `script/cibuild`, so it needed no change

## Lint remediation

The canonical config surfaced ~3,300 findings across 56k lines. All are fixed, behavior-preserving; incorporates and supersedes the per-package mechanical passes already merged to `main` (refs #61). Highlights:

- `err113`: dynamic errors replaced with package sentinels + `%w` wrapping; comparisons via `errors.Is`
- `goprintffuncname`: printf-style helpers renamed with an `f` suffix (`ui.Writer` message methods, `cli.ReportErrorf`, `database.Fatalf`) and all call sites updated
- `revive` stutter renames: `blob.Handler`, `blob.WithReader`, `blob.ChunkPosition`, `storage.URL`, `storage.Info`; missing doc comments added
- `contextcheck`/`noctx`: `context.Context` threaded through `blob.Packer` and the scanner call sites; context-aware `exec`/`sql` variants
- `funlen`/`cyclop`/`gocognit`/`dupl`: oversized and duplicated functions split into focused helpers (production and test code)
- tests: `t.Parallel()` added where safe (global logger init kept in the serial phase for `-race`), `t.TempDir()`/`t.Helper()` adopted, several suites converted to external test packages
- `gosec`: bounded integer conversions, `ReadHeaderTimeout` on the test HTTP server; remaining warnings suppressed per-site with justifications
- remaining `nolint` directives are rare, targeted, and each carries a reason (e.g. `nilnil` not-found contract in the repository layer, fx module globals, on-disk snake_case struct tags)
- removed the deprecated `log.LogOptions` alias (callers migrated to `log.Options`)

`make check` (tests with `-race`, lint, fmt-check) passes.

Co-authored-by: sneak <sneak@sneak.berlin>
Reviewed-on: #62
Co-authored-by: clawbot <clawbot@noreply.example.org>
Co-committed-by: clawbot <clawbot@noreply.example.org>
This commit was merged in pull request #62.
This commit is contained in:
clawbot
2026-08-07 23:22:48 +02:00
committed by Jeffrey Paul
parent b87b72d4b9
commit cc58583130
126 changed files with 8184 additions and 5470 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
internal/database/blob_chunks.go
View File

@@ -7,15 +7,21 @@ import (
"fmt"
)
// BlobChunkRepository provides access to the blob_chunks table, which maps
// blobs to the chunks they contain (with offset and length).
type BlobChunkRepository struct {
db *DB
}
// NewBlobChunkRepository creates a BlobChunkRepository backed by db.
func NewBlobChunkRepository(db *DB) *BlobChunkRepository {
return &BlobChunkRepository{db: db}
}
func (r *BlobChunkRepository) Create(ctx context.Context, tx *sql.Tx, bc *BlobChunk) error {
// Create inserts a blob_chunks row, using tx when non-nil.
func (r *BlobChunkRepository) Create(
ctx context.Context, tx *sql.Tx, bc *BlobChunk,
) error {
query := `
INSERT INTO blob_chunks (blob_id, chunk_hash, offset, length)
VALUES (?, ?, ?, ?)
@@ -35,7 +41,11 @@ func (r *BlobChunkRepository) Create(ctx context.Context, tx *sql.Tx, bc *BlobCh
return nil
}
func (r *BlobChunkRepository) GetByBlobID(ctx context.Context, blobID string) ([]*BlobChunk, error) {
// GetByBlobID returns all chunks contained in the given blob, ordered by
// their offset within the blob.
func (r *BlobChunkRepository) GetByBlobID(
ctx context.Context, blobID string,
) ([]*BlobChunk, error) {
query := `
SELECT blob_id, chunk_hash, offset, length
FROM blob_chunks
@@ -65,7 +75,11 @@ func (r *BlobChunkRepository) GetByBlobID(ctx context.Context, blobID string) ([
return blobChunks, rows.Err()
}
func (r *BlobChunkRepository) GetByChunkHash(ctx context.Context, chunkHash string) (*BlobChunk, error) {
// GetByChunkHash returns one blob_chunks row containing the given chunk,
// or nil if the chunk is not packed in any blob.
func (r *BlobChunkRepository) GetByChunkHash(
ctx context.Context, chunkHash string,
) (*BlobChunk, error) {
query := `
SELECT blob_id, chunk_hash, offset, length
FROM blob_chunks
@@ -87,7 +101,7 @@ func (r *BlobChunkRepository) GetByChunkHash(ctx context.Context, chunkHash stri
if errors.Is(err, sql.ErrNoRows) {
LogSQL("GetByChunkHash", "No rows found", chunkHash)
return nil, nil
return nil, nil //nolint:nilnil // nil,nil signals not-found; callers check nil
}
if err != nil {
@@ -102,7 +116,9 @@ func (r *BlobChunkRepository) GetByChunkHash(ctx context.Context, chunkHash stri
}
// GetByChunkHashTx retrieves a blob chunk within a transaction
func (r *BlobChunkRepository) GetByChunkHashTx(ctx context.Context, tx *sql.Tx, chunkHash string) (*BlobChunk, error) {
func (r *BlobChunkRepository) GetByChunkHashTx(
ctx context.Context, tx *sql.Tx, chunkHash string,
) (*BlobChunk, error) {
query := `
SELECT blob_id, chunk_hash, offset, length
FROM blob_chunks
@@ -124,7 +140,7 @@ func (r *BlobChunkRepository) GetByChunkHashTx(ctx context.Context, tx *sql.Tx,
if errors.Is(err, sql.ErrNoRows) {
LogSQL("GetByChunkHashTx", "No rows found", chunkHash)
return nil, nil
return nil, nil //nolint:nilnil // nil,nil signals not-found; callers check nil
}
if err != nil {
@@ -138,13 +154,14 @@ func (r *BlobChunkRepository) GetByChunkHashTx(ctx context.Context, tx *sql.Tx,
return &bc, nil
}
// DeleteOrphaned deletes blob_chunks entries where either the blob or chunk no longer exists
// DeleteOrphaned deletes blob_chunks entries where either the blob or the
// chunk no longer exists.
func (r *BlobChunkRepository) DeleteOrphaned(ctx context.Context) error {
// Delete blob_chunks where the blob doesn't exist
query1 := `
DELETE FROM blob_chunks
DELETE FROM blob_chunks
WHERE NOT EXISTS (
SELECT 1 FROM blobs
SELECT 1 FROM blobs
WHERE blobs.id = blob_chunks.blob_id
)
`
@@ -156,9 +173,9 @@ func (r *BlobChunkRepository) DeleteOrphaned(ctx context.Context) error {
// Delete blob_chunks where the chunk doesn't exist
query2 := `
DELETE FROM blob_chunks
DELETE FROM blob_chunks
WHERE NOT EXISTS (
SELECT 1 FROM chunks
SELECT 1 FROM chunks
WHERE chunks.chunk_hash = blob_chunks.chunk_hash
)
`

196
internal/database/blob_chunks_test.go
View File

@@ -1,4 +1,4 @@
package database
package database_test
import (
"context"
@@ -6,59 +6,91 @@ import (
"testing"
"time"
"sneak.berlin/go/vaultik/internal/database"
"sneak.berlin/go/vaultik/internal/types"
)
func TestBlobChunkRepository(t *testing.T) {
db, cleanup := setupTestDB(t)
defer cleanup()
// Chunk hashes used across the blob_chunks tests.
const (
chunk1Hash = "chunk1"
chunk2Hash = "chunk2"
chunk3Hash = "chunk3"
)
// mustCreateChunks registers the given chunk hashes (1024 bytes each).
func mustCreateChunks(
t *testing.T,
repos *database.Repositories,
hashes ...types.ChunkHash,
) {
t.Helper()
ctx := context.Background()
repos := NewRepositories(db)
// Create blob first
blob := &Blob{
ID: types.NewBlobID(),
Hash: types.BlobHash("blob1-hash"),
CreatedTS: time.Now(),
}
err := repos.Blobs.Create(ctx, nil, blob)
if err != nil {
t.Fatalf("failed to create blob: %v", err)
}
// Create chunks
chunks := []types.ChunkHash{"chunk1", "chunk2", "chunk3"}
for _, chunkHash := range chunks {
chunk := &Chunk{
for _, chunkHash := range hashes {
chunk := &database.Chunk{
ChunkHash: chunkHash,
Size: 1024,
}
err = repos.Chunks.Create(ctx, nil, chunk)
err := repos.Chunks.Create(ctx, nil, chunk)
if err != nil {
t.Fatalf("failed to create chunk %s: %v", chunkHash, err)
}
}
}
// mustCreateBlob creates a blob row with the given hash.
func mustCreateBlob(
t *testing.T,
repos *database.Repositories,
hash types.BlobHash,
) *database.Blob {
t.Helper()
blob := &database.Blob{
ID: types.NewBlobID(),
Hash: hash,
CreatedTS: time.Now(),
}
err := repos.Blobs.Create(context.Background(), nil, blob)
if err != nil {
t.Fatalf("failed to create blob %s: %v", hash, err)
}
return blob
}
func TestBlobChunkRepository(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repos := database.NewRepositories(db)
blob := mustCreateBlob(t, repos, "blob1-hash")
mustCreateChunks(t, repos, chunk1Hash, chunk2Hash, chunk3Hash)
// Test Create
bc1 := &BlobChunk{
bc1 := &database.BlobChunk{
BlobID: blob.ID,
ChunkHash: types.ChunkHash("chunk1"),
ChunkHash: types.ChunkHash(chunk1Hash),
Offset: 0,
Length: 1024,
}
err = repos.BlobChunks.Create(ctx, nil, bc1)
err := repos.BlobChunks.Create(ctx, nil, bc1)
if err != nil {
t.Fatalf("failed to create blob chunk: %v", err)
}
// Add more chunks to the same blob
bc2 := &BlobChunk{
bc2 := &database.BlobChunk{
BlobID: blob.ID,
ChunkHash: types.ChunkHash("chunk2"),
ChunkHash: types.ChunkHash(chunk2Hash),
Offset: 1024,
Length: 2048,
}
@@ -68,9 +100,9 @@ func TestBlobChunkRepository(t *testing.T) {
t.Fatalf("failed to create second blob chunk: %v", err)
}
bc3 := &BlobChunk{
bc3 := &database.BlobChunk{
BlobID: blob.ID,
ChunkHash: types.ChunkHash("chunk3"),
ChunkHash: types.ChunkHash(chunk3Hash),
Offset: 3072,
Length: 512,
}
@@ -94,12 +126,49 @@ func TestBlobChunkRepository(t *testing.T) {
expectedOffsets := []int64{0, 1024, 3072}
for i, bc := range blobChunks {
if bc.Offset != expectedOffsets[i] {
t.Errorf("wrong chunk order: expected offset %d, got %d", expectedOffsets[i], bc.Offset)
t.Errorf("wrong chunk order: expected offset %d, got %d",
expectedOffsets[i], bc.Offset)
}
}
// Test duplicate insert (should fail due to primary key constraint)
err = repos.BlobChunks.Create(ctx, nil, bc1)
if err == nil {
t.Fatal("duplicate blob_chunk insert should fail due to primary key constraint")
}
if !strings.Contains(err.Error(), "UNIQUE") &&
!strings.Contains(err.Error(), "constraint") {
t.Fatalf("expected constraint error, got: %v", err)
}
}
func TestBlobChunkRepositoryGetByChunkHash(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repos := database.NewRepositories(db)
blob := mustCreateBlob(t, repos, "blob-gbch-hash")
mustCreateChunks(t, repos, chunk2Hash)
bc2 := &database.BlobChunk{
BlobID: blob.ID,
ChunkHash: types.ChunkHash(chunk2Hash),
Offset: 1024,
Length: 2048,
}
err := repos.BlobChunks.Create(ctx, nil, bc2)
if err != nil {
t.Fatalf("failed to create blob chunk: %v", err)
}
// Test GetByChunkHash
bc, err := repos.BlobChunks.GetByChunkHash(ctx, "chunk2")
bc, err := repos.BlobChunks.GetByChunkHash(ctx, chunk2Hash)
if err != nil {
t.Fatalf("failed to get blob chunk by chunk hash: %v", err)
}
@@ -116,16 +185,6 @@ func TestBlobChunkRepository(t *testing.T) {
t.Errorf("wrong offset: expected 1024, got %d", bc.Offset)
}
// Test duplicate insert (should fail due to primary key constraint)
err = repos.BlobChunks.Create(ctx, nil, bc1)
if err == nil {
t.Fatal("duplicate blob_chunk insert should fail due to primary key constraint")
}
if !strings.Contains(err.Error(), "UNIQUE") && !strings.Contains(err.Error(), "constraint") {
t.Fatalf("expected constraint error, got: %v", err)
}
// Test non-existent chunk
bc, err = repos.BlobChunks.GetByChunkHash(ctx, "nonexistent")
if err != nil {
@@ -138,55 +197,26 @@ func TestBlobChunkRepository(t *testing.T) {
}
func TestBlobChunkRepositoryMultipleBlobs(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repos := NewRepositories(db)
repos := database.NewRepositories(db)
// Create blobs
blob1 := &Blob{
ID: types.NewBlobID(),
Hash: types.BlobHash("blob1-hash"),
CreatedTS: time.Now(),
}
blob2 := &Blob{
ID: types.NewBlobID(),
Hash: types.BlobHash("blob2-hash"),
CreatedTS: time.Now(),
}
err := repos.Blobs.Create(ctx, nil, blob1)
if err != nil {
t.Fatalf("failed to create blob1: %v", err)
}
err = repos.Blobs.Create(ctx, nil, blob2)
if err != nil {
t.Fatalf("failed to create blob2: %v", err)
}
// Create chunks
chunkHashes := []types.ChunkHash{"chunk1", "chunk2", "chunk3"}
for _, chunkHash := range chunkHashes {
chunk := &Chunk{
ChunkHash: chunkHash,
Size: 1024,
}
err = repos.Chunks.Create(ctx, nil, chunk)
if err != nil {
t.Fatalf("failed to create chunk %s: %v", chunkHash, err)
}
}
blob1 := mustCreateBlob(t, repos, "blob1-hash")
blob2 := mustCreateBlob(t, repos, "blob2-hash")
mustCreateChunks(t, repos, chunk1Hash, chunk2Hash, chunk3Hash)
// Create chunks across multiple blobs
// Some chunks are shared between blobs (deduplication scenario)
blobChunks := []BlobChunk{
{BlobID: blob1.ID, ChunkHash: types.ChunkHash("chunk1"), Offset: 0, Length: 1024},
{BlobID: blob1.ID, ChunkHash: types.ChunkHash("chunk2"), Offset: 1024, Length: 1024},
{BlobID: blob2.ID, ChunkHash: types.ChunkHash("chunk2"), Offset: 0, Length: 1024}, // chunk2 is shared
{BlobID: blob2.ID, ChunkHash: types.ChunkHash("chunk3"), Offset: 1024, Length: 1024},
blobChunks := []database.BlobChunk{
{BlobID: blob1.ID, ChunkHash: chunk1Hash, Offset: 0, Length: 1024},
{BlobID: blob1.ID, ChunkHash: chunk2Hash, Offset: 1024, Length: 1024},
// chunk2 is shared between the blobs
{BlobID: blob2.ID, ChunkHash: chunk2Hash, Offset: 0, Length: 1024},
{BlobID: blob2.ID, ChunkHash: chunk3Hash, Offset: 1024, Length: 1024},
}
for _, bc := range blobChunks {
@@ -217,7 +247,7 @@ func TestBlobChunkRepositoryMultipleBlobs(t *testing.T) {
}
// Verify shared chunk
bc, err := repos.BlobChunks.GetByChunkHash(ctx, "chunk2")
bc, err := repos.BlobChunks.GetByChunkHash(ctx, chunk2Hash)
if err != nil {
t.Fatalf("failed to get shared chunk: %v", err)
}

159
internal/database/blobs.go
View File

@@ -10,17 +10,22 @@ import (
"sneak.berlin/go/vaultik/internal/log"
)
// BlobRepository provides access to the blobs table, which tracks the
// packed, encrypted storage units uploaded to the destination.
type BlobRepository struct {
db *DB
}
// NewBlobRepository creates a BlobRepository backed by db.
func NewBlobRepository(db *DB) *BlobRepository {
return &BlobRepository{db: db}
}
// Create inserts a blob row, using tx when non-nil.
func (r *BlobRepository) Create(ctx context.Context, tx *sql.Tx, blob *Blob) error {
query := `
INSERT INTO blobs (id, blob_hash, created_ts, finished_ts, uncompressed_size, compressed_size, uploaded_ts)
INSERT INTO blobs (id, blob_hash, created_ts, finished_ts,
uncompressed_size, compressed_size, uploaded_ts)
VALUES (?, ?, ?, ?, ?, ?, ?)
`
@@ -52,95 +57,15 @@ func (r *BlobRepository) Create(ctx context.Context, tx *sql.Tx, blob *Blob) err
return nil
}
// GetByHash returns the blob with the given content hash, or nil if no
// such blob exists.
func (r *BlobRepository) GetByHash(ctx context.Context, hash string) (*Blob, error) {
query := `
SELECT id, blob_hash, created_ts, finished_ts, uncompressed_size, compressed_size, uploaded_ts
FROM blobs
WHERE blob_hash = ?
`
var (
blob Blob
createdTSUnix int64
finishedTSUnix, uploadedTSUnix sql.NullInt64
)
err := r.db.conn.QueryRowContext(ctx, query, hash).Scan(
&blob.ID,
&blob.Hash,
&createdTSUnix,
&finishedTSUnix,
&blob.UncompressedSize,
&blob.CompressedSize,
&uploadedTSUnix,
)
if errors.Is(err, sql.ErrNoRows) {
return nil, nil
}
if err != nil {
return nil, fmt.Errorf("querying blob: %w", err)
}
blob.CreatedTS = time.Unix(createdTSUnix, 0).UTC()
if finishedTSUnix.Valid {
ts := time.Unix(finishedTSUnix.Int64, 0).UTC()
blob.FinishedTS = &ts
}
if uploadedTSUnix.Valid {
ts := time.Unix(uploadedTSUnix.Int64, 0).UTC()
blob.UploadedTS = &ts
}
return &blob, nil
return r.getOne(ctx, "blob_hash", hash)
}
// GetByID retrieves a blob by its ID
func (r *BlobRepository) GetByID(ctx context.Context, id string) (*Blob, error) {
query := `
SELECT id, blob_hash, created_ts, finished_ts, uncompressed_size, compressed_size, uploaded_ts
FROM blobs
WHERE id = ?
`
var (
blob Blob
createdTSUnix int64
finishedTSUnix, uploadedTSUnix sql.NullInt64
)
err := r.db.conn.QueryRowContext(ctx, query, id).Scan(
&blob.ID,
&blob.Hash,
&createdTSUnix,
&finishedTSUnix,
&blob.UncompressedSize,
&blob.CompressedSize,
&uploadedTSUnix,
)
if errors.Is(err, sql.ErrNoRows) {
return nil, nil
}
if err != nil {
return nil, fmt.Errorf("querying blob: %w", err)
}
blob.CreatedTS = time.Unix(createdTSUnix, 0).UTC()
if finishedTSUnix.Valid {
ts := time.Unix(finishedTSUnix.Int64, 0).UTC()
blob.FinishedTS = &ts
}
if uploadedTSUnix.Valid {
ts := time.Unix(uploadedTSUnix.Int64, 0).UTC()
blob.UploadedTS = &ts
}
return &blob, nil
return r.getOne(ctx, "id", id)
}
// GetAll returns every blob row keyed by blob ID. Useful at restore
@@ -148,7 +73,8 @@ func (r *BlobRepository) GetByID(ctx context.Context, id string) (*Blob, error)
// into blob hashes without doing one GetByID query per chunk.
func (r *BlobRepository) GetAll(ctx context.Context) (map[string]*Blob, error) {
query := `
SELECT id, blob_hash, created_ts, finished_ts, uncompressed_size, compressed_size, uploaded_ts
SELECT id, blob_hash, created_ts, finished_ts,
uncompressed_size, compressed_size, uploaded_ts
FROM blobs
`
@@ -198,7 +124,13 @@ func (r *BlobRepository) GetAll(ctx context.Context) (map[string]*Blob, error) {
}
// UpdateFinished updates a blob when it's finalized
func (r *BlobRepository) UpdateFinished(ctx context.Context, tx *sql.Tx, id string, hash string, uncompressedSize, compressedSize int64) error {
func (r *BlobRepository) UpdateFinished(
ctx context.Context,
tx *sql.Tx,
id string,
hash string,
uncompressedSize, compressedSize int64,
) error {
query := `
UPDATE blobs
SET blob_hash = ?, finished_ts = ?, uncompressed_size = ?, compressed_size = ?
@@ -222,7 +154,9 @@ func (r *BlobRepository) UpdateFinished(ctx context.Context, tx *sql.Tx, id stri
}
// UpdateUploaded marks a blob as uploaded
func (r *BlobRepository) UpdateUploaded(ctx context.Context, tx *sql.Tx, id string) error {
func (r *BlobRepository) UpdateUploaded(
ctx context.Context, tx *sql.Tx, id string,
) error {
query := `
UPDATE blobs
SET uploaded_ts = ?
@@ -267,3 +201,52 @@ func (r *BlobRepository) DeleteOrphaned(ctx context.Context) error {
return nil
}
// getOne fetches a single blob row matched on the given column, or
// (nil, nil) when no row matches.
func (r *BlobRepository) getOne(
ctx context.Context, column, value string,
) (*Blob, error) {
query := `
SELECT id, blob_hash, created_ts, finished_ts,
uncompressed_size, compressed_size, uploaded_ts
FROM blobs
WHERE ` + column + ` = ?`
var (
blob Blob
createdTSUnix int64
finishedTSUnix, uploadedTSUnix sql.NullInt64
)
err := r.db.conn.QueryRowContext(ctx, query, value).Scan(
&blob.ID,
&blob.Hash,
&createdTSUnix,
&finishedTSUnix,
&blob.UncompressedSize,
&blob.CompressedSize,
&uploadedTSUnix,
)
if errors.Is(err, sql.ErrNoRows) {
return nil, nil //nolint:nilnil // nil,nil signals not-found; callers check nil
}
if err != nil {
return nil, fmt.Errorf("querying blob: %w", err)
}
blob.CreatedTS = time.Unix(createdTSUnix, 0).UTC()
if finishedTSUnix.Valid {
ts := time.Unix(finishedTSUnix.Int64, 0).UTC()
blob.FinishedTS = &ts
}
if uploadedTSUnix.Valid {
ts := time.Unix(uploadedTSUnix.Int64, 0).UTC()
blob.UploadedTS = &ts
}
return &blob, nil
}

41
internal/database/blobs_test.go
View File

@@ -1,22 +1,25 @@
package database
package database_test
import (
"context"
"testing"
"time"
"sneak.berlin/go/vaultik/internal/database"
"sneak.berlin/go/vaultik/internal/types"
)
func TestBlobRepository(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repo := NewBlobRepository(db)
repo := database.NewBlobRepository(db)
// Test Create
blob := &Blob{
blob := &database.Blob{
ID: types.NewBlobID(),
Hash: types.BlobHash("blobhash123"),
CreatedTS: time.Now().Truncate(time.Second),
@@ -42,7 +45,8 @@ func TestBlobRepository(t *testing.T) {
}
if !retrieved.CreatedTS.Equal(blob.CreatedTS) {
t.Errorf("created timestamp mismatch: got %v, want %v", retrieved.CreatedTS, blob.CreatedTS)
t.Errorf("created timestamp mismatch: got %v, want %v",
retrieved.CreatedTS, blob.CreatedTS)
}
// Test GetByID
@@ -60,7 +64,7 @@ func TestBlobRepository(t *testing.T) {
}
// Test with second blob
blob2 := &Blob{
blob2 := &database.Blob{
ID: types.NewBlobID(),
Hash: types.BlobHash("blobhash456"),
CreatedTS: time.Now().Truncate(time.Second),
@@ -70,6 +74,27 @@ func TestBlobRepository(t *testing.T) {
if err != nil {
t.Fatalf("failed to create second blob: %v", err)
}
}
func TestBlobRepositoryUpdates(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repo := database.NewBlobRepository(db)
blob := &database.Blob{
ID: types.NewBlobID(),
Hash: types.BlobHash("blobhash123"),
CreatedTS: time.Now().Truncate(time.Second),
}
err := repo.Create(ctx, nil, blob)
if err != nil {
t.Fatalf("failed to create blob: %v", err)
}
// Test UpdateFinished
now := time.Now()
@@ -119,13 +144,15 @@ func TestBlobRepository(t *testing.T) {
}
func TestBlobRepositoryDuplicate(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repo := NewBlobRepository(db)
repo := database.NewBlobRepository(db)
blob := &Blob{
blob := &database.Blob{
ID: types.NewBlobID(),
Hash: types.BlobHash("duplicate_blob"),
CreatedTS: time.Now().Truncate(time.Second),

92
internal/database/cascade_debug_test.go
View File

@@ -1,3 +1,4 @@
//nolint:testpackage // inspects the unexported database connection
package database
import (
@@ -9,25 +10,13 @@ import (
"sneak.berlin/go/vaultik/internal/types"
)
// TestCascadeDeleteDebug tests cascade delete with debug output
func TestCascadeDeleteDebug(t *testing.T) {
db, cleanup := setupTestDB(t)
defer cleanup()
// createCascadeFixtures creates a file with three chunk mappings for the
// cascade-delete test.
func createCascadeFixtures(t *testing.T, repos *Repositories) *File {
t.Helper()
ctx := context.Background()
repos := NewRepositories(db)
// Check if foreign keys are enabled
var fkEnabled int
err := db.conn.QueryRow("PRAGMA foreign_keys").Scan(&fkEnabled)
if err != nil {
t.Fatal(err)
}
t.Logf("Foreign keys enabled: %d", fkEnabled)
// Create a file
file := &File{
Path: "/cascade-test.txt",
MTime: time.Now().Truncate(time.Second),
@@ -37,7 +26,7 @@ func TestCascadeDeleteDebug(t *testing.T) {
GID: 1000,
}
err = repos.Files.Create(ctx, nil, file)
err := repos.Files.Create(ctx, nil, file)
if err != nil {
t.Fatalf("failed to create file: %v", err)
}
@@ -67,9 +56,56 @@ func TestCascadeDeleteDebug(t *testing.T) {
t.Fatalf("failed to create file chunk: %v", err)
}
t.Logf("Created file chunk mapping: file_id=%s, idx=%d, chunk=%s", fc.FileID, fc.Idx, fc.ChunkHash)
t.Logf("Created file chunk mapping: file_id=%s, idx=%d, chunk=%s",
fc.FileID, fc.Idx, fc.ChunkHash)
}
return file
}
// logCascadeDebugInfo logs foreign-key state and the file_chunks table
// definition for cascade-delete debugging.
func logCascadeDebugInfo(ctx context.Context, t *testing.T, db *DB) {
t.Helper()
// Check if foreign keys are enabled
var fkEnabled int
err := db.conn.QueryRowContext(ctx, "PRAGMA foreign_keys").Scan(&fkEnabled)
if err != nil {
t.Fatal(err)
}
t.Logf("Foreign keys enabled: %d", fkEnabled)
// Check the foreign key constraint
var fkInfo string
err = db.conn.QueryRowContext(ctx, `
SELECT sql FROM sqlite_master
WHERE type='table' AND name='file_chunks'
`).Scan(&fkInfo)
if err != nil {
t.Fatal(err)
}
t.Logf("file_chunks table definition:\n%s", fkInfo)
}
// TestCascadeDeleteDebug tests cascade delete with debug output
func TestCascadeDeleteDebug(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repos := NewRepositories(db)
logCascadeDebugInfo(ctx, t, db)
file := createCascadeFixtures(t, repos)
// Verify file chunks exist
fileChunks, err := repos.FileChunks.GetByFileID(ctx, file.ID)
if err != nil {
@@ -78,19 +114,6 @@ func TestCascadeDeleteDebug(t *testing.T) {
t.Logf("File chunks before delete: %d", len(fileChunks))
// Check the foreign key constraint
var fkInfo string
err = db.conn.QueryRow(`
SELECT sql FROM sqlite_master
WHERE type='table' AND name='file_chunks'
`).Scan(&fkInfo)
if err != nil {
t.Fatal(err)
}
t.Logf("file_chunks table definition:\n%s", fkInfo)
// Delete the file
t.Log("Deleting file...")
@@ -122,7 +145,9 @@ func TestCascadeDeleteDebug(t *testing.T) {
// Manually check the database
var count int
err = db.conn.QueryRow("SELECT COUNT(*) FROM file_chunks WHERE file_id = ?", file.ID).Scan(&count)
err = db.conn.QueryRowContext(ctx,
"SELECT COUNT(*) FROM file_chunks WHERE file_id = ?", file.ID,
).Scan(&count)
if err != nil {
t.Fatal(err)
}
@@ -133,7 +158,8 @@ func TestCascadeDeleteDebug(t *testing.T) {
t.Errorf("expected 0 file chunks after cascade delete, got %d", len(fileChunks))
// List the remaining chunks
for _, fc := range fileChunks {
t.Logf("Remaining chunk: file_id=%s, idx=%d, chunk=%s", fc.FileID, fc.Idx, fc.ChunkHash)
t.Logf("Remaining chunk: file_id=%s, idx=%d, chunk=%s",
fc.FileID, fc.Idx, fc.ChunkHash)
}
}
}

250
internal/database/chunk_files.go
View File

@@ -9,15 +9,21 @@ import (
"sneak.berlin/go/vaultik/internal/types"
)
// ChunkFileRepository provides access to the chunk_files table, the
// reverse mapping from chunks to the files that contain them.
type ChunkFileRepository struct {
db *DB
}
// NewChunkFileRepository creates a ChunkFileRepository backed by db.
func NewChunkFileRepository(db *DB) *ChunkFileRepository {
return &ChunkFileRepository{db: db}
}
func (r *ChunkFileRepository) Create(ctx context.Context, tx *sql.Tx, cf *ChunkFile) error {
// Create inserts a chunk_files row (idempotently), using tx when non-nil.
func (r *ChunkFileRepository) Create(
ctx context.Context, tx *sql.Tx, cf *ChunkFile,
) error {
query := `
INSERT INTO chunk_files (chunk_hash, file_id, file_offset, length)
VALUES (?, ?, ?, ?)
@@ -26,9 +32,11 @@ func (r *ChunkFileRepository) Create(ctx context.Context, tx *sql.Tx, cf *ChunkF
var err error
if tx != nil {
_, err = tx.ExecContext(ctx, query, cf.ChunkHash.String(), cf.FileID.String(), cf.FileOffset, cf.Length)
_, err = tx.ExecContext(ctx, query,
cf.ChunkHash.String(), cf.FileID.String(), cf.FileOffset, cf.Length)
} else {
_, err = r.db.ExecWithLog(ctx, query, cf.ChunkHash.String(), cf.FileID.String(), cf.FileOffset, cf.Length)
_, err = r.db.ExecWithLog(ctx, query,
cf.ChunkHash.String(), cf.FileID.String(), cf.FileOffset, cf.Length)
}
if err != nil {
@@ -38,7 +46,10 @@ func (r *ChunkFileRepository) Create(ctx context.Context, tx *sql.Tx, cf *ChunkF
return nil
}
func (r *ChunkFileRepository) GetByChunkHash(ctx context.Context, chunkHash types.ChunkHash) ([]*ChunkFile, error) {
// GetByChunkHash returns all chunk_files rows for the given chunk hash.
func (r *ChunkFileRepository) GetByChunkHash(
ctx context.Context, chunkHash types.ChunkHash,
) ([]*ChunkFile, error) {
query := `
SELECT chunk_hash, file_id, file_offset, length
FROM chunk_files
@@ -54,7 +65,10 @@ func (r *ChunkFileRepository) GetByChunkHash(ctx context.Context, chunkHash type
return r.scanChunkFiles(rows)
}
func (r *ChunkFileRepository) GetByFilePath(ctx context.Context, filePath string) ([]*ChunkFile, error) {
// GetByFilePath returns all chunk_files rows for the file at the given path.
func (r *ChunkFileRepository) GetByFilePath(
ctx context.Context, filePath string,
) ([]*ChunkFile, error) {
query := `
SELECT cf.chunk_hash, cf.file_id, cf.file_offset, cf.length
FROM chunk_files cf
@@ -72,7 +86,9 @@ func (r *ChunkFileRepository) GetByFilePath(ctx context.Context, filePath string
}
// GetByFileID retrieves chunk files by file ID
func (r *ChunkFileRepository) GetByFileID(ctx context.Context, fileID types.FileID) ([]*ChunkFile, error) {
func (r *ChunkFileRepository) GetByFileID(
ctx context.Context, fileID types.FileID,
) ([]*ChunkFile, error) {
query := `
SELECT chunk_hash, file_id, file_offset, length
FROM chunk_files
@@ -88,7 +104,124 @@ func (r *ChunkFileRepository) GetByFileID(ctx context.Context, fileID types.File
return r.scanChunkFiles(rows)
}
// scanChunkFiles is a helper that scans chunk file rows
// DeleteByFileID deletes all chunk_files entries for a given file ID
func (r *ChunkFileRepository) DeleteByFileID(
ctx context.Context, tx *sql.Tx, fileID types.FileID,
) error {
query := `DELETE FROM chunk_files WHERE file_id = ?`
var err error
if tx != nil {
_, err = tx.ExecContext(ctx, query, fileID.String())
} else {
_, err = r.db.ExecWithLog(ctx, query, fileID.String())
}
if err != nil {
return fmt.Errorf("deleting chunk files: %w", err)
}
return nil
}
// DeleteByFileIDs deletes all chunk_files for multiple files in a single statement.
//
//nolint:dupl // symmetric implementation for a parallel association table
func (r *ChunkFileRepository) DeleteByFileIDs(
ctx context.Context, tx *sql.Tx, fileIDs []types.FileID,
) error {
if len(fileIDs) == 0 {
return nil
}
// Batch at 500 to stay within SQLite's variable limit
const batchSize = 500
for i := 0; i < len(fileIDs); i += batchSize {
end := min(i+batchSize, len(fileIDs))
batch := fileIDs[i:end]
//nolint:gosec // G202: concatenates constant SQL and "?" placeholders only
query := "DELETE FROM chunk_files WHERE file_id IN (?" +
repeatPlaceholder(len(batch)-1) + ")"
args := make([]any, len(batch))
for j, id := range batch {
args[j] = id.String()
}
var err error
if tx != nil {
_, err = tx.ExecContext(ctx, query, args...)
} else {
_, err = r.db.ExecWithLog(ctx, query, args...)
}
if err != nil {
return fmt.Errorf("batch deleting chunk_files: %w", err)
}
}
return nil
}
// CreateBatch inserts multiple chunk_files in a single statement for efficiency.
func (r *ChunkFileRepository) CreateBatch(
ctx context.Context, tx *sql.Tx, cfs []ChunkFile,
) error {
if len(cfs) == 0 {
return nil
}
// Each chunk_files row binds this many SQL variables.
const chunkFileCols = 4
// Batch at 200 rows to be safe with SQLite's variable limit.
const batchSize = 200
for i := 0; i < len(cfs); i += batchSize {
end := min(i+batchSize, len(cfs))
batch := cfs[i:end]
query := "INSERT INTO chunk_files (chunk_hash, file_id, file_offset, length) VALUES "
args := make([]any, 0, len(batch)*chunkFileCols)
var querySb183 strings.Builder
for j, cf := range batch {
if j > 0 {
querySb183.WriteString(", ")
}
querySb183.WriteString("(?, ?, ?, ?)")
args = append(args,
cf.ChunkHash.String(), cf.FileID.String(), cf.FileOffset, cf.Length)
}
query += querySb183.String() //nolint:gosec // G202: appends "?" placeholders only
query += " ON CONFLICT(chunk_hash, file_id) DO NOTHING"
var err error
if tx != nil {
_, err = tx.ExecContext(ctx, query, args...)
} else {
_, err = r.db.ExecWithLog(ctx, query, args...)
}
if err != nil {
return fmt.Errorf("batch inserting chunk_files: %w", err)
}
}
return nil
}
// scanChunkFiles is a helper that scans chunk file rows.
func (r *ChunkFileRepository) scanChunkFiles(rows *sql.Rows) ([]*ChunkFile, error) {
var chunkFiles []*ChunkFile
@@ -115,106 +248,3 @@ func (r *ChunkFileRepository) scanChunkFiles(rows *sql.Rows) ([]*ChunkFile, erro
return chunkFiles, rows.Err()
}
// DeleteByFileID deletes all chunk_files entries for a given file ID
func (r *ChunkFileRepository) DeleteByFileID(ctx context.Context, tx *sql.Tx, fileID types.FileID) error {
query := `DELETE FROM chunk_files WHERE file_id = ?`
var err error
if tx != nil {
_, err = tx.ExecContext(ctx, query, fileID.String())
} else {
_, err = r.db.ExecWithLog(ctx, query, fileID.String())
}
if err != nil {
return fmt.Errorf("deleting chunk files: %w", err)
}
return nil
}
// DeleteByFileIDs deletes all chunk_files for multiple files in a single statement.
func (r *ChunkFileRepository) DeleteByFileIDs(ctx context.Context, tx *sql.Tx, fileIDs []types.FileID) error {
if len(fileIDs) == 0 {
return nil
}
// Batch at 500 to stay within SQLite's variable limit
const batchSize = 500
for i := 0; i < len(fileIDs); i += batchSize {
end := min(i+batchSize, len(fileIDs))
batch := fileIDs[i:end]
query := "DELETE FROM chunk_files WHERE file_id IN (?" + repeatPlaceholder(len(batch)-1) + ")"
args := make([]any, len(batch))
for j, id := range batch {
args[j] = id.String()
}
var err error
if tx != nil {
_, err = tx.ExecContext(ctx, query, args...)
} else {
_, err = r.db.ExecWithLog(ctx, query, args...)
}
if err != nil {
return fmt.Errorf("batch deleting chunk_files: %w", err)
}
}
return nil
}
// CreateBatch inserts multiple chunk_files in a single statement for efficiency.
func (r *ChunkFileRepository) CreateBatch(ctx context.Context, tx *sql.Tx, cfs []ChunkFile) error {
if len(cfs) == 0 {
return nil
}
// Each ChunkFile has 4 values, so batch at 200 to be safe with SQLite's variable limit
const batchSize = 200
for i := 0; i < len(cfs); i += batchSize {
end := min(i+batchSize, len(cfs))
batch := cfs[i:end]
query := "INSERT INTO chunk_files (chunk_hash, file_id, file_offset, length) VALUES "
args := make([]any, 0, len(batch)*4)
var querySb183 strings.Builder
for j, cf := range batch {
if j > 0 {
querySb183.WriteString(", ")
}
querySb183.WriteString("(?, ?, ?, ?)")
args = append(args, cf.ChunkHash.String(), cf.FileID.String(), cf.FileOffset, cf.Length)
}
query += querySb183.String()
query += " ON CONFLICT(chunk_hash, file_id) DO NOTHING"
var err error
if tx != nil {
_, err = tx.ExecContext(ctx, query, args...)
} else {
_, err = r.db.ExecWithLog(ctx, query, args...)
}
if err != nil {
return fmt.Errorf("batch inserting chunk_files: %w", err)
}
}
return nil
}

222
internal/database/chunk_files_test.go
View File

@@ -1,81 +1,105 @@
package database
package database_test
import (
"context"
"testing"
"time"
"sneak.berlin/go/vaultik/internal/database"
"sneak.berlin/go/vaultik/internal/types"
)
const chunk4Hash = "chunk4"
// verifyChunkFilePair asserts that the chunk-file rows cover both test
// files at their expected offsets.
func verifyChunkFilePair(
t *testing.T, chunkFiles []*database.ChunkFile,
file1ID, file2ID types.FileID,
) {
t.Helper()
foundFile1 := false
foundFile2 := false
for _, cf := range chunkFiles {
if cf.FileID == file1ID && cf.FileOffset == 0 {
foundFile1 = true
}
if cf.FileID == file2ID && cf.FileOffset == 2048 {
foundFile2 = true
}
}
if !foundFile1 || !foundFile2 {
t.Error("not all expected files found")
}
}
// createChunkFileTestFiles creates the two files used by the chunk-file
// repository tests.
func createChunkFileTestFiles(
t *testing.T, fileRepo *database.FileRepository,
) (*database.File, *database.File) {
t.Helper()
testTime := time.Now().Truncate(time.Second)
file1 := &database.File{
Path: testFilePath1,
MTime: testTime,
Size: 1024,
Mode: 0644,
UID: 1000,
GID: 1000,
LinkTarget: "",
}
file2 := &database.File{
Path: testFilePath2,
MTime: testTime,
Size: 1024,
Mode: 0644,
UID: 1000,
GID: 1000,
LinkTarget: "",
}
mustCreateFile(t, fileRepo, file1)
mustCreateFile(t, fileRepo, file2)
return file1, file2
}
func TestChunkFileRepository(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repo := NewChunkFileRepository(db)
fileRepo := NewFileRepository(db)
chunksRepo := NewChunkRepository(db)
repo := database.NewChunkFileRepository(db)
fileRepo := database.NewFileRepository(db)
repos := database.NewRepositories(db)
// Create test files first
testTime := time.Now().Truncate(time.Second)
file1 := &File{
Path: "/file1.txt",
MTime: testTime,
Size: 1024,
Mode: 0644,
UID: 1000,
GID: 1000,
LinkTarget: "",
}
err := fileRepo.Create(ctx, nil, file1)
if err != nil {
t.Fatalf("failed to create file1: %v", err)
}
file2 := &File{
Path: "/file2.txt",
MTime: testTime,
Size: 1024,
Mode: 0644,
UID: 1000,
GID: 1000,
LinkTarget: "",
}
err = fileRepo.Create(ctx, nil, file2)
if err != nil {
t.Fatalf("failed to create file2: %v", err)
}
// Create chunk first
chunk := &Chunk{
ChunkHash: types.ChunkHash("chunk1"),
Size: 1024,
}
err = chunksRepo.Create(ctx, nil, chunk)
if err != nil {
t.Fatalf("failed to create chunk: %v", err)
}
file1, file2 := createChunkFileTestFiles(t, fileRepo)
mustCreateChunks(t, repos, chunk1Hash)
// Test Create
cf1 := &ChunkFile{
ChunkHash: types.ChunkHash("chunk1"),
cf1 := &database.ChunkFile{
ChunkHash: types.ChunkHash(chunk1Hash),
FileID: file1.ID,
FileOffset: 0,
Length: 1024,
}
err = repo.Create(ctx, nil, cf1)
err := repo.Create(ctx, nil, cf1)
if err != nil {
t.Fatalf("failed to create chunk file: %v", err)
}
// Add same chunk in different file (deduplication scenario)
cf2 := &ChunkFile{
ChunkHash: types.ChunkHash("chunk1"),
cf2 := &database.ChunkFile{
ChunkHash: types.ChunkHash(chunk1Hash),
FileID: file2.ID,
FileOffset: 2048,
Length: 1024,
@@ -87,7 +111,7 @@ func TestChunkFileRepository(t *testing.T) {
}
// Test GetByChunkHash
chunkFiles, err := repo.GetByChunkHash(ctx, "chunk1")
chunkFiles, err := repo.GetByChunkHash(ctx, chunk1Hash)
if err != nil {
t.Fatalf("failed to get chunk files: %v", err)
}
@@ -97,22 +121,7 @@ func TestChunkFileRepository(t *testing.T) {
}
// Verify both files are returned
foundFile1 := false
foundFile2 := false
for _, cf := range chunkFiles {
if cf.FileID == file1.ID && cf.FileOffset == 0 {
foundFile1 = true
}
if cf.FileID == file2.ID && cf.FileOffset == 2048 {
foundFile2 = true
}
}
if !foundFile1 || !foundFile2 {
t.Error("not all expected files found")
}
verifyChunkFilePair(t, chunkFiles, file1.ID, file2.ID)
// Test GetByFileID
chunkFiles, err = repo.GetByFileID(ctx, file1.ID)
@@ -124,7 +133,7 @@ func TestChunkFileRepository(t *testing.T) {
t.Errorf("expected 1 chunk for file, got %d", len(chunkFiles))
}
if chunkFiles[0].ChunkHash != types.ChunkHash("chunk1") {
if chunkFiles[0].ChunkHash != types.ChunkHash(chunk1Hash) {
t.Errorf("wrong chunk hash: expected chunk1, got %s", chunkFiles[0].ChunkHash)
}
@@ -136,66 +145,53 @@ func TestChunkFileRepository(t *testing.T) {
}
func TestChunkFileRepositoryComplexDeduplication(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repo := NewChunkFileRepository(db)
fileRepo := NewFileRepository(db)
chunksRepo := NewChunkRepository(db)
repo := database.NewChunkFileRepository(db)
fileRepo := database.NewFileRepository(db)
repos := database.NewRepositories(db)
// Create test files
testTime := time.Now().Truncate(time.Second)
file1 := &File{Path: "/file1.txt", MTime: testTime, Size: 3072, Mode: 0644, UID: 1000, GID: 1000}
file2 := &File{Path: "/file2.txt", MTime: testTime, Size: 3072, Mode: 0644, UID: 1000, GID: 1000}
file3 := &File{Path: "/file3.txt", MTime: testTime, Size: 2048, Mode: 0644, UID: 1000, GID: 1000}
err := fileRepo.Create(ctx, nil, file1)
if err != nil {
t.Fatalf("failed to create file1: %v", err)
file1 := &database.File{
Path: testFilePath1, MTime: testTime, Size: 3072,
Mode: 0644, UID: 1000, GID: 1000,
}
file2 := &database.File{
Path: testFilePath2, MTime: testTime, Size: 3072,
Mode: 0644, UID: 1000, GID: 1000,
}
file3 := &database.File{
Path: "/file3.txt", MTime: testTime, Size: 2048,
Mode: 0644, UID: 1000, GID: 1000,
}
err = fileRepo.Create(ctx, nil, file2)
if err != nil {
t.Fatalf("failed to create file2: %v", err)
}
err = fileRepo.Create(ctx, nil, file3)
if err != nil {
t.Fatalf("failed to create file3: %v", err)
}
// Create chunks first
chunks := []types.ChunkHash{"chunk1", "chunk2", "chunk3", "chunk4"}
for _, chunkHash := range chunks {
chunk := &Chunk{
ChunkHash: chunkHash,
Size: 1024,
}
err := chunksRepo.Create(ctx, nil, chunk)
if err != nil {
t.Fatalf("failed to create chunk %s: %v", chunkHash, err)
}
}
mustCreateFile(t, fileRepo, file1)
mustCreateFile(t, fileRepo, file2)
mustCreateFile(t, fileRepo, file3)
mustCreateChunks(t, repos, chunk1Hash, chunk2Hash, chunk3Hash, chunk4Hash)
// Simulate a scenario where multiple files share chunks
// File1: chunk1, chunk2, chunk3
// File2: chunk2, chunk3, chunk4
// File3: chunk1, chunk4
chunkFiles := []ChunkFile{
chunkFiles := []database.ChunkFile{
// File1
{ChunkHash: types.ChunkHash("chunk1"), FileID: file1.ID, FileOffset: 0, Length: 1024},
{ChunkHash: types.ChunkHash("chunk2"), FileID: file1.ID, FileOffset: 1024, Length: 1024},
{ChunkHash: types.ChunkHash("chunk3"), FileID: file1.ID, FileOffset: 2048, Length: 1024},
{ChunkHash: chunk1Hash, FileID: file1.ID, FileOffset: 0, Length: 1024},
{ChunkHash: chunk2Hash, FileID: file1.ID, FileOffset: 1024, Length: 1024},
{ChunkHash: chunk3Hash, FileID: file1.ID, FileOffset: 2048, Length: 1024},
// File2
{ChunkHash: types.ChunkHash("chunk2"), FileID: file2.ID, FileOffset: 0, Length: 1024},
{ChunkHash: types.ChunkHash("chunk3"), FileID: file2.ID, FileOffset: 1024, Length: 1024},
{ChunkHash: types.ChunkHash("chunk4"), FileID: file2.ID, FileOffset: 2048, Length: 1024},
{ChunkHash: chunk2Hash, FileID: file2.ID, FileOffset: 0, Length: 1024},
{ChunkHash: chunk3Hash, FileID: file2.ID, FileOffset: 1024, Length: 1024},
{ChunkHash: chunk4Hash, FileID: file2.ID, FileOffset: 2048, Length: 1024},
// File3
{ChunkHash: types.ChunkHash("chunk1"), FileID: file3.ID, FileOffset: 0, Length: 1024},
{ChunkHash: types.ChunkHash("chunk4"), FileID: file3.ID, FileOffset: 1024, Length: 1024},
{ChunkHash: chunk1Hash, FileID: file3.ID, FileOffset: 0, Length: 1024},
{ChunkHash: chunk4Hash, FileID: file3.ID, FileOffset: 1024, Length: 1024},
}
for _, cf := range chunkFiles {
@@ -206,7 +202,7 @@ func TestChunkFileRepositoryComplexDeduplication(t *testing.T) {
}
// Test chunk1 (used by file1 and file3)
files, err := repo.GetByChunkHash(ctx, "chunk1")
files, err := repo.GetByChunkHash(ctx, chunk1Hash)
if err != nil {
t.Fatalf("failed to get files for chunk1: %v", err)
}
@@ -216,7 +212,7 @@ func TestChunkFileRepositoryComplexDeduplication(t *testing.T) {
}
// Test chunk2 (used by file1 and file2)
files, err = repo.GetByChunkHash(ctx, "chunk2")
files, err = repo.GetByChunkHash(ctx, chunk2Hash)
if err != nil {
t.Fatalf("failed to get files for chunk2: %v", err)
}

22
internal/database/chunks.go
View File

@@ -10,14 +10,18 @@ import (
"sneak.berlin/go/vaultik/internal/log"
)
// ChunkRepository provides access to the chunks table, which tracks
// content-defined chunks by hash and size.
type ChunkRepository struct {
db *DB
}
// NewChunkRepository creates a ChunkRepository backed by db.
func NewChunkRepository(db *DB) *ChunkRepository {
return &ChunkRepository{db: db}
}
// Create inserts a chunk row (idempotently), using tx when non-nil.
func (r *ChunkRepository) Create(ctx context.Context, tx *sql.Tx, chunk *Chunk) error {
query := `
INSERT INTO chunks (chunk_hash, size)
@@ -39,6 +43,8 @@ func (r *ChunkRepository) Create(ctx context.Context, tx *sql.Tx, chunk *Chunk)
return nil
}
// GetByHash returns the chunk with the given hash, or nil if it is not
// known to the index.
func (r *ChunkRepository) GetByHash(ctx context.Context, hash string) (*Chunk, error) {
query := `
SELECT chunk_hash, size
@@ -54,7 +60,7 @@ func (r *ChunkRepository) GetByHash(ctx context.Context, hash string) (*Chunk, e
)
if errors.Is(err, sql.ErrNoRows) {
return nil, nil
return nil, nil //nolint:nilnil // nil,nil signals not-found; callers check nil
}
if err != nil {
@@ -64,7 +70,11 @@ func (r *ChunkRepository) GetByHash(ctx context.Context, hash string) (*Chunk, e
return &chunk, nil
}
func (r *ChunkRepository) GetByHashes(ctx context.Context, hashes []string) ([]*Chunk, error) {
// GetByHashes returns the chunks whose hashes appear in hashes, ordered by
// chunk hash. Unknown hashes are silently omitted from the result.
func (r *ChunkRepository) GetByHashes(
ctx context.Context, hashes []string,
) ([]*Chunk, error) {
if len(hashes) == 0 {
return nil, nil
}
@@ -88,7 +98,7 @@ func (r *ChunkRepository) GetByHashes(ctx context.Context, hashes []string) ([]*
args[i] = hash
}
query += querySb75.String()
query += querySb75.String() //nolint:gosec // G202: appends "?" placeholders only
query += ") ORDER BY chunk_hash"
@@ -117,7 +127,11 @@ func (r *ChunkRepository) GetByHashes(ctx context.Context, hashes []string) ([]*
return chunks, rows.Err()
}
func (r *ChunkRepository) ListUnpacked(ctx context.Context, limit int) ([]*Chunk, error) {
// ListUnpacked returns up to limit chunks that are not yet stored in any
// blob, ordered by chunk hash.
func (r *ChunkRepository) ListUnpacked(
ctx context.Context, limit int,
) ([]*Chunk, error) {
query := `
SELECT c.chunk_hash, c.size
FROM chunks c

1
internal/database/chunks_ext.go
View File

@@ -5,6 +5,7 @@ import (
"fmt"
)
// List returns every chunk in the index, ordered by chunk hash.
func (r *ChunkRepository) List(ctx context.Context) ([]*Chunk, error) {
query := `
SELECT chunk_hash, size

19
internal/database/chunks_test.go
View File

@@ -1,21 +1,24 @@
package database
package database_test
import (
"context"
"testing"
"sneak.berlin/go/vaultik/internal/database"
"sneak.berlin/go/vaultik/internal/types"
)
func TestChunkRepository(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repo := NewChunkRepository(db)
repo := database.NewChunkRepository(db)
// Test Create
chunk := &Chunk{
chunk := &database.Chunk{
ChunkHash: types.ChunkHash("chunkhash123"),
Size: 4096,
}
@@ -50,7 +53,7 @@ func TestChunkRepository(t *testing.T) {
}
// Test GetByHashes
chunk2 := &Chunk{
chunk2 := &database.Chunk{
ChunkHash: types.ChunkHash("chunkhash456"),
Size: 8192,
}
@@ -60,7 +63,9 @@ func TestChunkRepository(t *testing.T) {
t.Fatalf("failed to create second chunk: %v", err)
}
chunks, err := repo.GetByHashes(ctx, []string{chunk.ChunkHash.String(), chunk2.ChunkHash.String()})
chunks, err := repo.GetByHashes(ctx, []string{
chunk.ChunkHash.String(), chunk2.ChunkHash.String(),
})
if err != nil {
t.Fatalf("failed to get chunks by hashes: %v", err)
}
@@ -81,11 +86,13 @@ func TestChunkRepository(t *testing.T) {
}
func TestChunkRepositoryNotFound(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repo := NewChunkRepository(db)
repo := database.NewChunkRepository(db)
// Test GetByHash with non-existent hash
chunk, err := repo.GetByHash(ctx, "nonexistent")

156
internal/database/database.go
View File

@@ -15,6 +15,7 @@ import (
"context"
"database/sql"
"embed"
"errors"
"fmt"
"os"
"path/filepath"
@@ -22,10 +23,15 @@ import (
"strconv"
"strings"
// Register the pure-Go sqlite driver.
_ "modernc.org/sqlite"
"sneak.berlin/go/vaultik/internal/log"
)
// errInvalidMigrationFilename is returned when an embedded migration file
// does not follow the "<version>[_<description>].sql" naming pattern.
var errInvalidMigrationFilename = errors.New("invalid migration filename")
//go:embed schema/*.sql
var schemaFS embed.FS
@@ -51,7 +57,7 @@ type DB struct {
func ParseMigrationVersion(filename string) (int, error) {
name := strings.TrimSuffix(filename, filepath.Ext(filename))
if name == "" {
return 0, fmt.Errorf("invalid migration filename %q: empty name", filename)
return 0, fmt.Errorf("%w %q: empty name", errInvalidMigrationFilename, filename)
}
// Split on underscore to separate version from description.
@@ -62,15 +68,17 @@ func ParseMigrationVersion(filename string) (int, error) {
}
if versionStr == "" {
return 0, fmt.Errorf("invalid migration filename %q: empty version prefix", filename)
return 0, fmt.Errorf(
"%w %q: empty version prefix", errInvalidMigrationFilename, filename,
)
}
// Validate the version is purely numeric.
for _, ch := range versionStr {
if ch < '0' || ch > '9' {
return 0, fmt.Errorf(
"invalid migration filename %q: version %q contains non-numeric character %q",
filename, versionStr, string(ch),
"%w %q: version %q contains non-numeric character %q",
errInvalidMigrationFilename, filename, versionStr, string(ch),
)
}
}
@@ -101,68 +109,87 @@ func New(ctx context.Context, path string) (*DB, error) {
conn, err := sql.Open(
"sqlite",
path+"?_journal_mode=WAL&_synchronous=NORMAL&_busy_timeout=10000&_locking_mode=NORMAL&_foreign_keys=ON",
path+"?_journal_mode=WAL&_synchronous=NORMAL&_busy_timeout=10000"+
"&_locking_mode=NORMAL&_foreign_keys=ON",
)
if err == nil {
// Set connection pool settings
// SQLite can handle multiple readers but only one writer at a time.
// Setting MaxOpenConns to 1 ensures all writes are serialized through
// a single connection, preventing SQLITE_BUSY errors.
conn.SetMaxOpenConns(1)
conn.SetMaxIdleConns(1)
configureConnPool(conn)
err := conn.PingContext(ctx)
err = conn.PingContext(ctx)
if err == nil {
// Success on first try
log.Debug("Database opened successfully with WAL mode", "path", path)
// Enable foreign keys explicitly
_, err = conn.ExecContext(ctx, "PRAGMA foreign_keys = ON")
if err != nil {
log.Warn("Failed to enable foreign keys", "error", err)
}
db := &DB{conn: conn, path: path}
err := applyMigrations(ctx, conn)
if err != nil {
_ = conn.Close()
return nil, fmt.Errorf("applying migrations: %w", err)
}
return db, nil
return finishOpen(ctx, conn, path)
}
log.Debug("Failed to ping database, closing connection", "path", path, "error", err)
log.Debug(
"Failed to ping database, closing connection",
"path", path, "error", err,
)
_ = conn.Close()
}
// If first attempt failed, try with TRUNCATE mode to clear any locks
return openWithRecovery(ctx, path)
}
// configureConnPool serializes all database access through one connection.
// SQLite can handle multiple readers but only one writer at a time; setting
// MaxOpenConns to 1 ensures all writes go through a single connection,
// preventing SQLITE_BUSY errors.
func configureConnPool(conn *sql.DB) {
conn.SetMaxOpenConns(1)
conn.SetMaxIdleConns(1)
}
// finishOpen enables foreign keys, wraps the connection, and applies any
// pending migrations. On migration failure the connection is closed.
func finishOpen(ctx context.Context, conn *sql.DB, path string) (*DB, error) {
// Enable foreign keys explicitly
_, err := conn.ExecContext(ctx, "PRAGMA foreign_keys = ON")
if err != nil {
log.Warn("Failed to enable foreign keys", "path", path, "error", err)
}
db := &DB{conn: conn, path: path}
err = applyMigrations(ctx, conn)
if err != nil {
_ = conn.Close()
return nil, fmt.Errorf("applying migrations: %w", err)
}
return db, nil
}
// openWithRecovery retries opening the database in TRUNCATE journal mode to
// clear stale locks, then switches back to WAL mode.
func openWithRecovery(ctx context.Context, path string) (*DB, error) {
log.Info(
"Database appears locked, attempting recovery with TRUNCATE mode",
"path", path,
)
conn, err = sql.Open(
conn, err := sql.Open(
"sqlite",
path+"?_journal_mode=TRUNCATE&_synchronous=NORMAL&_busy_timeout=10000&_foreign_keys=ON",
path+"?_journal_mode=TRUNCATE&_synchronous=NORMAL&_busy_timeout=10000"+
"&_foreign_keys=ON",
)
if err != nil {
return nil, fmt.Errorf("opening database in recovery mode: %w", err)
}
// Set connection pool settings
// SQLite can handle multiple readers but only one writer at a time.
// Setting MaxOpenConns to 1 ensures all writes are serialized through
// a single connection, preventing SQLITE_BUSY errors.
conn.SetMaxOpenConns(1)
conn.SetMaxIdleConns(1)
configureConnPool(conn)
err = conn.PingContext(ctx)
if err != nil {
log.Debug("Failed to ping database in recovery mode, closing", "path", path, "error", err)
log.Debug(
"Failed to ping database in recovery mode, closing",
"path", path, "error", err,
)
_ = conn.Close()
@@ -182,19 +209,9 @@ func New(ctx context.Context, path string) (*DB, error) {
log.Warn("Failed to switch back to WAL mode", "path", path, "error", err)
}
// Ensure foreign keys are enabled
_, err = conn.ExecContext(ctx, "PRAGMA foreign_keys=ON")
db, err := finishOpen(ctx, conn, path)
if err != nil {
log.Warn("Failed to enable foreign keys", "path", path, "error", err)
}
db := &DB{conn: conn, path: path}
err = applyMigrations(ctx, conn)
if err != nil {
_ = conn.Close()
return nil, fmt.Errorf("applying migrations: %w", err)
return nil, err
}
log.Debug("Database connection established successfully", "path", path)
@@ -202,6 +219,13 @@ func New(ctx context.Context, path string) (*DB, error) {
return db, nil
}
// NewTestDB creates an in-memory SQLite database for testing purposes.
// The database is automatically initialized with the schema and is ready
// for use. Each call creates a new independent database instance.
func NewTestDB() (*DB, error) {
return New(context.Background(), ":memory:")
}
// Close closes the database connection.
// It ensures all pending operations are completed before closing.
// Returns an error if the database connection cannot be closed properly.
@@ -259,10 +283,11 @@ func (db *DB) ExecWithLog(
return db.conn.ExecContext(ctx, query, args...)
}
// QueryRowWithLog executes a query that returns at most one row with SQL logging.
// This is useful for queries that modify data and return values (e.g., INSERT ... RETURNING).
// SQLite handles its own locking internally.
// The query and args parameters follow the same format as sql.DB.QueryRowContext.
// QueryRowWithLog executes a query that returns at most one row with SQL
// logging. This is useful for queries that modify data and return values
// (e.g., INSERT ... RETURNING). SQLite handles its own locking internally.
// The query and args parameters follow the same format as
// sql.DB.QueryRowContext.
func (db *DB) QueryRowWithLog(
ctx context.Context,
query string,
@@ -390,15 +415,8 @@ func applyMigrations(ctx context.Context, db *sql.DB) error {
return nil
}
// NewTestDB creates an in-memory SQLite database for testing purposes.
// The database is automatically initialized with the schema and is ready for use.
// Each call creates a new independent database instance.
func NewTestDB() (*DB, error) {
return New(context.Background(), ":memory:")
}
// repeatPlaceholder generates a string of ", ?" repeated n times for IN clause construction.
// For example, repeatPlaceholder(2) returns ", ?, ?".
// repeatPlaceholder generates a string of ", ?" repeated n times for IN
// clause construction. For example, repeatPlaceholder(2) returns ", ?, ?".
func repeatPlaceholder(n int) string {
if n <= 0 {
return ""
@@ -408,12 +426,14 @@ func repeatPlaceholder(n int) string {
}
// LogSQL logs SQL queries and their arguments when debug mode is enabled.
// Debug mode is activated by setting the GODEBUG environment variable to include "vaultik".
// This is useful for troubleshooting database operations and understanding query patterns.
// Debug mode is activated by setting the GODEBUG environment variable to
// include "vaultik". This is useful for troubleshooting database operations
// and understanding query patterns.
//
// The operation parameter describes the type of SQL operation (e.g., "Execute", "Query").
// The query parameter is the SQL statement being executed.
// The args parameter contains the query arguments that will be interpolated.
// The operation parameter describes the type of SQL operation (e.g.,
// "Execute", "Query"). The query parameter is the SQL statement being
// executed. The args parameter contains the query arguments that will be
// interpolated.
func LogSQL(operation, query string, args ...any) {
if strings.Contains(os.Getenv("GODEBUG"), "vaultik") {
log.Debug(

55
internal/database/database_test.go
View File

@@ -1,3 +1,4 @@
//nolint:testpackage // exercises unexported migration internals
package database
import (
@@ -9,6 +10,8 @@ import (
)
func TestDatabase(t *testing.T) {
t.Parallel()
ctx := context.Background()
dbPath := filepath.Join(t.TempDir(), "test.db")
@@ -39,7 +42,9 @@ func TestDatabase(t *testing.T) {
for _, table := range tables {
var name string
err := db.conn.QueryRow("SELECT name FROM sqlite_master WHERE type='table' AND name=?", table).Scan(&name)
err := db.conn.QueryRowContext(ctx,
"SELECT name FROM sqlite_master WHERE type='table' AND name=?", table,
).Scan(&name)
if err != nil {
t.Errorf("table %s does not exist: %v", table, err)
}
@@ -47,6 +52,8 @@ func TestDatabase(t *testing.T) {
}
func TestDatabaseInvalidPath(t *testing.T) {
t.Parallel()
ctx := context.Background()
// Test with invalid path
@@ -57,6 +64,8 @@ func TestDatabaseInvalidPath(t *testing.T) {
}
func TestDatabaseConcurrentAccess(t *testing.T) {
t.Parallel()
ctx := context.Background()
dbPath := filepath.Join(t.TempDir(), "test.db")
@@ -81,7 +90,8 @@ func TestDatabaseConcurrentAccess(t *testing.T) {
for i := range 10 {
go func(i int) {
_, err := db.ExecWithLog(ctx, "INSERT INTO chunks (chunk_hash, size) VALUES (?, ?)",
_, err := db.ExecWithLog(ctx,
"INSERT INTO chunks (chunk_hash, size) VALUES (?, ?)",
fmt.Sprintf("hash%d", i), i*1024)
results <- result{index: i, err: err}
}(i)
@@ -109,6 +119,8 @@ func TestDatabaseConcurrentAccess(t *testing.T) {
}
func TestParseMigrationVersion(t *testing.T) {
t.Parallel()
tests := []struct {
name string
filename string
@@ -118,8 +130,14 @@ func TestParseMigrationVersion(t *testing.T) {
{name: "valid 000.sql", filename: "000.sql", wantVer: 0, wantError: false},
{name: "valid 001.sql", filename: "001.sql", wantVer: 1, wantError: false},
{name: "valid 099.sql", filename: "099.sql", wantVer: 99, wantError: false},
{name: "valid with description", filename: "001_initial_schema.sql", wantVer: 1, wantError: false},
{name: "valid large version", filename: "123_big_migration.sql", wantVer: 123, wantError: false},
{
name: "valid with description", filename: "001_initial_schema.sql",
wantVer: 1, wantError: false,
},
{
name: "valid large version", filename: "123_big_migration.sql",
wantVer: 123, wantError: false,
},
{name: "invalid alpha version", filename: "abc.sql", wantVer: 0, wantError: true},
{name: "invalid mixed chars", filename: "12a.sql", wantVer: 0, wantError: true},
{name: "invalid no extension", filename: "schema.sql", wantVer: 0, wantError: true},
@@ -128,29 +146,36 @@ func TestParseMigrationVersion(t *testing.T) {
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
got, err := ParseMigrationVersion(tc.filename)
if tc.wantError {
if err == nil {
t.Errorf("ParseMigrationVersion(%q) = %d, nil; want error", tc.filename, got)
t.Errorf("ParseMigrationVersion(%q) = %d, nil; want error",
tc.filename, got)
}
return
}
if err != nil {
t.Errorf("ParseMigrationVersion(%q) unexpected error: %v", tc.filename, err)
t.Errorf("ParseMigrationVersion(%q) unexpected error: %v",
tc.filename, err)
return
}
if got != tc.wantVer {
t.Errorf("ParseMigrationVersion(%q) = %d; want %d", tc.filename, got, tc.wantVer)
t.Errorf("ParseMigrationVersion(%q) = %d; want %d",
tc.filename, got, tc.wantVer)
}
})
}
}
func TestApplyMigrations_Idempotent(t *testing.T) {
t.Parallel()
ctx := context.Background()
conn, err := sql.Open("sqlite", ":memory:?_foreign_keys=ON")
@@ -176,7 +201,9 @@ func TestApplyMigrations_Idempotent(t *testing.T) {
// Count rows in schema_migrations after first run.
var countBefore int
err = conn.QueryRowContext(ctx, "SELECT COUNT(*) FROM schema_migrations").Scan(&countBefore)
err = conn.QueryRowContext(ctx,
"SELECT COUNT(*) FROM schema_migrations",
).Scan(&countBefore)
if err != nil {
t.Fatalf("failed to count schema_migrations after first run: %v", err)
}
@@ -190,17 +217,22 @@ func TestApplyMigrations_Idempotent(t *testing.T) {
// Count rows in schema_migrations after second run — must be unchanged.
var countAfter int
err = conn.QueryRowContext(ctx, "SELECT COUNT(*) FROM schema_migrations").Scan(&countAfter)
err = conn.QueryRowContext(ctx,
"SELECT COUNT(*) FROM schema_migrations",
).Scan(&countAfter)
if err != nil {
t.Fatalf("failed to count schema_migrations after second run: %v", err)
}
if countBefore != countAfter {
t.Errorf("schema_migrations row count changed: before=%d, after=%d", countBefore, countAfter)
t.Errorf("schema_migrations row count changed: before=%d, after=%d",
countBefore, countAfter)
}
}
func TestBootstrapMigrationsTable_FreshDatabase(t *testing.T) {
t.Parallel()
ctx := context.Background()
conn, err := sql.Open("sqlite", ":memory:?_foreign_keys=ON")
@@ -248,7 +280,8 @@ func TestBootstrapMigrationsTable_FreshDatabase(t *testing.T) {
}
if tableAfter != 1 {
t.Fatalf("schema_migrations table should exist after bootstrap, got count=%d", tableAfter)
t.Fatalf("schema_migrations table should exist after bootstrap, got count=%d",
tableAfter)
}
// Verify version 0 row exists.

6
internal/database/errors.go
View File

@@ -6,8 +6,8 @@ import (
"os"
)
// Fatal prints an error message to stderr and exits with status 1
func Fatal(format string, args ...any) {
// Fatalf prints an error message to stderr and exits with status 1
func Fatalf(format string, args ...any) {
fmt.Fprintf(os.Stderr, "FATAL: "+format+"\n", args...)
os.Exit(1)
}
@@ -16,6 +16,6 @@ func Fatal(format string, args ...any) {
func CloseRows(rows *sql.Rows) {
err := rows.Close()
if err != nil {
Fatal("failed to close rows: %v", err)
Fatalf("failed to close rows: %v", err)
}
}

329
internal/database/file_chunks.go
View File

@@ -9,15 +9,21 @@ import (
"sneak.berlin/go/vaultik/internal/types"
)
// FileChunkRepository provides access to the file_chunks table, which maps
// files to their ordered constituent chunks.
type FileChunkRepository struct {
db *DB
}
// NewFileChunkRepository creates a FileChunkRepository backed by db.
func NewFileChunkRepository(db *DB) *FileChunkRepository {
return &FileChunkRepository{db: db}
}
func (r *FileChunkRepository) Create(ctx context.Context, tx *sql.Tx, fc *FileChunk) error {
// Create inserts a file_chunks row (idempotently), using tx when non-nil.
func (r *FileChunkRepository) Create(
ctx context.Context, tx *sql.Tx, fc *FileChunk,
) error {
query := `
INSERT INTO file_chunks (file_id, idx, chunk_hash)
VALUES (?, ?, ?)
@@ -28,7 +34,8 @@ func (r *FileChunkRepository) Create(ctx context.Context, tx *sql.Tx, fc *FileCh
if tx != nil {
_, err = tx.ExecContext(ctx, query, fc.FileID.String(), fc.Idx, fc.ChunkHash.String())
} else {
_, err = r.db.ExecWithLog(ctx, query, fc.FileID.String(), fc.Idx, fc.ChunkHash.String())
_, err = r.db.ExecWithLog(ctx, query,
fc.FileID.String(), fc.Idx, fc.ChunkHash.String())
}
if err != nil {
@@ -38,7 +45,10 @@ func (r *FileChunkRepository) Create(ctx context.Context, tx *sql.Tx, fc *FileCh
return nil
}
func (r *FileChunkRepository) GetByPath(ctx context.Context, path string) ([]*FileChunk, error) {
// GetByPath returns the ordered chunks of the file at the given path.
func (r *FileChunkRepository) GetByPath(
ctx context.Context, path string,
) ([]*FileChunk, error) {
query := `
SELECT fc.file_id, fc.idx, fc.chunk_hash
FROM file_chunks fc
@@ -57,7 +67,9 @@ func (r *FileChunkRepository) GetByPath(ctx context.Context, path string) ([]*Fi
}
// GetByFileID retrieves file chunks by file ID
func (r *FileChunkRepository) GetByFileID(ctx context.Context, fileID types.FileID) ([]*FileChunk, error) {
func (r *FileChunkRepository) GetByFileID(
ctx context.Context, fileID types.FileID,
) ([]*FileChunk, error) {
query := `
SELECT file_id, idx, chunk_hash
FROM file_chunks
@@ -75,7 +87,9 @@ func (r *FileChunkRepository) GetByFileID(ctx context.Context, fileID types.File
}
// GetByPathTx retrieves file chunks within a transaction
func (r *FileChunkRepository) GetByPathTx(ctx context.Context, tx *sql.Tx, path string) ([]*FileChunk, error) {
func (r *FileChunkRepository) GetByPathTx(
ctx context.Context, tx *sql.Tx, path string,
) ([]*FileChunk, error) {
query := `
SELECT fc.file_id, fc.idx, fc.chunk_hash
FROM file_chunks fc
@@ -98,6 +112,170 @@ func (r *FileChunkRepository) GetByPathTx(ctx context.Context, tx *sql.Tx, path
return fileChunks, err
}
// DeleteByPath deletes all file_chunks rows for the file at the given path.
func (r *FileChunkRepository) DeleteByPath(
ctx context.Context, tx *sql.Tx, path string,
) error {
query := `
DELETE FROM file_chunks
WHERE file_id = (SELECT id FROM files WHERE path = ?)
`
var err error
if tx != nil {
_, err = tx.ExecContext(ctx, query, path)
} else {
_, err = r.db.ExecWithLog(ctx, query, path)
}
if err != nil {
return fmt.Errorf("deleting file chunks: %w", err)
}
return nil
}
// DeleteByFileID deletes all chunks for a file by its UUID
func (r *FileChunkRepository) DeleteByFileID(
ctx context.Context, tx *sql.Tx, fileID types.FileID,
) error {
query := `DELETE FROM file_chunks WHERE file_id = ?`
var err error
if tx != nil {
_, err = tx.ExecContext(ctx, query, fileID.String())
} else {
_, err = r.db.ExecWithLog(ctx, query, fileID.String())
}
if err != nil {
return fmt.Errorf("deleting file chunks: %w", err)
}
return nil
}
// DeleteByFileIDs deletes all chunks for multiple files in a single statement.
//
//nolint:dupl // symmetric implementation for a parallel association table
func (r *FileChunkRepository) DeleteByFileIDs(
ctx context.Context, tx *sql.Tx, fileIDs []types.FileID,
) error {
if len(fileIDs) == 0 {
return nil
}
// Batch at 500 to stay within SQLite's variable limit
const batchSize = 500
for i := 0; i < len(fileIDs); i += batchSize {
end := min(i+batchSize, len(fileIDs))
batch := fileIDs[i:end]
//nolint:gosec // G202: concatenates constant SQL and "?" placeholders only
query := "DELETE FROM file_chunks WHERE file_id IN (?" +
repeatPlaceholder(len(batch)-1) + ")"
args := make([]any, len(batch))
for j, id := range batch {
args[j] = id.String()
}
var err error
if tx != nil {
_, err = tx.ExecContext(ctx, query, args...)
} else {
_, err = r.db.ExecWithLog(ctx, query, args...)
}
if err != nil {
return fmt.Errorf("batch deleting file_chunks: %w", err)
}
}
return nil
}
// CreateBatch inserts multiple file_chunks in a single statement for efficiency.
// Batches are automatically split to stay within SQLite's variable limit.
func (r *FileChunkRepository) CreateBatch(
ctx context.Context, tx *sql.Tx, fcs []FileChunk,
) error {
if len(fcs) == 0 {
return nil
}
// Each file_chunks row binds this many SQL variables.
const fileChunkCols = 3
// SQLite has a limit on variables (typically 999 or 32766), so batch
// at 300 rows to be safe.
const batchSize = 300
for i := 0; i < len(fcs); i += batchSize {
end := min(i+batchSize, len(fcs))
batch := fcs[i:end]
// Build the query with multiple value sets
query := "INSERT INTO file_chunks (file_id, idx, chunk_hash) VALUES "
args := make([]any, 0, len(batch)*fileChunkCols)
var querySb211 strings.Builder
for j, fc := range batch {
if j > 0 {
querySb211.WriteString(", ")
}
querySb211.WriteString("(?, ?, ?)")
args = append(args, fc.FileID.String(), fc.Idx, fc.ChunkHash.String())
}
query += querySb211.String() //nolint:gosec // G202: appends "?" placeholders only
query += " ON CONFLICT(file_id, idx) DO NOTHING"
var err error
if tx != nil {
_, err = tx.ExecContext(ctx, query, args...)
} else {
_, err = r.db.ExecWithLog(ctx, query, args...)
}
if err != nil {
return fmt.Errorf("batch inserting file_chunks: %w", err)
}
}
return nil
}
// GetByFile is an alias for GetByPath for compatibility
func (r *FileChunkRepository) GetByFile(
ctx context.Context, path string,
) ([]*FileChunk, error) {
LogSQL("GetByFile", "Starting", path)
result, err := r.GetByPath(ctx, path)
LogSQL("GetByFile", "Complete", path, "count", len(result))
return result, err
}
// GetByFileTx retrieves file chunks within a transaction
func (r *FileChunkRepository) GetByFileTx(
ctx context.Context, tx *sql.Tx, path string,
) ([]*FileChunk, error) {
LogSQL("GetByFileTx", "Starting", path)
result, err := r.GetByPathTx(ctx, tx, path)
LogSQL("GetByFileTx", "Complete", path, "count", len(result))
return result, err
}
// scanFileChunks is a helper that scans file chunk rows
func (r *FileChunkRepository) scanFileChunks(rows *sql.Rows) ([]*FileChunk, error) {
var fileChunks []*FileChunk
@@ -124,144 +302,3 @@ func (r *FileChunkRepository) scanFileChunks(rows *sql.Rows) ([]*FileChunk, erro
return fileChunks, rows.Err()
}
func (r *FileChunkRepository) DeleteByPath(ctx context.Context, tx *sql.Tx, path string) error {
query := `DELETE FROM file_chunks WHERE file_id = (SELECT id FROM files WHERE path = ?)`
var err error
if tx != nil {
_, err = tx.ExecContext(ctx, query, path)
} else {
_, err = r.db.ExecWithLog(ctx, query, path)
}
if err != nil {
return fmt.Errorf("deleting file chunks: %w", err)
}
return nil
}
// DeleteByFileID deletes all chunks for a file by its UUID
func (r *FileChunkRepository) DeleteByFileID(ctx context.Context, tx *sql.Tx, fileID types.FileID) error {
query := `DELETE FROM file_chunks WHERE file_id = ?`
var err error
if tx != nil {
_, err = tx.ExecContext(ctx, query, fileID.String())
} else {
_, err = r.db.ExecWithLog(ctx, query, fileID.String())
}
if err != nil {
return fmt.Errorf("deleting file chunks: %w", err)
}
return nil
}
// DeleteByFileIDs deletes all chunks for multiple files in a single statement.
func (r *FileChunkRepository) DeleteByFileIDs(ctx context.Context, tx *sql.Tx, fileIDs []types.FileID) error {
if len(fileIDs) == 0 {
return nil
}
// Batch at 500 to stay within SQLite's variable limit
const batchSize = 500
for i := 0; i < len(fileIDs); i += batchSize {
end := min(i+batchSize, len(fileIDs))
batch := fileIDs[i:end]
query := "DELETE FROM file_chunks WHERE file_id IN (?" + repeatPlaceholder(len(batch)-1) + ")"
args := make([]any, len(batch))
for j, id := range batch {
args[j] = id.String()
}
var err error
if tx != nil {
_, err = tx.ExecContext(ctx, query, args...)
} else {
_, err = r.db.ExecWithLog(ctx, query, args...)
}
if err != nil {
return fmt.Errorf("batch deleting file_chunks: %w", err)
}
}
return nil
}
// CreateBatch inserts multiple file_chunks in a single statement for efficiency.
// Batches are automatically split to stay within SQLite's variable limit.
func (r *FileChunkRepository) CreateBatch(ctx context.Context, tx *sql.Tx, fcs []FileChunk) error {
if len(fcs) == 0 {
return nil
}
// SQLite has a limit on variables (typically 999 or 32766).
// Each FileChunk has 3 values, so batch at 300 to be safe.
const batchSize = 300
for i := 0; i < len(fcs); i += batchSize {
end := min(i+batchSize, len(fcs))
batch := fcs[i:end]
// Build the query with multiple value sets
query := "INSERT INTO file_chunks (file_id, idx, chunk_hash) VALUES "
args := make([]any, 0, len(batch)*3)
var querySb211 strings.Builder
for j, fc := range batch {
if j > 0 {
querySb211.WriteString(", ")
}
querySb211.WriteString("(?, ?, ?)")
args = append(args, fc.FileID.String(), fc.Idx, fc.ChunkHash.String())
}
query += querySb211.String()
query += " ON CONFLICT(file_id, idx) DO NOTHING"
var err error
if tx != nil {
_, err = tx.ExecContext(ctx, query, args...)
} else {
_, err = r.db.ExecWithLog(ctx, query, args...)
}
if err != nil {
return fmt.Errorf("batch inserting file_chunks: %w", err)
}
}
return nil
}
// GetByFile is an alias for GetByPath for compatibility
func (r *FileChunkRepository) GetByFile(ctx context.Context, path string) ([]*FileChunk, error) {
LogSQL("GetByFile", "Starting", path)
result, err := r.GetByPath(ctx, path)
LogSQL("GetByFile", "Complete", path, "count", len(result))
return result, err
}
// GetByFileTx retrieves file chunks within a transaction
func (r *FileChunkRepository) GetByFileTx(ctx context.Context, tx *sql.Tx, path string) ([]*FileChunk, error) {
LogSQL("GetByFileTx", "Starting", path)
result, err := r.GetByPathTx(ctx, tx, path)
LogSQL("GetByFileTx", "Complete", path, "count", len(result))
return result, err
}

112
internal/database/file_chunks_test.go
View File

@@ -1,4 +1,4 @@
package database
package database_test
import (
"context"
@@ -6,21 +6,25 @@ import (
"testing"
"time"
"sneak.berlin/go/vaultik/internal/database"
"sneak.berlin/go/vaultik/internal/types"
)
func TestFileChunkRepository(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repo := NewFileChunkRepository(db)
fileRepo := NewFileRepository(db)
repo := database.NewFileChunkRepository(db)
fileRepo := database.NewFileRepository(db)
repos := database.NewRepositories(db)
// Create test file first
testTime := time.Now().Truncate(time.Second)
file := &File{
Path: "/test/file.txt",
file := &database.File{
Path: testFileTxt,
MTime: testTime,
Size: 3072,
Mode: 0644,
@@ -29,44 +33,26 @@ func TestFileChunkRepository(t *testing.T) {
LinkTarget: "",
}
err := fileRepo.Create(ctx, nil, file)
if err != nil {
t.Fatalf("failed to create file: %v", err)
}
// Create chunks first
chunks := []types.ChunkHash{"chunk1", "chunk2", "chunk3"}
chunkRepo := NewChunkRepository(db)
for _, chunkHash := range chunks {
chunk := &Chunk{
ChunkHash: chunkHash,
Size: 1024,
}
err = chunkRepo.Create(ctx, nil, chunk)
if err != nil {
t.Fatalf("failed to create chunk %s: %v", chunkHash, err)
}
}
mustCreateFile(t, fileRepo, file)
mustCreateChunks(t, repos, chunk1Hash, chunk2Hash, chunk3Hash)
// Test Create
fc1 := &FileChunk{
fc1 := &database.FileChunk{
FileID: file.ID,
Idx: 0,
ChunkHash: types.ChunkHash("chunk1"),
ChunkHash: types.ChunkHash(chunk1Hash),
}
err = repo.Create(ctx, nil, fc1)
err := repo.Create(ctx, nil, fc1)
if err != nil {
t.Fatalf("failed to create file chunk: %v", err)
}
// Add more chunks for the same file
fc2 := &FileChunk{
fc2 := &database.FileChunk{
FileID: file.ID,
Idx: 1,
ChunkHash: types.ChunkHash("chunk2"),
ChunkHash: types.ChunkHash(chunk2Hash),
}
err = repo.Create(ctx, nil, fc2)
@@ -74,10 +60,10 @@ func TestFileChunkRepository(t *testing.T) {
t.Fatalf("failed to create second file chunk: %v", err)
}
fc3 := &FileChunk{
fc3 := &database.FileChunk{
FileID: file.ID,
Idx: 2,
ChunkHash: types.ChunkHash("chunk3"),
ChunkHash: types.ChunkHash(chunk3Hash),
}
err = repo.Create(ctx, nil, fc3)
@@ -86,7 +72,7 @@ func TestFileChunkRepository(t *testing.T) {
}
// Test GetByFile
fileChunks, err := repo.GetByFile(ctx, "/test/file.txt")
fileChunks, err := repo.GetByFile(ctx, testFileTxt)
if err != nil {
t.Fatalf("failed to get file chunks: %v", err)
}
@@ -107,6 +93,41 @@ func TestFileChunkRepository(t *testing.T) {
if err != nil {
t.Fatalf("failed to create duplicate file chunk: %v", err)
}
}
func TestFileChunkRepositoryDeleteByFileID(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repo := database.NewFileChunkRepository(db)
fileRepo := database.NewFileRepository(db)
repos := database.NewRepositories(db)
file := &database.File{
Path: testFileTxt,
MTime: time.Now().Truncate(time.Second),
Size: 1024,
Mode: 0644,
UID: 1000,
GID: 1000,
}
mustCreateFile(t, fileRepo, file)
mustCreateChunks(t, repos, chunk1Hash)
fc := &database.FileChunk{
FileID: file.ID,
Idx: 0,
ChunkHash: types.ChunkHash(chunk1Hash),
}
err := repo.Create(ctx, nil, fc)
if err != nil {
t.Fatalf("failed to create file chunk: %v", err)
}
// Test DeleteByFileID
err = repo.DeleteByFileID(ctx, nil, file.ID)
@@ -114,7 +135,7 @@ func TestFileChunkRepository(t *testing.T) {
t.Fatalf("failed to delete file chunks: %v", err)
}
fileChunks, err = repo.GetByFileID(ctx, file.ID)
fileChunks, err := repo.GetByFileID(ctx, file.ID)
if err != nil {
t.Fatalf("failed to get deleted file chunks: %v", err)
}
@@ -125,20 +146,22 @@ func TestFileChunkRepository(t *testing.T) {
}
func TestFileChunkRepositoryMultipleFiles(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repo := NewFileChunkRepository(db)
fileRepo := NewFileRepository(db)
repo := database.NewFileChunkRepository(db)
fileRepo := database.NewFileRepository(db)
// Create test files
testTime := time.Now().Truncate(time.Second)
filePaths := []string{"/file1.txt", "/file2.txt", "/file3.txt"}
files := make([]*File, len(filePaths))
filePaths := []string{testFilePath1, testFilePath2, "/file3.txt"}
files := make([]*database.File, len(filePaths))
for i, path := range filePaths {
file := &File{
file := &database.File{
Path: types.FilePath(path),
MTime: testTime,
Size: 2048,
@@ -148,21 +171,18 @@ func TestFileChunkRepositoryMultipleFiles(t *testing.T) {
LinkTarget: "",
}
err := fileRepo.Create(ctx, nil, file)
if err != nil {
t.Fatalf("failed to create file %s: %v", path, err)
}
mustCreateFile(t, fileRepo, file)
files[i] = file
}
// Create all chunks first
chunkRepo := NewChunkRepository(db)
chunkRepo := database.NewChunkRepository(db)
for i := range files {
for j := range 2 {
chunkHash := types.ChunkHash(fmt.Sprintf("file%d_chunk%d", i, j))
chunk := &Chunk{
chunk := &database.Chunk{
ChunkHash: chunkHash,
Size: 1024,
}
@@ -177,7 +197,7 @@ func TestFileChunkRepositoryMultipleFiles(t *testing.T) {
// Create chunks for multiple files
for i, file := range files {
for j := range 2 {
fc := &FileChunk{
fc := &database.FileChunk{
FileID: file.ID,
Idx: j,
ChunkHash: types.ChunkHash(fmt.Sprintf("file%d_chunk%d", i, j)),

205
internal/database/files.go
View File

@@ -12,14 +12,20 @@ import (
"sneak.berlin/go/vaultik/internal/types"
)
// FileRepository provides access to the files table, which stores file
// metadata (path, times, permissions, ownership, symlink targets).
type FileRepository struct {
db *DB
}
// NewFileRepository creates a FileRepository backed by db.
func NewFileRepository(db *DB) *FileRepository {
return &FileRepository{db: db}
}
// Create inserts or updates a file row (upsert on path), using tx when
// non-nil. The file's ID is generated when zero and updated from the
// database's RETURNING clause.
func (r *FileRepository) Create(ctx context.Context, tx *sql.Tx, file *File) error {
// Generate UUID if not provided
if file.ID.IsZero() {
@@ -46,10 +52,19 @@ func (r *FileRepository) Create(ctx context.Context, tx *sql.Tx, file *File) err
)
if tx != nil {
LogSQL("Execute", query, file.ID.String(), file.Path.String(), file.SourcePath.String(), file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID, file.LinkTarget.String())
err = tx.QueryRowContext(ctx, query, file.ID.String(), file.Path.String(), file.SourcePath.String(), file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID, file.LinkTarget.String()).Scan(&idStr)
LogSQL("Execute", query,
file.ID.String(), file.Path.String(), file.SourcePath.String(),
file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID,
file.LinkTarget.String())
err = tx.QueryRowContext(ctx, query,
file.ID.String(), file.Path.String(), file.SourcePath.String(),
file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID,
file.LinkTarget.String()).Scan(&idStr)
} else {
err = r.db.QueryRowWithLog(ctx, query, file.ID.String(), file.Path.String(), file.SourcePath.String(), file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID, file.LinkTarget.String()).Scan(&idStr)
err = r.db.QueryRowWithLog(ctx, query,
file.ID.String(), file.Path.String(), file.SourcePath.String(),
file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID,
file.LinkTarget.String()).Scan(&idStr)
}
if err != nil {
@@ -65,6 +80,8 @@ func (r *FileRepository) Create(ctx context.Context, tx *sql.Tx, file *File) err
return nil
}
// GetByPath returns the file at the given path, or nil if the path is not
// in the index.
func (r *FileRepository) GetByPath(ctx context.Context, path string) (*File, error) {
query := `
SELECT id, path, source_path, mtime, size, mode, uid, gid, link_target
@@ -74,7 +91,7 @@ func (r *FileRepository) GetByPath(ctx context.Context, path string) (*File, err
file, err := r.scanFile(r.db.conn.QueryRowContext(ctx, query, path))
if errors.Is(err, sql.ErrNoRows) {
return nil, nil
return nil, nil //nolint:nilnil // nil,nil signals not-found; callers check nil
}
if err != nil {
@@ -94,7 +111,7 @@ func (r *FileRepository) GetByID(ctx context.Context, id types.FileID) (*File, e
file, err := r.scanFile(r.db.conn.QueryRowContext(ctx, query, id.String()))
if errors.Is(err, sql.ErrNoRows) {
return nil, nil
return nil, nil //nolint:nilnil // nil,nil signals not-found; callers check nil
}
if err != nil {
@@ -104,7 +121,11 @@ func (r *FileRepository) GetByID(ctx context.Context, id types.FileID) (*File, e
return file, nil
}
func (r *FileRepository) GetByPathTx(ctx context.Context, tx *sql.Tx, path string) (*File, error) {
// GetByPathTx returns the file at the given path within a transaction, or
// nil if the path is not in the index.
func (r *FileRepository) GetByPathTx(
ctx context.Context, tx *sql.Tx, path string,
) (*File, error) {
query := `
SELECT id, path, source_path, mtime, size, mode, uid, gid, link_target
FROM files
@@ -116,7 +137,7 @@ func (r *FileRepository) GetByPathTx(ctx context.Context, tx *sql.Tx, path strin
LogSQL("GetByPathTx Scan complete", query, path)
if errors.Is(err, sql.ErrNoRows) {
return nil, nil
return nil, nil //nolint:nilnil // nil,nil signals not-found; callers check nil
}
if err != nil {
@@ -126,87 +147,16 @@ func (r *FileRepository) GetByPathTx(ctx context.Context, tx *sql.Tx, path strin
return file, nil
}
// scanFile is a helper that scans a single file row
func (r *FileRepository) scanFile(row *sql.Row) (*File, error) {
var (
file File
idStr, pathStr, sourcePathStr string
mtimeUnix int64
linkTarget sql.NullString
)
err := row.Scan(
&idStr,
&pathStr,
&sourcePathStr,
&mtimeUnix,
&file.Size,
&file.Mode,
&file.UID,
&file.GID,
&linkTarget,
)
if err != nil {
return nil, err
}
file.ID, err = types.ParseFileID(idStr)
if err != nil {
return nil, fmt.Errorf("parsing file ID: %w", err)
}
file.Path = types.FilePath(pathStr)
file.SourcePath = types.SourcePath(sourcePathStr)
file.MTime = time.Unix(mtimeUnix, 0).UTC()
if linkTarget.Valid {
file.LinkTarget = types.FilePath(linkTarget.String)
}
return &file, nil
// fileRowScanner abstracts *sql.Row and *sql.Rows for scanning a file row.
type fileRowScanner interface {
Scan(dest ...any) error
}
// scanFileRows is a helper that scans a file row from rows iterator
func (r *FileRepository) scanFileRows(rows *sql.Rows) (*File, error) {
var (
file File
idStr, pathStr, sourcePathStr string
mtimeUnix int64
linkTarget sql.NullString
)
err := rows.Scan(
&idStr,
&pathStr,
&sourcePathStr,
&mtimeUnix,
&file.Size,
&file.Mode,
&file.UID,
&file.GID,
&linkTarget,
)
if err != nil {
return nil, err
}
file.ID, err = types.ParseFileID(idStr)
if err != nil {
return nil, fmt.Errorf("parsing file ID: %w", err)
}
file.Path = types.FilePath(pathStr)
file.SourcePath = types.SourcePath(sourcePathStr)
file.MTime = time.Unix(mtimeUnix, 0).UTC()
if linkTarget.Valid {
file.LinkTarget = types.FilePath(linkTarget.String)
}
return &file, nil
}
func (r *FileRepository) ListModifiedSince(ctx context.Context, since time.Time) ([]*File, error) {
// ListModifiedSince returns all files whose recorded mtime is at or after
// since, ordered by path.
func (r *FileRepository) ListModifiedSince(
ctx context.Context, since time.Time,
) ([]*File, error) {
query := `
SELECT id, path, source_path, mtime, size, mode, uid, gid, link_target
FROM files
@@ -234,6 +184,7 @@ func (r *FileRepository) ListModifiedSince(ctx context.Context, since time.Time)
return files, rows.Err()
}
// Delete removes the file row at the given path, using tx when non-nil.
func (r *FileRepository) Delete(ctx context.Context, tx *sql.Tx, path string) error {
query := `DELETE FROM files WHERE path = ?`
@@ -252,7 +203,9 @@ func (r *FileRepository) Delete(ctx context.Context, tx *sql.Tx, path string) er
}
// DeleteByID deletes a file by its UUID
func (r *FileRepository) DeleteByID(ctx context.Context, tx *sql.Tx, id types.FileID) error {
func (r *FileRepository) DeleteByID(
ctx context.Context, tx *sql.Tx, id types.FileID,
) error {
query := `DELETE FROM files WHERE id = ?`
var err error
@@ -269,7 +222,11 @@ func (r *FileRepository) DeleteByID(ctx context.Context, tx *sql.Tx, id types.Fi
return nil
}
func (r *FileRepository) ListByPrefix(ctx context.Context, prefix string) ([]*File, error) {
// ListByPrefix returns all files whose path starts with prefix, ordered by
// path.
func (r *FileRepository) ListByPrefix(
ctx context.Context, prefix string,
) ([]*File, error) {
query := `
SELECT id, path, source_path, mtime, size, mode, uid, gid, link_target
FROM files
@@ -327,12 +284,17 @@ func (r *FileRepository) ListAll(ctx context.Context) ([]*File, error) {
// CreateBatch inserts or updates multiple files in a single statement for efficiency.
// File IDs must be pre-generated before calling this method.
func (r *FileRepository) CreateBatch(ctx context.Context, tx *sql.Tx, files []*File) error {
func (r *FileRepository) CreateBatch(
ctx context.Context, tx *sql.Tx, files []*File,
) error {
if len(files) == 0 {
return nil
}
// Each File has 9 values, so batch at 100 to be safe with SQLite's variable limit
// Each files row binds this many SQL variables.
const fileCols = 9
// Batch at 100 rows to be safe with SQLite's variable limit.
const batchSize = 100
for i := 0; i < len(files); i += batchSize {
@@ -340,9 +302,11 @@ func (r *FileRepository) CreateBatch(ctx context.Context, tx *sql.Tx, files []*F
batch := files[i:end]
query := `INSERT INTO files (id, path, source_path, mtime, size, mode, uid, gid, link_target) VALUES `
query := `INSERT INTO files
(id, path, source_path, mtime, size, mode, uid, gid, link_target)
VALUES `
args := make([]any, 0, len(batch)*9)
args := make([]any, 0, len(batch)*fileCols)
var querySb325 strings.Builder
@@ -353,10 +317,13 @@ func (r *FileRepository) CreateBatch(ctx context.Context, tx *sql.Tx, files []*F
querySb325.WriteString("(?, ?, ?, ?, ?, ?, ?, ?, ?)")
args = append(args, f.ID.String(), f.Path.String(), f.SourcePath.String(), f.MTime.Unix(), f.Size, f.Mode, f.UID, f.GID, f.LinkTarget.String())
args = append(args,
f.ID.String(), f.Path.String(), f.SourcePath.String(),
f.MTime.Unix(), f.Size, f.Mode, f.UID, f.GID,
f.LinkTarget.String())
}
query += querySb325.String()
query += querySb325.String() //nolint:gosec // G202: appends "?" placeholders only
query += ` ON CONFLICT(path) DO UPDATE SET
source_path = excluded.source_path,
@@ -404,3 +371,53 @@ func (r *FileRepository) DeleteOrphaned(ctx context.Context) error {
return nil
}
// scanFile is a helper that scans a single file row
func (r *FileRepository) scanFile(row *sql.Row) (*File, error) {
return r.scanFileFrom(row)
}
// scanFileRows is a helper that scans a file row from rows iterator
func (r *FileRepository) scanFileRows(rows *sql.Rows) (*File, error) {
return r.scanFileFrom(rows)
}
// scanFileFrom scans one file row from any row scanner.
func (r *FileRepository) scanFileFrom(row fileRowScanner) (*File, error) {
var (
file File
idStr, pathStr, sourcePathStr string
mtimeUnix int64
linkTarget sql.NullString
)
err := row.Scan(
&idStr,
&pathStr,
&sourcePathStr,
&mtimeUnix,
&file.Size,
&file.Mode,
&file.UID,
&file.GID,
&linkTarget,
)
if err != nil {
return nil, err
}
file.ID, err = types.ParseFileID(idStr)
if err != nil {
return nil, fmt.Errorf("parsing file ID: %w", err)
}
file.Path = types.FilePath(pathStr)
file.SourcePath = types.SourcePath(sourcePathStr)
file.MTime = time.Unix(mtimeUnix, 0).UTC()
if linkTarget.Valid {
file.LinkTarget = types.FilePath(linkTarget.String)
}
return &file, nil
}

84
internal/database/files_test.go
View File

@@ -1,44 +1,32 @@
package database
package database_test
import (
"context"
"database/sql"
"errors"
"os"
"path/filepath"
"testing"
"time"
"sneak.berlin/go/vaultik/internal/database"
)
func setupTestDB(t *testing.T) (*DB, func()) {
ctx := context.Background()
dbPath := filepath.Join(t.TempDir(), "test.db")
db, err := New(ctx, dbPath)
if err != nil {
t.Fatalf("failed to create database: %v", err)
}
cleanup := func() {
err := db.Close()
if err != nil {
t.Errorf("failed to close database: %v", err)
}
}
return db, cleanup
}
// errTestRollback is the sentinel returned from transaction bodies to
// force a rollback in tests.
var errTestRollback = errors.New("test rollback")
func TestFileRepository(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repo := NewFileRepository(db)
repo := database.NewFileRepository(db)
// Test Create
file := &File{
Path: "/test/file.txt",
file := &database.File{
Path: testFileTxt,
MTime: time.Now().Truncate(time.Second),
Size: 1024,
Mode: 0644,
@@ -95,6 +83,30 @@ func TestFileRepository(t *testing.T) {
if retrieved.Size != 2048 {
t.Errorf("size not updated: got %d, want %d", retrieved.Size, 2048)
}
}
func TestFileRepositoryListDelete(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repo := database.NewFileRepository(db)
file := &database.File{
Path: testFileTxt,
MTime: time.Now().Truncate(time.Second),
Size: 1024,
Mode: 0644,
UID: 1000,
GID: 1000,
}
err := repo.Create(ctx, nil, file)
if err != nil {
t.Fatalf("failed to create file: %v", err)
}
// Test ListModifiedSince
files, err := repo.ListModifiedSince(ctx, time.Now().Add(-1*time.Hour))
@@ -112,7 +124,7 @@ func TestFileRepository(t *testing.T) {
t.Fatalf("failed to delete file: %v", err)
}
retrieved, err = repo.GetByPath(ctx, file.Path.String())
retrieved, err := repo.GetByPath(ctx, file.Path.String())
if err != nil {
t.Fatalf("error getting deleted file: %v", err)
}
@@ -123,14 +135,16 @@ func TestFileRepository(t *testing.T) {
}
func TestFileRepositorySymlink(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repo := NewFileRepository(db)
repo := database.NewFileRepository(db)
// Test symlink
symlink := &File{
symlink := &database.File{
Path: "/test/link",
MTime: time.Now().Truncate(time.Second),
Size: 0,
@@ -155,21 +169,24 @@ func TestFileRepositorySymlink(t *testing.T) {
}
if retrieved.LinkTarget != symlink.LinkTarget {
t.Errorf("link target mismatch: got %s, want %s", retrieved.LinkTarget, symlink.LinkTarget)
t.Errorf("link target mismatch: got %s, want %s",
retrieved.LinkTarget, symlink.LinkTarget)
}
}
func TestFileRepositoryTransaction(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repos := NewRepositories(db)
repos := database.NewRepositories(db)
// Test transaction rollback
err := repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
file := &File{
Path: "/test/tx_file.txt",
file := &database.File{
Path: testTxFile,
MTime: time.Now().Truncate(time.Second),
Size: 1024,
Mode: 0644,
@@ -183,15 +200,14 @@ func TestFileRepositoryTransaction(t *testing.T) {
}
// Return error to trigger rollback
return errors.New("test rollback")
return errTestRollback
})
if err == nil || err.Error() != "test rollback" {
if !errors.Is(err, errTestRollback) {
t.Fatalf("expected rollback error, got: %v", err)
}
// Verify file was not created
retrieved, err := repos.Files.GetByPath(ctx, "/test/tx_file.txt")
retrieved, err := repos.Files.GetByPath(ctx, testTxFile)
if err != nil {
t.Fatalf("error checking for file: %v", err)
}

81
internal/database/helpers_internal_test.go Normal file
View File

@@ -0,0 +1,81 @@
package database
import (
"context"
"path/filepath"
"testing"
"sneak.berlin/go/vaultik/internal/types"
)
// Common fixture values shared by the internal repository tests.
const (
internalTestHost = "test-host"
internalTestSnapshotID = "test-snapshot"
internalTestFilePath = "/test.txt"
internalTestFile1 = "/file1.txt"
internalTestFile2 = "/file2.txt"
// countFilesQuery counts the rows of the files table.
countFilesQuery = "SELECT COUNT(*) FROM files"
)
// mustCreateFileRow inserts the file row, failing the test on error.
func mustCreateFileRow(t *testing.T, repos *Repositories, file *File) {
t.Helper()
err := repos.Files.Create(context.Background(), nil, file)
if err != nil {
t.Fatalf("failed to create file %s: %v", file.Path, err)
}
}
// mustAddFileToSnapshot associates a file with a snapshot, failing the
// test on error.
func mustAddFileToSnapshot(
t *testing.T, repos *Repositories, snapshotID string, fileID types.FileID,
) {
t.Helper()
err := repos.Snapshots.AddFileByID(context.Background(), nil, snapshotID, fileID)
if err != nil {
t.Fatal(err)
}
}
// setupTestDB creates an on-disk test database in a per-test temp
// directory and returns it along with a cleanup func that closes it.
func setupTestDB(t *testing.T) (*DB, func()) {
t.Helper()
ctx := context.Background()
dbPath := filepath.Join(t.TempDir(), "test.db")
db, err := New(ctx, dbPath)
if err != nil {
t.Fatalf("failed to create database: %v", err)
}
cleanup := func() {
err := db.Close()
if err != nil {
t.Errorf("failed to close database: %v", err)
}
}
return db, cleanup
}
// countRow runs a single-integer COUNT-style query and returns the value.
func countRow(t *testing.T, db *DB, query string, args ...any) int {
t.Helper()
var count int
err := db.conn.QueryRowContext(context.Background(), query, args...).Scan(&count)
if err != nil {
t.Fatal(err)
}
return count
}

52
internal/database/helpers_test.go Normal file
View File

@@ -0,0 +1,52 @@
package database_test
import (
"context"
"path/filepath"
"testing"
"sneak.berlin/go/vaultik/internal/database"
)
// Common fixture values shared by the repository tests.
const (
testFilePath1 = "/file1.txt"
testFilePath2 = "/file2.txt"
testFileTxt = "/test/file.txt"
testTxFile = "/test/tx_file.txt"
testHostname = "test-host"
testVersion = "1.0.0"
)
// mustCreateFile inserts the given file row, failing the test on error.
func mustCreateFile(t *testing.T, repo *database.FileRepository, file *database.File) {
t.Helper()
err := repo.Create(context.Background(), nil, file)
if err != nil {
t.Fatalf("failed to create file %s: %v", file.Path, err)
}
}
// setupTestDB creates an on-disk test database in a per-test temp
// directory and returns it along with a cleanup func that closes it.
func setupTestDB(t *testing.T) (*database.DB, func()) {
t.Helper()
ctx := context.Background()
dbPath := filepath.Join(t.TempDir(), "test.db")
db, err := database.New(ctx, dbPath)
if err != nil {
t.Fatalf("failed to create database: %v", err)
}
cleanup := func() {
err := db.Close()
if err != nil {
t.Errorf("failed to close database: %v", err)
}
}
return db, cleanup
}

1
internal/database/local_meta.go
View File

@@ -18,6 +18,7 @@ type LocalMetaRepository struct {
db *DB
}
// NewLocalMetaRepository creates a LocalMetaRepository backed by db.
func NewLocalMetaRepository(db *DB) *LocalMetaRepository {
return &LocalMetaRepository{db: db}
}

15
internal/database/local_meta_test.go
View File

@@ -9,6 +9,8 @@ import (
)
func TestLocalMetaEmptyOnFresh(t *testing.T) {
t.Parallel()
db, err := database.NewTestDB()
require.NoError(t, err)
@@ -22,6 +24,8 @@ func TestLocalMetaEmptyOnFresh(t *testing.T) {
}
func TestLocalMetaSetGetRoundTrip(t *testing.T) {
t.Parallel()
db, err := database.NewTestDB()
require.NoError(t, err)
@@ -30,7 +34,8 @@ func TestLocalMetaSetGetRoundTrip(t *testing.T) {
repos := database.NewRepositories(db)
ctx := context.Background()
require.NoError(t, repos.LocalMeta.Set(ctx, database.LocalMetaKeyStorageURL, "file:///mnt/backups"))
require.NoError(t, repos.LocalMeta.Set(
ctx, database.LocalMetaKeyStorageURL, "file:///mnt/backups"))
got, err := repos.LocalMeta.Get(ctx, database.LocalMetaKeyStorageURL)
require.NoError(t, err)
@@ -38,6 +43,8 @@ func TestLocalMetaSetGetRoundTrip(t *testing.T) {
}
func TestLocalMetaSetOverwrites(t *testing.T) {
t.Parallel()
db, err := database.NewTestDB()
require.NoError(t, err)
@@ -46,8 +53,10 @@ func TestLocalMetaSetOverwrites(t *testing.T) {
repos := database.NewRepositories(db)
ctx := context.Background()
require.NoError(t, repos.LocalMeta.Set(ctx, database.LocalMetaKeyStorageURL, "s3://old"))
require.NoError(t, repos.LocalMeta.Set(ctx, database.LocalMetaKeyStorageURL, "s3://new"))
require.NoError(t, repos.LocalMeta.Set(
ctx, database.LocalMetaKeyStorageURL, "s3://old"))
require.NoError(t, repos.LocalMeta.Set(
ctx, database.LocalMetaKeyStorageURL, "s3://new"))
got, err := repos.LocalMeta.Get(ctx, database.LocalMetaKeyStorageURL)
require.NoError(t, err)

60
internal/database/models.go
View File

@@ -1,5 +1,3 @@
// Package database provides data models and repository interfaces for the Vaultik backup system.
// It includes types for files, chunks, blobs, snapshots, and their relationships.
package database
import (
@@ -13,9 +11,12 @@ import (
// and symlink targets. This information is used to restore files with their
// original attributes.
type File struct {
ID types.FileID // UUID primary key
Path types.FilePath // Absolute path of the file
SourcePath types.SourcePath // The source directory this file came from (for restore path stripping)
ID types.FileID // UUID primary key
Path types.FilePath // Absolute path of the file
// SourcePath is the source directory this file came from (used for
// restore path stripping).
SourcePath types.SourcePath
MTime time.Time
Size int64
Mode uint32
@@ -55,13 +56,16 @@ type Chunk struct {
// The blob creation process is: chunks are accumulated -> compressed with zstd
// -> encrypted with age -> hashed -> uploaded to S3 with the hash as filename.
type Blob struct {
ID types.BlobID // UUID assigned when blob creation starts
Hash types.BlobHash // SHA256 of final compressed+encrypted content (empty until finalized)
CreatedTS time.Time // When blob creation started
FinishedTS *time.Time // When blob was finalized (nil if still packing)
UncompressedSize int64 // Total size of raw chunks before compression
CompressedSize int64 // Size after compression and encryption
UploadedTS *time.Time // When blob was uploaded to S3 (nil if not uploaded)
ID types.BlobID // UUID assigned when blob creation starts
// Hash is the SHA256 of the final compressed+encrypted content
// (empty until finalized).
Hash types.BlobHash
CreatedTS time.Time // When blob creation started
FinishedTS *time.Time // When blob was finalized (nil if still packing)
UncompressedSize int64 // Total size of raw chunks before compression
CompressedSize int64 // Size after compression and encryption
UploadedTS *time.Time // When blob was uploaded to S3 (nil if not uploaded)
}
// BlobChunk represents the mapping between blobs and the chunks they contain.
@@ -75,9 +79,10 @@ type BlobChunk struct {
Length int64
}
// ChunkFile represents the reverse mapping showing which files contain a specific chunk.
// This is used during deduplication to identify all files that share a chunk,
// which is important for garbage collection and integrity verification.
// ChunkFile represents the reverse mapping showing which files contain a
// specific chunk. This is used during deduplication to identify all files
// that share a chunk, which is important for garbage collection and
// integrity verification.
type ChunkFile struct {
ChunkHash types.ChunkHash
FileID types.FileID
@@ -87,17 +92,20 @@ type ChunkFile struct {
// Snapshot represents a snapshot record in the database
type Snapshot struct {
ID types.SnapshotID
Hostname types.Hostname
VaultikVersion types.Version
VaultikGitRevision types.GitRevision
StartedAt time.Time
CompletedAt *time.Time // nil if still in progress
FileCount int64
ChunkCount int64
BlobCount int64
TotalSize int64 // Total size of all referenced files
BlobSize int64 // Total size of all referenced blobs (compressed and encrypted)
ID types.SnapshotID
Hostname types.Hostname
VaultikVersion types.Version
VaultikGitRevision types.GitRevision
StartedAt time.Time
CompletedAt *time.Time // nil if still in progress
FileCount int64
ChunkCount int64
BlobCount int64
TotalSize int64 // Total size of all referenced files
// BlobSize is the total size of all referenced blobs (compressed and
// encrypted).
BlobSize int64
BlobUncompressedSize int64 // Total uncompressed size of all referenced blobs
CompressionRatio float64 // Compression ratio (BlobSize / BlobUncompressedSize)
CompressionLevel int // Compression level used for this snapshot

12
internal/database/module.go
View File

@@ -11,7 +11,13 @@ import (
"sneak.berlin/go/vaultik/internal/log"
)
// indexDirPerm restricts the local index directory to the owning user;
// the index describes the backed-up file tree and must stay private.
const indexDirPerm = 0o700
// Module provides database dependencies
//
//nolint:gochecknoglobals // fx module definitions are package globals by convention
var Module = fx.Module("database",
fx.Provide(
provideDatabase,
@@ -22,7 +28,9 @@ var Module = fx.Module("database",
func provideDatabase(lc fx.Lifecycle, cfg *config.Config) (*DB, error) {
// Ensure the index directory exists
indexDir := filepath.Dir(cfg.IndexPath)
if err := os.MkdirAll(indexDir, 0700); err != nil {
err := os.MkdirAll(indexDir, indexDirPerm)
if err != nil {
return nil, fmt.Errorf("creating index directory: %w", err)
}
@@ -32,7 +40,7 @@ func provideDatabase(lc fx.Lifecycle, cfg *config.Config) (*DB, error) {
}
lc.Append(fx.Hook{
OnStop: func(ctx context.Context) error {
OnStop: func(_ context.Context) error {
log.Debug("Database module OnStop hook called")
err := db.Close()

8
internal/database/repositories.go
View File

@@ -62,14 +62,14 @@ func (r *Repositories) WithTx(ctx context.Context, fn TxFunc) error {
if p := recover(); p != nil {
rollbackErr := tx.Rollback()
if rollbackErr != nil {
Fatal("failed to rollback transaction: %v", rollbackErr)
Fatalf("failed to rollback transaction: %v", rollbackErr)
}
panic(p)
} else if err != nil {
rollbackErr := tx.Rollback()
if rollbackErr != nil {
Fatal("failed to rollback transaction: %v", rollbackErr)
Fatalf("failed to rollback transaction: %v", rollbackErr)
}
}
}()
@@ -105,14 +105,14 @@ func (r *Repositories) WithReadTx(ctx context.Context, fn TxFunc) error {
if p := recover(); p != nil {
rollbackErr := tx.Rollback()
if rollbackErr != nil {
Fatal("failed to rollback transaction: %v", rollbackErr)
Fatalf("failed to rollback transaction: %v", rollbackErr)
}
panic(p)
} else if err != nil {
rollbackErr := tx.Rollback()
if rollbackErr != nil {
Fatal("failed to rollback transaction: %v", rollbackErr)
Fatalf("failed to rollback transaction: %v", rollbackErr)
}
}
}()

206
internal/database/repositories_test.go
View File

@@ -1,4 +1,4 @@
package database
package database_test
import (
"context"
@@ -7,21 +7,21 @@ import (
"testing"
"time"
"sneak.berlin/go/vaultik/internal/database"
"sneak.berlin/go/vaultik/internal/types"
)
func TestRepositoriesTransaction(t *testing.T) {
db, cleanup := setupTestDB(t)
defer cleanup()
// errIntentionalRollback forces a transaction rollback in tests.
var errIntentionalRollback = errors.New("intentional rollback")
ctx := context.Background()
repos := NewRepositories(db)
// Test successful transaction with multiple operations
err := repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
// Create a file
file := &File{
Path: "/test/tx_file.txt",
// createTxTestData returns a transaction body that creates a file with
// two chunks packed into one blob.
func createTxTestData(
repos *database.Repositories,
) func(context.Context, *sql.Tx) error {
return func(ctx context.Context, tx *sql.Tx) error {
file := &database.File{
Path: testTxFile,
MTime: time.Now().Truncate(time.Second),
Size: 1024,
Mode: 0644,
@@ -34,95 +34,116 @@ func TestRepositoriesTransaction(t *testing.T) {
return err
}
// Create chunks
chunk1 := &Chunk{
ChunkHash: types.ChunkHash("tx_chunk1"),
Size: 512,
}
err = repos.Chunks.Create(ctx, tx, chunk1)
err = createTxFileChunks(ctx, tx, repos, file.ID)
if err != nil {
return err
}
chunk2 := &Chunk{
ChunkHash: types.ChunkHash("tx_chunk2"),
Size: 512,
}
return createTxBlob(ctx, tx, repos)
}
}
err = repos.Chunks.Create(ctx, tx, chunk2)
if err != nil {
return err
}
// createTxFileChunks creates the two test chunks and maps them to the file.
func createTxFileChunks(
ctx context.Context, tx *sql.Tx,
repos *database.Repositories, fileID types.FileID,
) error {
// Create chunks
chunk1 := &database.Chunk{
ChunkHash: types.ChunkHash("tx_chunk1"),
Size: 512,
}
// Map chunks to file
fc1 := &FileChunk{
FileID: file.ID,
Idx: 0,
ChunkHash: chunk1.ChunkHash,
}
err := repos.Chunks.Create(ctx, tx, chunk1)
if err != nil {
return err
}
err = repos.FileChunks.Create(ctx, tx, fc1)
if err != nil {
return err
}
chunk2 := &database.Chunk{
ChunkHash: types.ChunkHash("tx_chunk2"),
Size: 512,
}
fc2 := &FileChunk{
FileID: file.ID,
Idx: 1,
ChunkHash: chunk2.ChunkHash,
}
err = repos.Chunks.Create(ctx, tx, chunk2)
if err != nil {
return err
}
err = repos.FileChunks.Create(ctx, tx, fc2)
if err != nil {
return err
}
// Map chunks to file
fc1 := &database.FileChunk{
FileID: fileID,
Idx: 0,
ChunkHash: chunk1.ChunkHash,
}
// Create blob
blob := &Blob{
ID: types.NewBlobID(),
Hash: types.BlobHash("tx_blob1"),
CreatedTS: time.Now().Truncate(time.Second),
}
err = repos.FileChunks.Create(ctx, tx, fc1)
if err != nil {
return err
}
err = repos.Blobs.Create(ctx, tx, blob)
if err != nil {
return err
}
fc2 := &database.FileChunk{
FileID: fileID,
Idx: 1,
ChunkHash: chunk2.ChunkHash,
}
// Map chunks to blob
bc1 := &BlobChunk{
BlobID: blob.ID,
ChunkHash: chunk1.ChunkHash,
Offset: 0,
Length: 512,
}
return repos.FileChunks.Create(ctx, tx, fc2)
}
err = repos.BlobChunks.Create(ctx, tx, bc1)
if err != nil {
return err
}
// createTxBlob creates the test blob and maps both chunks into it.
func createTxBlob(
ctx context.Context, tx *sql.Tx, repos *database.Repositories,
) error {
blob := &database.Blob{
ID: types.NewBlobID(),
Hash: types.BlobHash("tx_blob1"),
CreatedTS: time.Now().Truncate(time.Second),
}
bc2 := &BlobChunk{
BlobID: blob.ID,
ChunkHash: chunk2.ChunkHash,
Offset: 512,
Length: 512,
}
err := repos.Blobs.Create(ctx, tx, blob)
if err != nil {
return err
}
err = repos.BlobChunks.Create(ctx, tx, bc2)
if err != nil {
return err
}
// Map chunks to blob
bc1 := &database.BlobChunk{
BlobID: blob.ID,
ChunkHash: types.ChunkHash("tx_chunk1"),
Offset: 0,
Length: 512,
}
return nil
})
err = repos.BlobChunks.Create(ctx, tx, bc1)
if err != nil {
return err
}
bc2 := &database.BlobChunk{
BlobID: blob.ID,
ChunkHash: types.ChunkHash("tx_chunk2"),
Offset: 512,
Length: 512,
}
return repos.BlobChunks.Create(ctx, tx, bc2)
}
func TestRepositoriesTransaction(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repos := database.NewRepositories(db)
err := repos.WithTx(ctx, createTxTestData(repos))
if err != nil {
t.Fatalf("transaction failed: %v", err)
}
// Verify all data was committed
file, err := repos.Files.GetByPath(ctx, "/test/tx_file.txt")
file, err := repos.Files.GetByPath(ctx, testTxFile)
if err != nil {
t.Fatalf("failed to get file: %v", err)
}
@@ -131,7 +152,7 @@ func TestRepositoriesTransaction(t *testing.T) {
t.Error("expected file after transaction")
}
chunks, err := repos.FileChunks.GetByFile(ctx, "/test/tx_file.txt")
chunks, err := repos.FileChunks.GetByFile(ctx, testTxFile)
if err != nil {
t.Fatalf("failed to get file chunks: %v", err)
}
@@ -151,16 +172,18 @@ func TestRepositoriesTransaction(t *testing.T) {
}
func TestRepositoriesTransactionRollback(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repos := NewRepositories(db)
repos := database.NewRepositories(db)
// Test transaction rollback
err := repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
// Create a file
file := &File{
file := &database.File{
Path: "/test/rollback_file.txt",
MTime: time.Now().Truncate(time.Second),
Size: 1024,
@@ -175,7 +198,7 @@ func TestRepositoriesTransactionRollback(t *testing.T) {
}
// Create a chunk
chunk := &Chunk{
chunk := &database.Chunk{
ChunkHash: types.ChunkHash("rollback_chunk"),
Size: 1024,
}
@@ -186,10 +209,9 @@ func TestRepositoriesTransactionRollback(t *testing.T) {
}
// Return error to trigger rollback
return errors.New("intentional rollback")
return errIntentionalRollback
})
if err == nil || err.Error() != "intentional rollback" {
if !errors.Is(err, errIntentionalRollback) {
t.Fatalf("expected rollback error, got: %v", err)
}
@@ -214,14 +236,16 @@ func TestRepositoriesTransactionRollback(t *testing.T) {
}
func TestRepositoriesReadTransaction(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repos := NewRepositories(db)
repos := database.NewRepositories(db)
// First, create some data
file := &File{
file := &database.File{
Path: "/test/read_file.txt",
MTime: time.Now().Truncate(time.Second),
Size: 1024,
@@ -236,7 +260,7 @@ func TestRepositoriesReadTransaction(t *testing.T) {
}
// Test read-only transaction
var retrievedFile *File
var retrievedFile *database.File
err = repos.WithReadTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
var err error
@@ -247,7 +271,7 @@ func TestRepositoriesReadTransaction(t *testing.T) {
}
// Try to write in read-only transaction (should fail)
_ = repos.Files.Create(ctx, tx, &File{
_ = repos.Files.Create(ctx, tx, &database.File{
Path: "/test/should_fail.txt",
MTime: time.Now(),
Size: 0,

214
internal/database/repository_comprehensive_test.go
View File

@@ -1,3 +1,4 @@
//nolint:testpackage // inspects the unexported database connection
package database
import (
@@ -11,8 +12,13 @@ import (
"sneak.berlin/go/vaultik/internal/types"
)
// errTxIntentionalRollback forces a transaction rollback in tests.
var errTxIntentionalRollback = errors.New("intentional rollback")
// TestFileRepositoryUUIDGeneration tests that files get unique UUIDs
func TestFileRepositoryUUIDGeneration(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
@@ -22,7 +28,7 @@ func TestFileRepositoryUUIDGeneration(t *testing.T) {
// Create multiple files
files := []*File{
{
Path: "/file1.txt",
Path: internalTestFile1,
MTime: time.Now().Truncate(time.Second),
Size: 1024,
Mode: 0644,
@@ -30,7 +36,7 @@ func TestFileRepositoryUUIDGeneration(t *testing.T) {
GID: 1000,
},
{
Path: "/file2.txt",
Path: internalTestFile2,
MTime: time.Now().Truncate(time.Second),
Size: 2048,
Mode: 0644,
@@ -63,6 +69,8 @@ func TestFileRepositoryUUIDGeneration(t *testing.T) {
// TestFileRepositoryGetByID tests retrieving files by UUID
func TestFileRepositoryGetByID(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
@@ -71,7 +79,7 @@ func TestFileRepositoryGetByID(t *testing.T) {
// Create a file
file := &File{
Path: "/test.txt",
Path: internalTestFilePath,
MTime: time.Now().Truncate(time.Second),
Size: 1024,
Mode: 0644,
@@ -98,8 +106,9 @@ func TestFileRepositoryGetByID(t *testing.T) {
t.Errorf("Path mismatch: expected %s, got %s", file.Path, retrieved.Path)
}
// Test non-existent ID
nonExistentID := types.NewFileID() // Generate a new UUID that won't exist in the database
// Test non-existent ID: generate a new UUID that won't exist in the
// database.
nonExistentID := types.NewFileID()
nonExistent, err := repo.GetByID(ctx, nonExistentID)
if err != nil {
@@ -113,6 +122,8 @@ func TestFileRepositoryGetByID(t *testing.T) {
// TestOrphanedFileCleanup tests the cleanup of orphaned files
func TestOrphanedFileCleanup(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
@@ -149,8 +160,8 @@ func TestOrphanedFileCleanup(t *testing.T) {
// Create a snapshot and reference only file2
snapshot := &Snapshot{
ID: "test-snapshot",
Hostname: "test-host",
ID: internalTestSnapshotID,
Hostname: internalTestHost,
StartedAt: time.Now(),
}
@@ -160,10 +171,7 @@ func TestOrphanedFileCleanup(t *testing.T) {
}
// Add file2 to snapshot
err = repos.Snapshots.AddFileByID(ctx, nil, snapshot.ID.String(), file2.ID)
if err != nil {
t.Fatalf("failed to add file to snapshot: %v", err)
}
mustAddFileToSnapshot(t, repos, snapshot.ID.String(), file2.ID)
// Run orphaned cleanup
err = repos.Files.DeleteOrphaned(ctx)
@@ -194,6 +202,8 @@ func TestOrphanedFileCleanup(t *testing.T) {
// TestOrphanedChunkCleanup tests the cleanup of orphaned chunks
func TestOrphanedChunkCleanup(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
@@ -222,7 +232,7 @@ func TestOrphanedChunkCleanup(t *testing.T) {
// Create a file and reference only chunk2
file := &File{
Path: "/test.txt",
Path: internalTestFilePath,
MTime: time.Now().Truncate(time.Second),
Size: 1024,
Mode: 0644,
@@ -276,6 +286,8 @@ func TestOrphanedChunkCleanup(t *testing.T) {
// TestOrphanedBlobCleanup tests the cleanup of orphaned blobs
func TestOrphanedBlobCleanup(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
@@ -306,8 +318,8 @@ func TestOrphanedBlobCleanup(t *testing.T) {
// Create a snapshot and reference only blob2
snapshot := &Snapshot{
ID: "test-snapshot",
Hostname: "test-host",
ID: internalTestSnapshotID,
Hostname: internalTestHost,
StartedAt: time.Now(),
}
@@ -351,6 +363,8 @@ func TestOrphanedBlobCleanup(t *testing.T) {
// TestFileChunkRepositoryWithUUIDs tests file-chunk relationships with UUIDs
func TestFileChunkRepositoryWithUUIDs(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
@@ -359,7 +373,7 @@ func TestFileChunkRepositoryWithUUIDs(t *testing.T) {
// Create a file
file := &File{
Path: "/test.txt",
Path: internalTestFilePath,
MTime: time.Now().Truncate(time.Second),
Size: 3072,
Mode: 0644,
@@ -367,10 +381,7 @@ func TestFileChunkRepositoryWithUUIDs(t *testing.T) {
GID: 1000,
}
err := repos.Files.Create(ctx, nil, file)
if err != nil {
t.Fatalf("failed to create file: %v", err)
}
mustCreateFileRow(t, repos, file)
// Create chunks
chunks := []types.ChunkHash{"chunk1", "chunk2", "chunk3"}
@@ -380,7 +391,7 @@ func TestFileChunkRepositoryWithUUIDs(t *testing.T) {
Size: 1024,
}
err = repos.Chunks.Create(ctx, nil, chunk)
err := repos.Chunks.Create(ctx, nil, chunk)
if err != nil {
t.Fatalf("failed to create chunk: %v", err)
}
@@ -426,6 +437,8 @@ func TestFileChunkRepositoryWithUUIDs(t *testing.T) {
// TestChunkFileRepositoryWithUUIDs tests chunk-file relationships with UUIDs
func TestChunkFileRepositoryWithUUIDs(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
@@ -434,7 +447,7 @@ func TestChunkFileRepositoryWithUUIDs(t *testing.T) {
// Create files
file1 := &File{
Path: "/file1.txt",
Path: internalTestFile1,
MTime: time.Now().Truncate(time.Second),
Size: 1024,
Mode: 0644,
@@ -442,7 +455,7 @@ func TestChunkFileRepositoryWithUUIDs(t *testing.T) {
GID: 1000,
}
file2 := &File{
Path: "/file2.txt",
Path: internalTestFile2,
MTime: time.Now().Truncate(time.Second),
Size: 1024,
Mode: 0644,
@@ -450,15 +463,8 @@ func TestChunkFileRepositoryWithUUIDs(t *testing.T) {
GID: 1000,
}
err := repos.Files.Create(ctx, nil, file1)
if err != nil {
t.Fatalf("failed to create file1: %v", err)
}
err = repos.Files.Create(ctx, nil, file2)
if err != nil {
t.Fatalf("failed to create file2: %v", err)
}
mustCreateFileRow(t, repos, file1)
mustCreateFileRow(t, repos, file2)
// Create a chunk that appears in both files (deduplication)
chunk := &Chunk{
@@ -466,7 +472,7 @@ func TestChunkFileRepositoryWithUUIDs(t *testing.T) {
Size: 1024,
}
err = repos.Chunks.Create(ctx, nil, chunk)
err := repos.Chunks.Create(ctx, nil, chunk)
if err != nil {
t.Fatalf("failed to create chunk: %v", err)
}
@@ -518,6 +524,8 @@ func TestChunkFileRepositoryWithUUIDs(t *testing.T) {
// TestSnapshotRepositoryExtendedFields tests snapshot with version and git revision
func TestSnapshotRepositoryExtendedFields(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
@@ -527,7 +535,7 @@ func TestSnapshotRepositoryExtendedFields(t *testing.T) {
// Create snapshot with extended fields
snapshot := &Snapshot{
ID: "test-20250722-120000Z",
Hostname: "test-host",
Hostname: internalTestHost,
VaultikVersion: "0.0.1",
VaultikGitRevision: "abc123def456",
StartedAt: time.Now(),
@@ -555,35 +563,39 @@ func TestSnapshotRepositoryExtendedFields(t *testing.T) {
}
if retrieved.VaultikVersion != snapshot.VaultikVersion {
t.Errorf("version mismatch: expected %s, got %s", snapshot.VaultikVersion, retrieved.VaultikVersion)
t.Errorf("version mismatch: expected %s, got %s",
snapshot.VaultikVersion, retrieved.VaultikVersion)
}
if retrieved.VaultikGitRevision != snapshot.VaultikGitRevision {
t.Errorf("git revision mismatch: expected %s, got %s", snapshot.VaultikGitRevision, retrieved.VaultikGitRevision)
t.Errorf("git revision mismatch: expected %s, got %s",
snapshot.VaultikGitRevision, retrieved.VaultikGitRevision)
}
if retrieved.CompressionLevel != snapshot.CompressionLevel {
t.Errorf("compression level mismatch: expected %d, got %d", snapshot.CompressionLevel, retrieved.CompressionLevel)
t.Errorf("compression level mismatch: expected %d, got %d",
snapshot.CompressionLevel, retrieved.CompressionLevel)
}
if retrieved.BlobUncompressedSize != snapshot.BlobUncompressedSize {
t.Errorf("uncompressed size mismatch: expected %d, got %d", snapshot.BlobUncompressedSize, retrieved.BlobUncompressedSize)
t.Errorf("uncompressed size mismatch: expected %d, got %d",
snapshot.BlobUncompressedSize, retrieved.BlobUncompressedSize)
}
if retrieved.UploadDurationMs != snapshot.UploadDurationMs {
t.Errorf("upload duration mismatch: expected %d, got %d", snapshot.UploadDurationMs, retrieved.UploadDurationMs)
t.Errorf("upload duration mismatch: expected %d, got %d",
snapshot.UploadDurationMs, retrieved.UploadDurationMs)
}
}
// TestComplexOrphanedDataScenario tests a complex scenario with multiple relationships
func TestComplexOrphanedDataScenario(t *testing.T) {
db, cleanup := setupTestDB(t)
defer cleanup()
// createOrphanScenarioFixtures creates two snapshots and three files for
// the orphaned-data cleanup scenario.
func createOrphanScenarioFixtures(
ctx context.Context, t *testing.T, repos *Repositories,
) (*Snapshot, *Snapshot, []*File) {
t.Helper()
ctx := context.Background()
repos := NewRepositories(db)
// Create snapshots
snapshot1 := &Snapshot{
ID: "snapshot1",
Hostname: "host1",
@@ -623,34 +635,33 @@ func TestComplexOrphanedDataScenario(t *testing.T) {
}
}
return snapshot1, snapshot2, files
}
func TestComplexOrphanedDataScenario(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repos := NewRepositories(db)
snapshot1, snapshot2, files := createOrphanScenarioFixtures(ctx, t, repos)
// Add files to snapshots
// Snapshot1: file0, file1
// Snapshot2: file1, file2
// file0: only in snapshot1
// file1: in both snapshots
// file2: only in snapshot2
err = repos.Snapshots.AddFileByID(ctx, nil, snapshot1.ID.String(), files[0].ID)
if err != nil {
t.Fatal(err)
}
err = repos.Snapshots.AddFileByID(ctx, nil, snapshot1.ID.String(), files[1].ID)
if err != nil {
t.Fatal(err)
}
err = repos.Snapshots.AddFileByID(ctx, nil, snapshot2.ID.String(), files[1].ID)
if err != nil {
t.Fatal(err)
}
err = repos.Snapshots.AddFileByID(ctx, nil, snapshot2.ID.String(), files[2].ID)
if err != nil {
t.Fatal(err)
}
mustAddFileToSnapshot(t, repos, snapshot1.ID.String(), files[0].ID)
mustAddFileToSnapshot(t, repos, snapshot1.ID.String(), files[1].ID)
mustAddFileToSnapshot(t, repos, snapshot2.ID.String(), files[1].ID)
mustAddFileToSnapshot(t, repos, snapshot2.ID.String(), files[2].ID)
// Delete snapshot1
err = repos.Snapshots.DeleteSnapshotFiles(ctx, snapshot1.ID.String())
err := repos.Snapshots.DeleteSnapshotFiles(ctx, snapshot1.ID.String())
if err != nil {
t.Fatal(err)
}
@@ -700,6 +711,8 @@ func TestComplexOrphanedDataScenario(t *testing.T) {
// TestCascadeDelete tests that cascade deletes work properly
func TestCascadeDelete(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
@@ -774,6 +787,8 @@ func TestCascadeDelete(t *testing.T) {
// TestTransactionIsolation tests that transactions properly isolate changes
func TestTransactionIsolation(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
@@ -802,7 +817,7 @@ func TestTransactionIsolation(t *testing.T) {
// For now, we'll just test that rollback works
// Return an error to trigger rollback
return errors.New("intentional rollback")
return errTxIntentionalRollback
})
if err == nil {
t.Fatal("expected error from transaction")
@@ -819,32 +834,15 @@ func TestTransactionIsolation(t *testing.T) {
}
}
// TestConcurrentOrphanedCleanup tests that concurrent cleanup operations don't interfere
func TestConcurrentOrphanedCleanup(t *testing.T) {
db, cleanup := setupTestDB(t)
defer cleanup()
// TestConcurrentOrphanedCleanup tests that concurrent cleanup operations
// don't interfere.
// createConcurrentCleanupFiles creates 20 files and associates the
// even-numbered ones with the snapshot, leaving the rest orphaned.
func createConcurrentCleanupFiles(
ctx context.Context, t *testing.T, repos *Repositories, snapshotID string,
) {
t.Helper()
ctx := context.Background()
repos := NewRepositories(db)
// Set a 5-second busy timeout to handle concurrent operations
if _, err := db.conn.Exec("PRAGMA busy_timeout = 5000"); err != nil {
t.Fatalf("failed to set busy timeout: %v", err)
}
// Create a snapshot
snapshot := &Snapshot{
ID: "concurrent-test",
Hostname: "test-host",
StartedAt: time.Now(),
}
err := repos.Snapshots.Create(ctx, nil, snapshot)
if err != nil {
t.Fatal(err)
}
// Create many files, some orphaned
for i := range 20 {
file := &File{
Path: types.FilePath(fmt.Sprintf("/concurrent-%d.txt", i)),
@@ -855,19 +853,49 @@ func TestConcurrentOrphanedCleanup(t *testing.T) {
GID: 1000,
}
err = repos.Files.Create(ctx, nil, file)
err := repos.Files.Create(ctx, nil, file)
if err != nil {
t.Fatal(err)
}
// Add even-numbered files to snapshot
if i%2 == 0 {
err = repos.Snapshots.AddFileByID(ctx, nil, snapshot.ID.String(), file.ID)
err = repos.Snapshots.AddFileByID(ctx, nil, snapshotID, file.ID)
if err != nil {
t.Fatal(err)
}
}
}
}
func TestConcurrentOrphanedCleanup(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repos := NewRepositories(db)
// Set a 5-second busy timeout to handle concurrent operations
_, err := db.conn.ExecContext(ctx, "PRAGMA busy_timeout = 5000")
if err != nil {
t.Fatalf("failed to set busy timeout: %v", err)
}
// Create a snapshot
snapshot := &Snapshot{
ID: "concurrent-test",
Hostname: internalTestHost,
StartedAt: time.Now(),
}
err = repos.Snapshots.Create(ctx, nil, snapshot)
if err != nil {
t.Fatal(err)
}
createConcurrentCleanupFiles(ctx, t, repos, snapshot.ID.String())
// Run multiple cleanup operations concurrently
// Note: SQLite has limited support for concurrent writes, so we expect some to fail

132
internal/database/repository_debug_test.go
View File

@@ -1,3 +1,4 @@
//nolint:testpackage // inspects the unexported database connection
package database
import (
@@ -6,15 +7,50 @@ import (
"time"
)
// TestOrphanedFileCleanupDebug tests orphaned file cleanup with debug output
func TestOrphanedFileCleanupDebug(t *testing.T) {
db, cleanup := setupTestDB(t)
defer cleanup()
// logSnapshotFileIDs logs every file_id present in snapshot_files.
func logSnapshotFileIDs(t *testing.T, db *DB) {
t.Helper()
ctx := context.Background()
repos := NewRepositories(db)
// Create files
rows, err := db.conn.QueryContext(ctx, "SELECT file_id FROM snapshot_files")
if err != nil {
t.Fatal(err)
}
defer func() {
err := rows.Close()
if err != nil {
t.Logf("failed to close rows: %v", err)
}
}()
t.Log("Files in snapshot_files:")
for rows.Next() {
var fileID string
err := rows.Scan(&fileID)
if err != nil {
t.Fatal(err)
}
t.Logf(" - %s", fileID)
}
err = rows.Err()
if err != nil {
t.Fatal(err)
}
}
// TestOrphanedFileCleanupDebug tests orphaned file cleanup with debug output
// createOrphanDebugFixtures creates one orphaned file, one referenced
// file, and the snapshot that will reference the latter.
func createOrphanDebugFixtures(
ctx context.Context, t *testing.T, repos *Repositories,
) (*File, *File, *Snapshot) {
t.Helper()
file1 := &File{
Path: "/orphaned.txt",
MTime: time.Now().Truncate(time.Second),
@@ -48,8 +84,8 @@ func TestOrphanedFileCleanupDebug(t *testing.T) {
// Create a snapshot and reference only file2
snapshot := &Snapshot{
ID: "test-snapshot",
Hostname: "test-host",
ID: internalTestSnapshotID,
Hostname: internalTestHost,
StartedAt: time.Now(),
}
@@ -60,18 +96,26 @@ func TestOrphanedFileCleanupDebug(t *testing.T) {
t.Logf("Created snapshot: %s", snapshot.ID)
return file1, file2, snapshot
}
func TestOrphanedFileCleanupDebug(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repos := NewRepositories(db)
file1, file2, snapshot := createOrphanDebugFixtures(ctx, t, repos)
// Check snapshot_files before adding
var count int
err = db.conn.QueryRow("SELECT COUNT(*) FROM snapshot_files").Scan(&count)
if err != nil {
t.Fatal(err)
}
count := countRow(t, db, "SELECT COUNT(*) FROM snapshot_files")
t.Logf("snapshot_files count before add: %d", count)
// Add file2 to snapshot
err = repos.Snapshots.AddFileByID(ctx, nil, snapshot.ID.String(), file2.ID)
err := repos.Snapshots.AddFileByID(ctx, nil, snapshot.ID.String(), file2.ID)
if err != nil {
t.Fatalf("failed to add file to snapshot: %v", err)
}
@@ -79,44 +123,14 @@ func TestOrphanedFileCleanupDebug(t *testing.T) {
t.Logf("Added file2 to snapshot")
// Check snapshot_files after adding
err = db.conn.QueryRow("SELECT COUNT(*) FROM snapshot_files").Scan(&count)
if err != nil {
t.Fatal(err)
}
count = countRow(t, db, "SELECT COUNT(*) FROM snapshot_files")
t.Logf("snapshot_files count after add: %d", count)
// Check which files are referenced
rows, err := db.conn.Query("SELECT file_id FROM snapshot_files")
if err != nil {
t.Fatal(err)
}
defer func() {
err := rows.Close()
if err != nil {
t.Logf("failed to close rows: %v", err)
}
}()
t.Log("Files in snapshot_files:")
for rows.Next() {
var fileID string
err := rows.Scan(&fileID)
if err != nil {
t.Fatal(err)
}
t.Logf(" - %s", fileID)
}
logSnapshotFileIDs(t, db)
// Check files before cleanup
err = db.conn.QueryRow("SELECT COUNT(*) FROM files").Scan(&count)
if err != nil {
t.Fatal(err)
}
count = countRow(t, db, countFilesQuery)
t.Logf("Files count before cleanup: %d", count)
// Run orphaned cleanup
@@ -128,11 +142,7 @@ func TestOrphanedFileCleanupDebug(t *testing.T) {
t.Log("Ran orphaned cleanup")
// Check files after cleanup
err = db.conn.QueryRow("SELECT COUNT(*) FROM files").Scan(&count)
if err != nil {
t.Fatal(err)
}
count = countRow(t, db, countFilesQuery)
t.Logf("Files count after cleanup: %d", count)
// List remaining files
@@ -156,18 +166,12 @@ func TestOrphanedFileCleanupDebug(t *testing.T) {
if orphanedFile != nil {
t.Error("orphaned file should have been deleted")
// Let's check why it wasn't deleted
var exists bool
err = db.conn.QueryRow(`
stillReferenced := countRow(t, db, `
SELECT EXISTS(
SELECT 1 FROM snapshot_files
SELECT 1 FROM snapshot_files
WHERE file_id = ?
)`, file1.ID).Scan(&exists)
if err != nil {
t.Fatal(err)
}
t.Logf("File1 exists in snapshot_files: %v", exists)
)`, file1.ID)
t.Logf("File1 exists in snapshot_files: %v", stillReferenced != 0)
} else {
t.Log("Orphaned file was correctly deleted")
}

407
internal/database/repository_edge_cases_test.go
View File

@@ -1,3 +1,4 @@
//nolint:testpackage // inspects the unexported database connection
package database
import (
@@ -10,20 +11,17 @@ import (
"sneak.berlin/go/vaultik/internal/types"
)
// TestFileRepositoryEdgeCases tests edge cases for file repository
func TestFileRepositoryEdgeCases(t *testing.T) {
db, cleanup := setupTestDB(t)
defer cleanup()
// fileEdgeCase describes one Create edge-case scenario.
type fileEdgeCase struct {
name string
file *File
wantErr bool
errMsg string
}
ctx := context.Background()
repo := NewFileRepository(db)
tests := []struct {
name string
file *File
wantErr bool
errMsg string
}{
// fileEdgeCases returns the Create edge-case table.
func fileEdgeCases() []fileEdgeCase {
return []fileEdgeCase{
{
name: "empty path",
file: &File{
@@ -51,6 +49,7 @@ func TestFileRepositoryEdgeCases(t *testing.T) {
{
name: "path with special characters",
file: &File{
//nolint:gosmopolitan // non-ASCII path is deliberate test data
Path: "/test/file with spaces and 特殊文字.txt",
MTime: time.Now(),
Size: 1024,
@@ -86,12 +85,26 @@ func TestFileRepositoryEdgeCases(t *testing.T) {
wantErr: false,
},
}
}
for i, tt := range tests {
// TestFileRepositoryEdgeCases tests edge cases for file repository
func TestFileRepositoryEdgeCases(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
t.Cleanup(cleanup)
ctx := context.Background()
repo := NewFileRepository(db)
for i, tt := range fileEdgeCases() {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
// Add a unique suffix to paths to avoid UNIQUE constraint violations
if tt.file.Path != "" {
tt.file.Path = types.FilePath(fmt.Sprintf("%s_%d_%d", tt.file.Path, i, time.Now().UnixNano()))
tt.file.Path = types.FilePath(fmt.Sprintf("%s_%d_%d",
tt.file.Path, i, time.Now().UnixNano()))
}
err := repo.Create(ctx, nil, tt.file)
@@ -106,65 +119,128 @@ func TestFileRepositoryEdgeCases(t *testing.T) {
}
}
// testDuplicateFilePaths exercises the UPSERT behavior for duplicate paths.
func testDuplicateFilePaths(t *testing.T, repos *Repositories) {
t.Helper()
ctx := context.Background()
file1 := &File{
Path: "/duplicate.txt",
MTime: time.Now(),
Size: 1024,
Mode: 0644,
UID: 1000,
GID: 1000,
}
file2 := &File{
Path: "/duplicate.txt", // Same path
MTime: time.Now().Add(time.Hour),
Size: 2048,
Mode: 0644,
UID: 1000,
GID: 1000,
}
err := repos.Files.Create(ctx, nil, file1)
if err != nil {
t.Fatalf("failed to create file1: %v", err)
}
originalID := file1.ID
// Create with same path should update the existing record (UPSERT behavior)
err = repos.Files.Create(ctx, nil, file2)
if err != nil {
t.Fatalf("failed to create file2: %v", err)
}
// Verify the file was updated, not duplicated
retrievedFile, err := repos.Files.GetByPath(ctx, "/duplicate.txt")
if err != nil {
t.Fatalf("failed to retrieve file: %v", err)
}
// The file should have been updated with file2's data
if retrievedFile.Size != 2048 {
t.Errorf("expected size 2048, got %d", retrievedFile.Size)
}
// ID might be different due to the UPSERT
if retrievedFile.ID != file2.ID {
t.Logf("File ID changed from %s to %s during upsert",
originalID, retrievedFile.ID)
}
}
// testDuplicateFileChunks exercises idempotent file-chunk mapping creation.
func testDuplicateFileChunks(t *testing.T, repos *Repositories) {
t.Helper()
ctx := context.Background()
file := &File{
Path: "/test-dup-fc.txt",
MTime: time.Now(),
Size: 1024,
Mode: 0644,
UID: 1000,
GID: 1000,
}
err := repos.Files.Create(ctx, nil, file)
if err != nil {
t.Fatal(err)
}
chunk := &Chunk{
ChunkHash: types.ChunkHash("test-chunk-dup"),
Size: 1024,
}
err = repos.Chunks.Create(ctx, nil, chunk)
if err != nil {
t.Fatal(err)
}
fc := &FileChunk{
FileID: file.ID,
Idx: 0,
ChunkHash: chunk.ChunkHash,
}
err = repos.FileChunks.Create(ctx, nil, fc)
if err != nil {
t.Fatal(err)
}
// Creating the same mapping again should be idempotent
err = repos.FileChunks.Create(ctx, nil, fc)
if err != nil {
t.Error("file-chunk creation should be idempotent")
}
}
// TestDuplicateHandling tests handling of duplicate entries
func TestDuplicateHandling(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
t.Cleanup(cleanup)
ctx := context.Background()
repos := NewRepositories(db)
// Test duplicate file paths - Create uses UPSERT logic
t.Run("duplicate file paths", func(t *testing.T) {
file1 := &File{
Path: "/duplicate.txt",
MTime: time.Now(),
Size: 1024,
Mode: 0644,
UID: 1000,
GID: 1000,
}
file2 := &File{
Path: "/duplicate.txt", // Same path
MTime: time.Now().Add(time.Hour),
Size: 2048,
Mode: 0644,
UID: 1000,
GID: 1000,
}
err := repos.Files.Create(ctx, nil, file1)
if err != nil {
t.Fatalf("failed to create file1: %v", err)
}
originalID := file1.ID
// Create with same path should update the existing record (UPSERT behavior)
err = repos.Files.Create(ctx, nil, file2)
if err != nil {
t.Fatalf("failed to create file2: %v", err)
}
// Verify the file was updated, not duplicated
retrievedFile, err := repos.Files.GetByPath(ctx, "/duplicate.txt")
if err != nil {
t.Fatalf("failed to retrieve file: %v", err)
}
// The file should have been updated with file2's data
if retrievedFile.Size != 2048 {
t.Errorf("expected size 2048, got %d", retrievedFile.Size)
}
// ID might be different due to the UPSERT
if retrievedFile.ID != file2.ID {
t.Logf("File ID changed from %s to %s during upsert", originalID, retrievedFile.ID)
}
t.Parallel()
testDuplicateFilePaths(t, repos)
})
// Test duplicate chunk hashes
t.Run("duplicate chunk hashes", func(t *testing.T) {
t.Parallel()
chunk := &Chunk{
ChunkHash: types.ChunkHash("duplicate-chunk"),
Size: 1024,
@@ -184,59 +260,25 @@ func TestDuplicateHandling(t *testing.T) {
// Test duplicate file-chunk mappings
t.Run("duplicate file-chunk mappings", func(t *testing.T) {
file := &File{
Path: "/test-dup-fc.txt",
MTime: time.Now(),
Size: 1024,
Mode: 0644,
UID: 1000,
GID: 1000,
}
err := repos.Files.Create(ctx, nil, file)
if err != nil {
t.Fatal(err)
}
chunk := &Chunk{
ChunkHash: types.ChunkHash("test-chunk-dup"),
Size: 1024,
}
err = repos.Chunks.Create(ctx, nil, chunk)
if err != nil {
t.Fatal(err)
}
fc := &FileChunk{
FileID: file.ID,
Idx: 0,
ChunkHash: chunk.ChunkHash,
}
err = repos.FileChunks.Create(ctx, nil, fc)
if err != nil {
t.Fatal(err)
}
// Creating the same mapping again should be idempotent
err = repos.FileChunks.Create(ctx, nil, fc)
if err != nil {
t.Error("file-chunk creation should be idempotent")
}
t.Parallel()
testDuplicateFileChunks(t, repos)
})
}
// TestNullHandling tests handling of NULL values
func TestNullHandling(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
t.Cleanup(cleanup)
ctx := context.Background()
repos := NewRepositories(db)
// Test file with no link target
t.Run("file without link target", func(t *testing.T) {
t.Parallel()
file := &File{
Path: "/regular.txt",
MTime: time.Now(),
@@ -264,9 +306,11 @@ func TestNullHandling(t *testing.T) {
// Test snapshot with NULL completed_at
t.Run("incomplete snapshot", func(t *testing.T) {
t.Parallel()
snapshot := &Snapshot{
ID: "incomplete-test",
Hostname: "test-host",
Hostname: internalTestHost,
StartedAt: time.Now(),
CompletedAt: nil, // Should remain NULL until completed
}
@@ -288,31 +332,86 @@ func TestNullHandling(t *testing.T) {
// Test blob with NULL uploaded_ts
t.Run("blob not uploaded", func(t *testing.T) {
blob := &Blob{
ID: types.NewBlobID(),
Hash: types.BlobHash("test-hash"),
CreatedTS: time.Now(),
UploadedTS: nil, // Not uploaded yet
}
err := repos.Blobs.Create(ctx, nil, blob)
if err != nil {
t.Fatal(err)
}
retrieved, err := repos.Blobs.GetByID(ctx, blob.ID.String())
if err != nil {
t.Fatal(err)
}
if retrieved.UploadedTS != nil {
t.Error("expected nil UploadedTS for non-uploaded blob")
}
t.Parallel()
verifyBlobNullUploadTS(ctx, t, repos)
})
}
// verifyBlobNullUploadTS checks that a blob created without an upload
// timestamp round-trips with UploadedTS nil.
func verifyBlobNullUploadTS(
ctx context.Context, t *testing.T, repos *Repositories,
) {
t.Helper()
blob := &Blob{
ID: types.NewBlobID(),
Hash: types.BlobHash("test-hash"),
CreatedTS: time.Now(),
UploadedTS: nil, // Not uploaded yet
}
err := repos.Blobs.Create(ctx, nil, blob)
if err != nil {
t.Fatal(err)
}
retrieved, err := repos.Blobs.GetByID(ctx, blob.ID.String())
if err != nil {
t.Fatal(err)
}
if retrieved.UploadedTS != nil {
t.Error("expected nil UploadedTS for non-uploaded blob")
}
}
// createLargeDatasetFiles creates fileCount files and adds every other
// one to the snapshot.
func createLargeDatasetFiles(
t *testing.T,
repos *Repositories,
snapshotID string,
fileCount int,
) {
t.Helper()
ctx := context.Background()
start := time.Now()
for i := range fileCount {
file := &File{
Path: types.FilePath(fmt.Sprintf("/large/file%05d.txt", i)),
MTime: time.Now(),
Size: int64(i * 1024),
Mode: 0644,
UID: uint32(1000 + (i % 10)),
GID: uint32(1000 + (i % 10)),
}
err := repos.Files.Create(ctx, nil, file)
if err != nil {
t.Fatalf("failed to create file %d: %v", i, err)
}
// Add half to snapshot
if i%2 == 0 {
err = repos.Snapshots.AddFileByID(ctx, nil, snapshotID, file.ID)
if err != nil {
t.Fatal(err)
}
}
}
t.Logf("Created %d files in %v", fileCount, time.Since(start))
}
// TestLargeDatasets tests operations with large amounts of data
//
//nolint:tparallel // subtests share one database and are order-dependent
func TestLargeDatasets(t *testing.T) {
t.Parallel()
if testing.Short() {
t.Skip("skipping large dataset test in short mode")
}
@@ -326,7 +425,7 @@ func TestLargeDatasets(t *testing.T) {
// Create a snapshot
snapshot := &Snapshot{
ID: "large-dataset-test",
Hostname: "test-host",
Hostname: internalTestHost,
StartedAt: time.Now(),
}
@@ -338,40 +437,13 @@ func TestLargeDatasets(t *testing.T) {
// Create many files
const fileCount = 1000
fileIDs := make([]types.FileID, fileCount)
//nolint:paralleltest // phases share one database and are order-dependent
t.Run("create many files", func(t *testing.T) {
start := time.Now()
for i := range fileCount {
file := &File{
Path: types.FilePath(fmt.Sprintf("/large/file%05d.txt", i)),
MTime: time.Now(),
Size: int64(i * 1024),
Mode: 0644,
UID: uint32(1000 + (i % 10)),
GID: uint32(1000 + (i % 10)),
}
err := repos.Files.Create(ctx, nil, file)
if err != nil {
t.Fatalf("failed to create file %d: %v", i, err)
}
fileIDs[i] = file.ID
// Add half to snapshot
if i%2 == 0 {
err = repos.Snapshots.AddFileByID(ctx, nil, snapshot.ID.String(), file.ID)
if err != nil {
t.Fatal(err)
}
}
}
t.Logf("Created %d files in %v", fileCount, time.Since(start))
createLargeDatasetFiles(t, repos, snapshot.ID.String(), fileCount)
})
// Test ListByPrefix performance
//nolint:paralleltest // phases share one database and are order-dependent
t.Run("list by prefix performance", func(t *testing.T) {
start := time.Now()
@@ -388,6 +460,7 @@ func TestLargeDatasets(t *testing.T) {
})
// Test orphaned cleanup performance
//nolint:paralleltest // phases share one database and are order-dependent
t.Run("orphaned cleanup performance", func(t *testing.T) {
start := time.Now()
@@ -405,21 +478,26 @@ func TestLargeDatasets(t *testing.T) {
}
if len(files) != fileCount/2 {
t.Errorf("expected %d files after cleanup, got %d", fileCount/2, len(files))
t.Errorf("expected %d files after cleanup, got %d",
fileCount/2, len(files))
}
})
}
// TestErrorPropagation tests that errors are properly propagated
func TestErrorPropagation(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
t.Cleanup(cleanup)
ctx := context.Background()
repos := NewRepositories(db)
// Test GetByID with non-existent ID
t.Run("GetByID non-existent", func(t *testing.T) {
t.Parallel()
file, err := repos.Files.GetByID(ctx, types.NewFileID())
if err != nil {
t.Errorf("GetByID should not return error for non-existent ID, got: %v", err)
@@ -432,9 +510,12 @@ func TestErrorPropagation(t *testing.T) {
// Test GetByPath with non-existent path
t.Run("GetByPath non-existent", func(t *testing.T) {
t.Parallel()
file, err := repos.Files.GetByPath(ctx, "/non/existent/path.txt")
if err != nil {
t.Errorf("GetByPath should not return error for non-existent path, got: %v", err)
t.Errorf("GetByPath should not return error for non-existent path, got: %v",
err)
}
if file != nil {
@@ -444,6 +525,8 @@ func TestErrorPropagation(t *testing.T) {
// Test invalid foreign key reference
t.Run("invalid foreign key", func(t *testing.T) {
t.Parallel()
fc := &FileChunk{
FileID: types.NewFileID(),
Idx: 0,
@@ -463,8 +546,10 @@ func TestErrorPropagation(t *testing.T) {
// TestQueryInjection tests that the system is safe from SQL injection
func TestQueryInjection(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
t.Cleanup(cleanup)
ctx := context.Background()
repos := NewRepositories(db)
@@ -479,6 +564,8 @@ func TestQueryInjection(t *testing.T) {
for _, injection := range injectionTests {
t.Run("injection attempt", func(t *testing.T) {
t.Parallel()
// Try injection in file path
file := &File{
Path: types.FilePath(injection),
@@ -495,7 +582,7 @@ func TestQueryInjection(t *testing.T) {
// Verify tables still exist
var count int
err := db.conn.QueryRow("SELECT COUNT(*) FROM files").Scan(&count)
err := db.conn.QueryRowContext(ctx, countFilesQuery).Scan(&count)
if err != nil {
t.Fatal("files table was damaged by injection")
}
@@ -505,6 +592,8 @@ func TestQueryInjection(t *testing.T) {
// TestTimezoneHandling tests that times are properly handled in UTC
func TestTimezoneHandling(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()

354
internal/database/snapshots.go
View File

@@ -11,19 +11,27 @@ import (
"sneak.berlin/go/vaultik/internal/types"
)
// SnapshotRepository provides access to the snapshots table and its
// snapshot_files / snapshot_blobs association tables.
type SnapshotRepository struct {
db *DB
}
// NewSnapshotRepository creates a SnapshotRepository backed by db.
func NewSnapshotRepository(db *DB) *SnapshotRepository {
return &SnapshotRepository{db: db}
}
func (r *SnapshotRepository) Create(ctx context.Context, tx *sql.Tx, snapshot *Snapshot) error {
// Create inserts a snapshot row, using tx when non-nil.
func (r *SnapshotRepository) Create(
ctx context.Context, tx *sql.Tx, snapshot *Snapshot,
) error {
query := `
INSERT INTO snapshots (id, hostname, vaultik_version, vaultik_git_revision, started_at, completed_at,
file_count, chunk_count, blob_count, total_size, blob_size, blob_uncompressed_size,
compression_ratio, compression_level, upload_bytes, upload_duration_ms)
INSERT INTO snapshots (id, hostname, vaultik_version,
vaultik_git_revision, started_at, completed_at,
file_count, chunk_count, blob_count, total_size, blob_size,
blob_uncompressed_size, compression_ratio, compression_level,
upload_bytes, upload_duration_ms)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`
@@ -34,15 +42,21 @@ func (r *SnapshotRepository) Create(ctx context.Context, tx *sql.Tx, snapshot *S
completedAt = &ts
}
args := []any{
snapshot.ID, snapshot.Hostname, snapshot.VaultikVersion,
snapshot.VaultikGitRevision, snapshot.StartedAt.Unix(),
completedAt, snapshot.FileCount, snapshot.ChunkCount,
snapshot.BlobCount, snapshot.TotalSize, snapshot.BlobSize,
snapshot.BlobUncompressedSize, snapshot.CompressionRatio,
snapshot.CompressionLevel, snapshot.UploadBytes,
snapshot.UploadDurationMs,
}
var err error
if tx != nil {
_, err = tx.ExecContext(ctx, query, snapshot.ID, snapshot.Hostname, snapshot.VaultikVersion, snapshot.VaultikGitRevision, snapshot.StartedAt.Unix(),
completedAt, snapshot.FileCount, snapshot.ChunkCount, snapshot.BlobCount, snapshot.TotalSize, snapshot.BlobSize, snapshot.BlobUncompressedSize,
snapshot.CompressionRatio, snapshot.CompressionLevel, snapshot.UploadBytes, snapshot.UploadDurationMs)
_, err = tx.ExecContext(ctx, query, args...)
} else {
_, err = r.db.ExecWithLog(ctx, query, snapshot.ID, snapshot.Hostname, snapshot.VaultikVersion, snapshot.VaultikGitRevision, snapshot.StartedAt.Unix(),
completedAt, snapshot.FileCount, snapshot.ChunkCount, snapshot.BlobCount, snapshot.TotalSize, snapshot.BlobSize, snapshot.BlobUncompressedSize,
snapshot.CompressionRatio, snapshot.CompressionLevel, snapshot.UploadBytes, snapshot.UploadDurationMs)
_, err = r.db.ExecWithLog(ctx, query, args...)
}
if err != nil {
@@ -52,7 +66,14 @@ func (r *SnapshotRepository) Create(ctx context.Context, tx *sql.Tx, snapshot *S
return nil
}
func (r *SnapshotRepository) UpdateCounts(ctx context.Context, tx *sql.Tx, snapshotID string, fileCount, chunkCount, blobCount, totalSize, blobSize int64) error {
// UpdateCounts updates a snapshot's file/chunk/blob counters and sizes,
// recomputing the compression ratio, using tx when non-nil.
func (r *SnapshotRepository) UpdateCounts(
ctx context.Context,
tx *sql.Tx,
snapshotID string,
fileCount, chunkCount, blobCount, totalSize, blobSize int64,
) error {
compressionRatio := 1.0
if totalSize > 0 {
compressionRatio = float64(blobSize) / float64(totalSize)
@@ -71,9 +92,13 @@ func (r *SnapshotRepository) UpdateCounts(ctx context.Context, tx *sql.Tx, snaps
var err error
if tx != nil {
_, err = tx.ExecContext(ctx, query, fileCount, chunkCount, blobCount, totalSize, blobSize, compressionRatio, snapshotID)
_, err = tx.ExecContext(ctx, query,
fileCount, chunkCount, blobCount, totalSize, blobSize,
compressionRatio, snapshotID)
} else {
_, err = r.db.ExecWithLog(ctx, query, fileCount, chunkCount, blobCount, totalSize, blobSize, compressionRatio, snapshotID)
_, err = r.db.ExecWithLog(ctx, query,
fileCount, chunkCount, blobCount, totalSize, blobSize,
compressionRatio, snapshotID)
}
if err != nil {
@@ -84,34 +109,23 @@ func (r *SnapshotRepository) UpdateCounts(ctx context.Context, tx *sql.Tx, snaps
}
// UpdateExtendedStats updates extended statistics for a snapshot
func (r *SnapshotRepository) UpdateExtendedStats(ctx context.Context, tx *sql.Tx, snapshotID string, blobUncompressedSize int64, compressionLevel int, uploadDurationMs int64) error {
// Calculate compression ratio based on uncompressed vs compressed sizes
var compressionRatio float64
if blobUncompressedSize > 0 {
// Get current blob_size from DB to calculate ratio
var blobSize int64
queryGet := `SELECT blob_size FROM snapshots WHERE id = ?`
if tx != nil {
err := tx.QueryRowContext(ctx, queryGet, snapshotID).Scan(&blobSize)
if err != nil {
return fmt.Errorf("getting blob size: %w", err)
}
} else {
err := r.db.conn.QueryRowContext(ctx, queryGet, snapshotID).Scan(&blobSize)
if err != nil {
return fmt.Errorf("getting blob size: %w", err)
}
}
compressionRatio = float64(blobSize) / float64(blobUncompressedSize)
} else {
compressionRatio = 1.0
func (r *SnapshotRepository) UpdateExtendedStats(
ctx context.Context,
tx *sql.Tx,
snapshotID string,
blobUncompressedSize int64,
compressionLevel int,
uploadDurationMs int64,
) error {
compressionRatio, err := r.extendedCompressionRatio(
ctx, tx, snapshotID, blobUncompressedSize,
)
if err != nil {
return err
}
query := `
UPDATE snapshots
UPDATE snapshots
SET blob_uncompressed_size = ?,
compression_ratio = ?,
compression_level = ?,
@@ -120,11 +134,14 @@ func (r *SnapshotRepository) UpdateExtendedStats(ctx context.Context, tx *sql.Tx
WHERE id = ?
`
var err error
if tx != nil {
_, err = tx.ExecContext(ctx, query, blobUncompressedSize, compressionRatio, compressionLevel, uploadDurationMs, snapshotID)
_, err = tx.ExecContext(ctx, query,
blobUncompressedSize, compressionRatio, compressionLevel,
uploadDurationMs, snapshotID)
} else {
_, err = r.db.ExecWithLog(ctx, query, blobUncompressedSize, compressionRatio, compressionLevel, uploadDurationMs, snapshotID)
_, err = r.db.ExecWithLog(ctx, query,
blobUncompressedSize, compressionRatio, compressionLevel,
uploadDurationMs, snapshotID)
}
if err != nil {
@@ -134,7 +151,11 @@ func (r *SnapshotRepository) UpdateExtendedStats(ctx context.Context, tx *sql.Tx
return nil
}
func (r *SnapshotRepository) GetByID(ctx context.Context, snapshotID string) (*Snapshot, error) {
// GetByID returns the snapshot with the given ID, or nil if no such
// snapshot exists.
func (r *SnapshotRepository) GetByID(
ctx context.Context, snapshotID string,
) (*Snapshot, error) {
query := `
SELECT id, hostname, vaultik_version, vaultik_git_revision, started_at, completed_at,
file_count, chunk_count, blob_count, total_size, blob_size, blob_uncompressed_size,
@@ -169,7 +190,7 @@ func (r *SnapshotRepository) GetByID(ctx context.Context, snapshotID string) (*S
)
if errors.Is(err, sql.ErrNoRows) {
return nil, nil
return nil, nil //nolint:nilnil // nil,nil signals not-found; callers check nil
}
if err != nil {
@@ -185,9 +206,14 @@ func (r *SnapshotRepository) GetByID(ctx context.Context, snapshotID string) (*S
return &snapshot, nil
}
func (r *SnapshotRepository) ListRecent(ctx context.Context, limit int) ([]*Snapshot, error) {
// ListRecent returns up to limit snapshots, most recently started first.
func (r *SnapshotRepository) ListRecent(
ctx context.Context, limit int,
) ([]*Snapshot, error) {
query := `
SELECT id, hostname, vaultik_version, vaultik_git_revision, started_at, completed_at, file_count, chunk_count, blob_count, total_size, blob_size, compression_ratio
SELECT id, hostname, vaultik_version, vaultik_git_revision,
started_at, completed_at, file_count, chunk_count, blob_count,
total_size, blob_size, compression_ratio
FROM snapshots
ORDER BY started_at DESC
LIMIT ?
@@ -199,47 +225,13 @@ func (r *SnapshotRepository) ListRecent(ctx context.Context, limit int) ([]*Snap
}
defer CloseRows(rows)
var snapshots []*Snapshot
for rows.Next() {
var (
snapshot Snapshot
startedAtUnix int64
completedAtUnix *int64
)
err := rows.Scan(
&snapshot.ID,
&snapshot.Hostname,
&snapshot.VaultikVersion,
&snapshot.VaultikGitRevision,
&startedAtUnix,
&completedAtUnix,
&snapshot.FileCount,
&snapshot.ChunkCount,
&snapshot.BlobCount,
&snapshot.TotalSize,
&snapshot.BlobSize,
&snapshot.CompressionRatio,
)
if err != nil {
return nil, fmt.Errorf("scanning snapshot: %w", err)
}
snapshot.StartedAt = time.Unix(startedAtUnix, 0)
if completedAtUnix != nil {
t := time.Unix(*completedAtUnix, 0)
snapshot.CompletedAt = &t
}
snapshots = append(snapshots, &snapshot)
}
return snapshots, rows.Err()
return r.scanSnapshotRows(rows)
}
// MarkComplete marks a snapshot as completed with the current timestamp
func (r *SnapshotRepository) MarkComplete(ctx context.Context, tx *sql.Tx, snapshotID string) error {
func (r *SnapshotRepository) MarkComplete(
ctx context.Context, tx *sql.Tx, snapshotID string,
) error {
query := `
UPDATE snapshots
SET completed_at = ?
@@ -263,7 +255,9 @@ func (r *SnapshotRepository) MarkComplete(ctx context.Context, tx *sql.Tx, snaps
}
// AddFile adds a file to a snapshot
func (r *SnapshotRepository) AddFile(ctx context.Context, tx *sql.Tx, snapshotID string, filePath string) error {
func (r *SnapshotRepository) AddFile(
ctx context.Context, tx *sql.Tx, snapshotID string, filePath string,
) error {
query := `
INSERT OR IGNORE INTO snapshot_files (snapshot_id, file_id)
SELECT ?, id FROM files WHERE path = ?
@@ -284,7 +278,9 @@ func (r *SnapshotRepository) AddFile(ctx context.Context, tx *sql.Tx, snapshotID
}
// AddFileByID adds a file to a snapshot by file ID
func (r *SnapshotRepository) AddFileByID(ctx context.Context, tx *sql.Tx, snapshotID string, fileID types.FileID) error {
func (r *SnapshotRepository) AddFileByID(
ctx context.Context, tx *sql.Tx, snapshotID string, fileID types.FileID,
) error {
query := `
INSERT OR IGNORE INTO snapshot_files (snapshot_id, file_id)
VALUES (?, ?)
@@ -305,12 +301,17 @@ func (r *SnapshotRepository) AddFileByID(ctx context.Context, tx *sql.Tx, snapsh
}
// AddFilesByIDBatch adds multiple files to a snapshot in batched inserts
func (r *SnapshotRepository) AddFilesByIDBatch(ctx context.Context, tx *sql.Tx, snapshotID string, fileIDs []types.FileID) error {
func (r *SnapshotRepository) AddFilesByIDBatch(
ctx context.Context, tx *sql.Tx, snapshotID string, fileIDs []types.FileID,
) error {
if len(fileIDs) == 0 {
return nil
}
// Each entry has 2 values, so batch at 400 to be safe
// Each snapshot_files row binds this many SQL variables.
const snapshotFileCols = 2
// Batch at 400 rows to be safe with SQLite's variable limit.
const batchSize = 400
for i := 0; i < len(fileIDs); i += batchSize {
@@ -320,7 +321,7 @@ func (r *SnapshotRepository) AddFilesByIDBatch(ctx context.Context, tx *sql.Tx,
query := "INSERT OR IGNORE INTO snapshot_files (snapshot_id, file_id) VALUES "
args := make([]any, 0, len(batch)*2)
args := make([]any, 0, len(batch)*snapshotFileCols)
var querySb312 strings.Builder
@@ -334,7 +335,7 @@ func (r *SnapshotRepository) AddFilesByIDBatch(ctx context.Context, tx *sql.Tx,
args = append(args, snapshotID, fileID.String())
}
query += querySb312.String()
query += querySb312.String() //nolint:gosec // G202: appends "?" placeholders only
var err error
if tx != nil {
@@ -361,7 +362,9 @@ func (r *SnapshotRepository) AddFilesByIDBatch(ctx context.Context, tx *sql.Tx,
// Returns the number of rows inserted (i.e. blobs that were previously
// referenced indirectly via file_chunks but not yet recorded in
// snapshot_blobs for this snapshot).
func (r *SnapshotRepository) PopulateReferencedBlobs(ctx context.Context, tx *sql.Tx, snapshotID string) (int64, error) {
func (r *SnapshotRepository) PopulateReferencedBlobs(
ctx context.Context, tx *sql.Tx, snapshotID string,
) (int64, error) {
query := `
INSERT OR IGNORE INTO snapshot_blobs (snapshot_id, blob_id, blob_hash)
SELECT DISTINCT ?, blobs.id, blobs.blob_hash
@@ -393,7 +396,13 @@ func (r *SnapshotRepository) PopulateReferencedBlobs(ctx context.Context, tx *sq
}
// AddBlob adds a blob to a snapshot
func (r *SnapshotRepository) AddBlob(ctx context.Context, tx *sql.Tx, snapshotID string, blobID types.BlobID, blobHash types.BlobHash) error {
func (r *SnapshotRepository) AddBlob(
ctx context.Context,
tx *sql.Tx,
snapshotID string,
blobID types.BlobID,
blobHash types.BlobHash,
) error {
query := `
INSERT OR IGNORE INTO snapshot_blobs (snapshot_id, blob_id, blob_hash)
VALUES (?, ?, ?)
@@ -414,7 +423,9 @@ func (r *SnapshotRepository) AddBlob(ctx context.Context, tx *sql.Tx, snapshotID
}
// GetBlobHashes returns all blob hashes for a snapshot
func (r *SnapshotRepository) GetBlobHashes(ctx context.Context, snapshotID string) ([]string, error) {
func (r *SnapshotRepository) GetBlobHashes(
ctx context.Context, snapshotID string,
) ([]string, error) {
query := `
SELECT sb.blob_hash
FROM snapshot_blobs sb
@@ -444,8 +455,11 @@ func (r *SnapshotRepository) GetBlobHashes(ctx context.Context, snapshotID strin
return blobs, rows.Err()
}
// GetSnapshotTotalCompressedSize returns the total compressed size of all blobs referenced by a snapshot
func (r *SnapshotRepository) GetSnapshotTotalCompressedSize(ctx context.Context, snapshotID string) (int64, error) {
// GetSnapshotTotalCompressedSize returns the total compressed size of all
// blobs referenced by a snapshot.
func (r *SnapshotRepository) GetSnapshotTotalCompressedSize(
ctx context.Context, snapshotID string,
) (int64, error) {
query := `
SELECT COALESCE(SUM(b.compressed_size), 0)
FROM snapshot_blobs sb
@@ -465,7 +479,9 @@ func (r *SnapshotRepository) GetSnapshotTotalCompressedSize(ctx context.Context,
// GetSnapshotUncompressedChunkSize returns the sum of plaintext sizes of all unique
// chunks referenced by a snapshot (via snapshot_files → file_chunks → chunks).
func (r *SnapshotRepository) GetSnapshotUncompressedChunkSize(ctx context.Context, snapshotID string) (int64, error) {
func (r *SnapshotRepository) GetSnapshotUncompressedChunkSize(
ctx context.Context, snapshotID string,
) (int64, error) {
query := `
SELECT COALESCE(SUM(c.size), 0)
FROM (
@@ -491,7 +507,9 @@ func (r *SnapshotRepository) GetSnapshotUncompressedChunkSize(ctx context.Contex
// referenced by this snapshot but not by any earlier completed snapshot known to
// the local database. The result is the marginal uncompressed data this snapshot
// added to the dedup pool — i.e., the delta from prior snapshots.
func (r *SnapshotRepository) GetSnapshotNewChunkSize(ctx context.Context, snapshotID string) (int64, error) {
func (r *SnapshotRepository) GetSnapshotNewChunkSize(
ctx context.Context, snapshotID string,
) (int64, error) {
query := `
WITH this_snap_chunks AS (
SELECT DISTINCT fc.chunk_hash
@@ -516,7 +534,9 @@ func (r *SnapshotRepository) GetSnapshotNewChunkSize(ctx context.Context, snapsh
var totalSize int64
err := r.db.conn.QueryRowContext(ctx, query, snapshotID, snapshotID, snapshotID).Scan(&totalSize)
err := r.db.conn.QueryRowContext(
ctx, query, snapshotID, snapshotID, snapshotID,
).Scan(&totalSize)
if err != nil {
return 0, fmt.Errorf("querying new chunk size: %w", err)
}
@@ -525,9 +545,13 @@ func (r *SnapshotRepository) GetSnapshotNewChunkSize(ctx context.Context, snapsh
}
// GetIncompleteSnapshots returns all snapshots that haven't been completed
func (r *SnapshotRepository) GetIncompleteSnapshots(ctx context.Context) ([]*Snapshot, error) {
func (r *SnapshotRepository) GetIncompleteSnapshots(
ctx context.Context,
) ([]*Snapshot, error) {
query := `
SELECT id, hostname, vaultik_version, vaultik_git_revision, started_at, completed_at, file_count, chunk_count, blob_count, total_size, blob_size, compression_ratio
SELECT id, hostname, vaultik_version, vaultik_git_revision,
started_at, completed_at, file_count, chunk_count, blob_count,
total_size, blob_size, compression_ratio
FROM snapshots
WHERE completed_at IS NULL
ORDER BY started_at DESC
@@ -539,49 +563,17 @@ func (r *SnapshotRepository) GetIncompleteSnapshots(ctx context.Context) ([]*Sna
}
defer CloseRows(rows)
var snapshots []*Snapshot
for rows.Next() {
var (
snapshot Snapshot
startedAtUnix int64
completedAtUnix *int64
)
err := rows.Scan(
&snapshot.ID,
&snapshot.Hostname,
&snapshot.VaultikVersion,
&snapshot.VaultikGitRevision,
&startedAtUnix,
&completedAtUnix,
&snapshot.FileCount,
&snapshot.ChunkCount,
&snapshot.BlobCount,
&snapshot.TotalSize,
&snapshot.BlobSize,
&snapshot.CompressionRatio,
)
if err != nil {
return nil, fmt.Errorf("scanning snapshot: %w", err)
}
snapshot.StartedAt = time.Unix(startedAtUnix, 0)
if completedAtUnix != nil {
t := time.Unix(*completedAtUnix, 0)
snapshot.CompletedAt = &t
}
snapshots = append(snapshots, &snapshot)
}
return snapshots, rows.Err()
return r.scanSnapshotRows(rows)
}
// GetIncompleteByHostname returns all incomplete snapshots for a specific hostname
func (r *SnapshotRepository) GetIncompleteByHostname(ctx context.Context, hostname string) ([]*Snapshot, error) {
func (r *SnapshotRepository) GetIncompleteByHostname(
ctx context.Context, hostname string,
) ([]*Snapshot, error) {
query := `
SELECT id, hostname, vaultik_version, vaultik_git_revision, started_at, completed_at, file_count, chunk_count, blob_count, total_size, blob_size, compression_ratio
SELECT id, hostname, vaultik_version, vaultik_git_revision,
started_at, completed_at, file_count, chunk_count, blob_count,
total_size, blob_size, compression_ratio
FROM snapshots
WHERE completed_at IS NULL AND hostname = ?
ORDER BY started_at DESC
@@ -645,7 +637,9 @@ func (r *SnapshotRepository) Delete(ctx context.Context, snapshotID string) erro
}
// DeleteSnapshotFiles removes all snapshot_files entries for a snapshot
func (r *SnapshotRepository) DeleteSnapshotFiles(ctx context.Context, snapshotID string) error {
func (r *SnapshotRepository) DeleteSnapshotFiles(
ctx context.Context, snapshotID string,
) error {
query := `DELETE FROM snapshot_files WHERE snapshot_id = ?`
_, err := r.db.ExecWithLog(ctx, query, snapshotID)
@@ -657,7 +651,9 @@ func (r *SnapshotRepository) DeleteSnapshotFiles(ctx context.Context, snapshotID
}
// DeleteSnapshotBlobs removes all snapshot_blobs entries for a snapshot
func (r *SnapshotRepository) DeleteSnapshotBlobs(ctx context.Context, snapshotID string) error {
func (r *SnapshotRepository) DeleteSnapshotBlobs(
ctx context.Context, snapshotID string,
) error {
query := `DELETE FROM snapshot_blobs WHERE snapshot_id = ?`
_, err := r.db.ExecWithLog(ctx, query, snapshotID)
@@ -669,7 +665,9 @@ func (r *SnapshotRepository) DeleteSnapshotBlobs(ctx context.Context, snapshotID
}
// DeleteSnapshotUploads removes all uploads entries for a snapshot
func (r *SnapshotRepository) DeleteSnapshotUploads(ctx context.Context, snapshotID string) error {
func (r *SnapshotRepository) DeleteSnapshotUploads(
ctx context.Context, snapshotID string,
) error {
query := `DELETE FROM uploads WHERE snapshot_id = ?`
_, err := r.db.ExecWithLog(ctx, query, snapshotID)
@@ -679,3 +677,77 @@ func (r *SnapshotRepository) DeleteSnapshotUploads(ctx context.Context, snapshot
return nil
}
// extendedCompressionRatio computes the compression ratio for a snapshot
// from its stored blob_size and the given uncompressed size. Returns 1.0
// when the uncompressed size is zero.
func (r *SnapshotRepository) extendedCompressionRatio(
ctx context.Context,
tx *sql.Tx,
snapshotID string,
blobUncompressedSize int64,
) (float64, error) {
if blobUncompressedSize <= 0 {
return 1.0, nil
}
// Get current blob_size from DB to calculate ratio
var blobSize int64
queryGet := `SELECT blob_size FROM snapshots WHERE id = ?`
var err error
if tx != nil {
err = tx.QueryRowContext(ctx, queryGet, snapshotID).Scan(&blobSize)
} else {
err = r.db.conn.QueryRowContext(ctx, queryGet, snapshotID).Scan(&blobSize)
}
if err != nil {
return 0, fmt.Errorf("getting blob size: %w", err)
}
return float64(blobSize) / float64(blobUncompressedSize), nil
}
// scanSnapshotRows scans the standard snapshot column set from a rows
// iterator into Snapshot records.
func (r *SnapshotRepository) scanSnapshotRows(rows *sql.Rows) ([]*Snapshot, error) {
var snapshots []*Snapshot
for rows.Next() {
var (
snapshot Snapshot
startedAtUnix int64
completedAtUnix *int64
)
err := rows.Scan(
&snapshot.ID,
&snapshot.Hostname,
&snapshot.VaultikVersion,
&snapshot.VaultikGitRevision,
&startedAtUnix,
&completedAtUnix,
&snapshot.FileCount,
&snapshot.ChunkCount,
&snapshot.BlobCount,
&snapshot.TotalSize,
&snapshot.BlobSize,
&snapshot.CompressionRatio,
)
if err != nil {
return nil, fmt.Errorf("scanning snapshot: %w", err)
}
snapshot.StartedAt = time.Unix(startedAtUnix, 0)
if completedAtUnix != nil {
t := time.Unix(*completedAtUnix, 0)
snapshot.CompletedAt = &t
}
snapshots = append(snapshots, &snapshot)
}
return snapshots, rows.Err()
}

103
internal/database/snapshots_test.go
View File

@@ -1,4 +1,4 @@
package database
package database_test
import (
"context"
@@ -7,6 +7,7 @@ import (
"testing"
"time"
"sneak.berlin/go/vaultik/internal/database"
"sneak.berlin/go/vaultik/internal/types"
)
@@ -21,17 +22,19 @@ const (
)
func TestSnapshotRepository(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repo := NewSnapshotRepository(db)
repo := database.NewSnapshotRepository(db)
// Test Create
snapshot := &Snapshot{
snapshot := &database.Snapshot{
ID: "2024-01-01T12:00:00Z",
Hostname: "test-host",
VaultikVersion: "1.0.0",
Hostname: testHostname,
VaultikVersion: testVersion,
StartedAt: time.Now().Truncate(time.Second),
CompletedAt: nil,
FileCount: 100,
@@ -62,20 +65,52 @@ func TestSnapshotRepository(t *testing.T) {
}
if retrieved.Hostname != snapshot.Hostname {
t.Errorf("hostname mismatch: got %s, want %s", retrieved.Hostname, snapshot.Hostname)
t.Errorf("hostname mismatch: got %s, want %s",
retrieved.Hostname, snapshot.Hostname)
}
if retrieved.FileCount != snapshot.FileCount {
t.Errorf("file count mismatch: got %d, want %d", retrieved.FileCount, snapshot.FileCount)
t.Errorf("file count mismatch: got %d, want %d",
retrieved.FileCount, snapshot.FileCount)
}
}
func TestSnapshotRepositoryUpdateCounts(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repo := database.NewSnapshotRepository(db)
snapshot := &database.Snapshot{
ID: "2024-01-02T12:00:00Z",
Hostname: testHostname,
VaultikVersion: testVersion,
StartedAt: time.Now().Truncate(time.Second),
CompletedAt: nil,
FileCount: 100,
ChunkCount: 500,
BlobCount: 10,
TotalSize: oneHundredMebibytes,
BlobSize: fortyMebibytes,
CompressionRatio: compressionRatioPoint4,
}
err := repo.Create(ctx, nil, snapshot)
if err != nil {
t.Fatalf("failed to create snapshot: %v", err)
}
// Test UpdateCounts
err = repo.UpdateCounts(ctx, nil, snapshot.ID.String(), 200, 1000, 20, twoHundredMebibytes, sixtyMebibytes)
err = repo.UpdateCounts(ctx, nil, snapshot.ID.String(),
200, 1000, 20, twoHundredMebibytes, sixtyMebibytes)
if err != nil {
t.Fatalf("failed to update counts: %v", err)
}
retrieved, err = repo.GetByID(ctx, snapshot.ID.String())
retrieved, err := repo.GetByID(ctx, snapshot.ID.String())
if err != nil {
t.Fatalf("failed to get updated snapshot: %v", err)
}
@@ -85,7 +120,8 @@ func TestSnapshotRepository(t *testing.T) {
}
if retrieved.ChunkCount != 1000 {
t.Errorf("chunk count not updated: got %d, want %d", retrieved.ChunkCount, 1000)
t.Errorf("chunk count not updated: got %d, want %d",
retrieved.ChunkCount, 1000)
}
if retrieved.BlobCount != 20 {
@@ -93,25 +129,37 @@ func TestSnapshotRepository(t *testing.T) {
}
if retrieved.TotalSize != twoHundredMebibytes {
t.Errorf("total size not updated: got %d, want %d", retrieved.TotalSize, twoHundredMebibytes)
t.Errorf("total size not updated: got %d, want %d",
retrieved.TotalSize, twoHundredMebibytes)
}
if retrieved.BlobSize != sixtyMebibytes {
t.Errorf("blob size not updated: got %d, want %d", retrieved.BlobSize, sixtyMebibytes)
t.Errorf("blob size not updated: got %d, want %d",
retrieved.BlobSize, sixtyMebibytes)
}
expectedRatio := compressionRatioPoint3 // 0.3
if math.Abs(retrieved.CompressionRatio-expectedRatio) > 0.001 {
t.Errorf("compression ratio not updated: got %f, want %f", retrieved.CompressionRatio, expectedRatio)
t.Errorf("compression ratio not updated: got %f, want %f",
retrieved.CompressionRatio, expectedRatio)
}
}
// Test ListRecent
// Add more snapshots
for i := 2; i <= 5; i++ {
s := &Snapshot{
func TestSnapshotRepositoryListRecent(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repo := database.NewSnapshotRepository(db)
// Add snapshots
for i := 1; i <= 5; i++ {
s := &database.Snapshot{
ID: types.SnapshotID(fmt.Sprintf("2024-01-0%dT12:00:00Z", i)),
Hostname: "test-host",
VaultikVersion: "1.0.0",
Hostname: testHostname,
VaultikVersion: testVersion,
StartedAt: time.Now().Add(time.Duration(i) * time.Hour).Truncate(time.Second),
CompletedAt: nil,
FileCount: int64(100 * i),
@@ -144,11 +192,13 @@ func TestSnapshotRepository(t *testing.T) {
}
func TestSnapshotRepositoryNotFound(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repo := NewSnapshotRepository(db)
repo := database.NewSnapshotRepository(db)
// Test GetByID with non-existent ID
snapshot, err := repo.GetByID(ctx, "nonexistent")
@@ -161,7 +211,8 @@ func TestSnapshotRepositoryNotFound(t *testing.T) {
}
// Test UpdateCounts on non-existent snapshot
err = repo.UpdateCounts(ctx, nil, "nonexistent", 100, 200, 10, oneHundredMebibytes, fortyMebibytes)
err = repo.UpdateCounts(ctx, nil, "nonexistent",
100, 200, 10, oneHundredMebibytes, fortyMebibytes)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@@ -169,16 +220,18 @@ func TestSnapshotRepositoryNotFound(t *testing.T) {
}
func TestSnapshotRepositoryDuplicate(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repo := NewSnapshotRepository(db)
repo := database.NewSnapshotRepository(db)
snapshot := &Snapshot{
snapshot := &database.Snapshot{
ID: "2024-01-01T12:00:00Z",
Hostname: "test-host",
VaultikVersion: "1.0.0",
Hostname: testHostname,
VaultikVersion: testVersion,
StartedAt: time.Now().Truncate(time.Second),
CompletedAt: nil,
FileCount: 100,

36
internal/database/uploads.go
View File

@@ -29,7 +29,9 @@ func NewUploadRepository(conn *sql.DB) *UploadRepository {
}
// Create inserts a new upload record
func (r *UploadRepository) Create(ctx context.Context, tx *sql.Tx, upload *Upload) error {
func (r *UploadRepository) Create(
ctx context.Context, tx *sql.Tx, upload *Upload,
) error {
query := `
INSERT INTO uploads (blob_hash, snapshot_id, uploaded_at, size, duration_ms)
VALUES (?, ?, ?, ?, ?)
@@ -37,16 +39,22 @@ func (r *UploadRepository) Create(ctx context.Context, tx *sql.Tx, upload *Uploa
var err error
if tx != nil {
_, err = tx.ExecContext(ctx, query, upload.BlobHash, upload.SnapshotID, upload.UploadedAt, upload.Size, upload.DurationMs)
_, err = tx.ExecContext(ctx, query,
upload.BlobHash, upload.SnapshotID, upload.UploadedAt,
upload.Size, upload.DurationMs)
} else {
_, err = r.conn.ExecContext(ctx, query, upload.BlobHash, upload.SnapshotID, upload.UploadedAt, upload.Size, upload.DurationMs)
_, err = r.conn.ExecContext(ctx, query,
upload.BlobHash, upload.SnapshotID, upload.UploadedAt,
upload.Size, upload.DurationMs)
}
return err
}
// GetByBlobHash retrieves an upload record by blob hash
func (r *UploadRepository) GetByBlobHash(ctx context.Context, blobHash string) (*Upload, error) {
func (r *UploadRepository) GetByBlobHash(
ctx context.Context, blobHash string,
) (*Upload, error) {
query := `
SELECT blob_hash, uploaded_at, size, duration_ms
FROM uploads
@@ -63,7 +71,7 @@ func (r *UploadRepository) GetByBlobHash(ctx context.Context, blobHash string) (
)
if errors.Is(err, sql.ErrNoRows) {
return nil, nil
return nil, nil //nolint:nilnil // nil,nil signals not-found; callers check nil
}
if err != nil {
@@ -74,7 +82,9 @@ func (r *UploadRepository) GetByBlobHash(ctx context.Context, blobHash string) (
}
// GetRecentUploads retrieves recent uploads ordered by upload time
func (r *UploadRepository) GetRecentUploads(ctx context.Context, limit int) ([]*Upload, error) {
func (r *UploadRepository) GetRecentUploads(
ctx context.Context, limit int,
) ([]*Upload, error) {
query := `
SELECT blob_hash, uploaded_at, size, duration_ms
FROM uploads
@@ -98,7 +108,9 @@ func (r *UploadRepository) GetRecentUploads(ctx context.Context, limit int) ([]*
for rows.Next() {
var upload Upload
err := rows.Scan(&upload.BlobHash, &upload.UploadedAt, &upload.Size, &upload.DurationMs)
err := rows.Scan(
&upload.BlobHash, &upload.UploadedAt, &upload.Size, &upload.DurationMs,
)
if err != nil {
return nil, err
}
@@ -110,9 +122,11 @@ func (r *UploadRepository) GetRecentUploads(ctx context.Context, limit int) ([]*
}
// GetUploadStats returns aggregate statistics for uploads
func (r *UploadRepository) GetUploadStats(ctx context.Context, since time.Time) (*UploadStats, error) {
func (r *UploadRepository) GetUploadStats(
ctx context.Context, since time.Time,
) (*UploadStats, error) {
query := `
SELECT
SELECT
COUNT(*) as count,
COALESCE(SUM(size), 0) as total_size,
COALESCE(AVG(duration_ms), 0) as avg_duration_ms,
@@ -145,7 +159,9 @@ type UploadStats struct {
}
// GetCountBySnapshot returns the count of uploads for a specific snapshot
func (r *UploadRepository) GetCountBySnapshot(ctx context.Context, snapshotID string) (int64, error) {
func (r *UploadRepository) GetCountBySnapshot(
ctx context.Context, snapshotID string,
) (int64, error) {
query := `SELECT COUNT(*) FROM uploads WHERE snapshot_id = ?`
var count int64