From 8529ae97358a006f0f75bcd76153fe6ccaf0de45 Mon Sep 17 00:00:00 2001 From: sneak Date: Sun, 20 Jul 2025 10:56:30 +0200 Subject: [PATCH] Implement SQLite index database layer - Add pure Go SQLite driver (modernc.org/sqlite) to avoid CGO dependency - Implement database connection management with WAL mode - Add write mutex for serializing concurrent writes - Create schema for all tables matching DESIGN.md specifications - Implement repository pattern for all database entities: - Files, FileChunks, Chunks, Blobs, BlobChunks, ChunkFiles, Snapshots - Add transaction support with proper rollback handling - Add fatal error handling for database integrity issues - Add snapshot fields for tracking file sizes and compression ratios - Make index path configurable via VAULTIK_INDEX_PATH environment variable - Add comprehensive test coverage for all repositories - Add format check to Makefile to ensure code formatting --- Makefile | 9 +- go.mod | 22 ++- go.sum | 60 +++++- internal/database/blob_chunks.go | 2 +- internal/database/blob_chunks_test.go | 146 +++++++++++++++ internal/database/blobs.go | 2 +- internal/database/blobs_test.go | 100 ++++++++++ internal/database/chunk_files.go | 2 +- internal/database/chunk_files_test.go | 142 ++++++++++++++ internal/database/chunks.go | 2 +- internal/database/chunks_test.go | 104 +++++++++++ internal/database/database.go | 37 +++- internal/database/database_test.go | 96 ++++++++++ internal/database/file_chunks.go | 4 +- internal/database/file_chunks_test.go | 119 ++++++++++++ internal/database/files.go | 4 +- internal/database/files_test.go | 191 +++++++++++++++++++ internal/database/models.go | 17 +- internal/database/repositories.go | 4 + internal/database/repositories_test.go | 247 +++++++++++++++++++++++++ internal/database/snapshots.go | 36 +++- internal/database/snapshots_test.go | 181 ++++++++++++++++++ 22 files changed, 1488 insertions(+), 39 deletions(-) create mode 100644 internal/database/blob_chunks_test.go create mode 100644 internal/database/blobs_test.go create mode 100644 internal/database/chunk_files_test.go create mode 100644 internal/database/chunks_test.go create mode 100644 internal/database/database_test.go create mode 100644 internal/database/file_chunks_test.go create mode 100644 internal/database/files_test.go create mode 100644 internal/database/repositories_test.go create mode 100644 internal/database/snapshots_test.go diff --git a/Makefile b/Makefile index 6f533de..59e5cb4 100644 --- a/Makefile +++ b/Makefile @@ -12,9 +12,16 @@ LDFLAGS := -X 'git.eeqj.de/sneak/vaultik/internal/globals.Version=$(VERSION)' \ all: test # Run tests -test: lint +test: lint fmt-check go test -v ./... +# Check if code is formatted +fmt-check: + @if [ -n "$$(go fmt ./...)" ]; then \ + echo "Error: Code is not formatted. Run 'make fmt' to fix."; \ + exit 1; \ + fi + # Format code fmt: go fmt ./... diff --git a/go.mod b/go.mod index c24abd7..4a620cf 100644 --- a/go.mod +++ b/go.mod @@ -3,14 +3,26 @@ module git.eeqj.de/sneak/vaultik go 1.24.4 require ( + github.com/spf13/cobra v1.9.1 + go.uber.org/fx v1.24.0 + gopkg.in/yaml.v3 v3.0.1 + modernc.org/sqlite v1.38.0 +) + +require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/mattn/go-sqlite3 v1.14.28 // indirect - github.com/spf13/cobra v1.9.1 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/ncruces/go-strftime v0.1.9 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/spf13/pflag v1.0.6 // indirect go.uber.org/dig v1.19.0 // indirect - go.uber.org/fx v1.24.0 // indirect go.uber.org/multierr v1.10.0 // indirect go.uber.org/zap v1.26.0 // indirect - golang.org/x/sys v0.0.0-20220412211240-33da011f77ad // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect + golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 // indirect + golang.org/x/sys v0.33.0 // indirect + modernc.org/libc v1.65.10 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.11.0 // indirect ) diff --git a/go.sum b/go.sum index b20294a..7c83a6b 100644 --- a/go.sum +++ b/go.sum @@ -1,23 +1,75 @@ github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/mattn/go-sqlite3 v1.14.28 h1:ThEiQrnbtumT+QMknw63Befp/ce/nUPgBPMlRFEum7A= -github.com/mattn/go-sqlite3 v1.14.28/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= +github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= go.uber.org/dig v1.19.0 h1:BACLhebsYdpQ7IROQ1AGPjrXcP5dF80U3gKoFzbaq/4= go.uber.org/dig v1.19.0/go.mod h1:Us0rSJiThwCv2GteUN0Q7OKvU7n5J4dxZ9JKUXozFdE= go.uber.org/fx v1.24.0 h1:wE8mruvpg2kiiL1Vqd0CC+tr0/24XIB10Iwp2lLWzkg= go.uber.org/fx v1.24.0/go.mod h1:AmDeGyS+ZARGKM4tlH4FY2Jr63VjbEDJHtqXTGP5hbo= +go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= +go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo= go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= -golang.org/x/sys v0.0.0-20220412211240-33da011f77ad h1:ntjMns5wyP/fN65tdBD4g8J5w8n015+iIIs9rtjXkY0= -golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 h1:R84qjqJb5nVJMxqWYb3np9L5ZsaDtB+a39EqjV0JSUM= +golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0/go.mod h1:S9Xr4PYopiDyqSyp5NjCrhFrqg6A5zA2E/iPHPhqnS8= +golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU= +golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= +golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ= +golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= +golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/tools v0.33.0 h1:4qz2S3zmRxbGIhDIAgjxvFutSvH5EfnsYrRBj0UI0bc= +golang.org/x/tools v0.33.0/go.mod h1:CIJMaWEY88juyUfo7UbgPqbC8rU2OqfAV1h2Qp0oMYI= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +modernc.org/cc/v4 v4.26.1 h1:+X5NtzVBn0KgsBCBe+xkDC7twLb/jNVj9FPgiwSQO3s= +modernc.org/cc/v4 v4.26.1/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0= +modernc.org/ccgo/v4 v4.28.0 h1:rjznn6WWehKq7dG4JtLRKxb52Ecv8OUGah8+Z/SfpNU= +modernc.org/ccgo/v4 v4.28.0/go.mod h1:JygV3+9AV6SmPhDasu4JgquwU81XAKLd3OKTUDNOiKE= +modernc.org/fileutil v1.3.3 h1:3qaU+7f7xxTUmvU1pJTZiDLAIoJVdUSSauJNHg9yXoA= +modernc.org/fileutil v1.3.3/go.mod h1:HxmghZSZVAz/LXcMNwZPA/DRrQZEVP9VX0V4LQGQFOc= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/libc v1.65.10 h1:ZwEk8+jhW7qBjHIT+wd0d9VjitRyQef9BnzlzGwMODc= +modernc.org/libc v1.65.10/go.mod h1:StFvYpx7i/mXtBAfVOjaU0PWZOvIRoZSgXhrwXzr8Po= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8= +modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.38.0 h1:+4OrfPQ8pxHKuWG4md1JpR/EYAh3Md7TdejuuzE7EUI= +modernc.org/sqlite v1.38.0/go.mod h1:1Bj+yES4SVvBZ4cBOpVZ6QgesMCKpJZDq0nxYzOpmNE= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/internal/database/blob_chunks.go b/internal/database/blob_chunks.go index c519e9c..89ada71 100644 --- a/internal/database/blob_chunks.go +++ b/internal/database/blob_chunks.go @@ -24,7 +24,7 @@ func (r *BlobChunkRepository) Create(ctx context.Context, tx *sql.Tx, bc *BlobCh if tx != nil { _, err = tx.ExecContext(ctx, query, bc.BlobHash, bc.ChunkHash, bc.Offset, bc.Length) } else { - _, err = r.db.conn.ExecContext(ctx, query, bc.BlobHash, bc.ChunkHash, bc.Offset, bc.Length) + _, err = r.db.ExecWithLock(ctx, query, bc.BlobHash, bc.ChunkHash, bc.Offset, bc.Length) } if err != nil { diff --git a/internal/database/blob_chunks_test.go b/internal/database/blob_chunks_test.go new file mode 100644 index 0000000..756bcae --- /dev/null +++ b/internal/database/blob_chunks_test.go @@ -0,0 +1,146 @@ +package database + +import ( + "context" + "testing" +) + +func TestBlobChunkRepository(t *testing.T) { + db, cleanup := setupTestDB(t) + defer cleanup() + + ctx := context.Background() + repo := NewBlobChunkRepository(db) + + // Test Create + bc1 := &BlobChunk{ + BlobHash: "blob1", + ChunkHash: "chunk1", + Offset: 0, + Length: 1024, + } + + err := repo.Create(ctx, nil, bc1) + if err != nil { + t.Fatalf("failed to create blob chunk: %v", err) + } + + // Add more chunks to the same blob + bc2 := &BlobChunk{ + BlobHash: "blob1", + ChunkHash: "chunk2", + Offset: 1024, + Length: 2048, + } + err = repo.Create(ctx, nil, bc2) + if err != nil { + t.Fatalf("failed to create second blob chunk: %v", err) + } + + bc3 := &BlobChunk{ + BlobHash: "blob1", + ChunkHash: "chunk3", + Offset: 3072, + Length: 512, + } + err = repo.Create(ctx, nil, bc3) + if err != nil { + t.Fatalf("failed to create third blob chunk: %v", err) + } + + // Test GetByBlobHash + chunks, err := repo.GetByBlobHash(ctx, "blob1") + if err != nil { + t.Fatalf("failed to get blob chunks: %v", err) + } + if len(chunks) != 3 { + t.Errorf("expected 3 chunks, got %d", len(chunks)) + } + + // Verify order by offset + expectedOffsets := []int64{0, 1024, 3072} + for i, chunk := range chunks { + if chunk.Offset != expectedOffsets[i] { + t.Errorf("wrong chunk order: expected offset %d, got %d", expectedOffsets[i], chunk.Offset) + } + } + + // Test GetByChunkHash + bc, err := repo.GetByChunkHash(ctx, "chunk2") + if err != nil { + t.Fatalf("failed to get blob chunk by chunk hash: %v", err) + } + if bc == nil { + t.Fatal("expected blob chunk, got nil") + } + if bc.BlobHash != "blob1" { + t.Errorf("wrong blob hash: expected blob1, got %s", bc.BlobHash) + } + if bc.Offset != 1024 { + t.Errorf("wrong offset: expected 1024, got %d", bc.Offset) + } + + // Test non-existent chunk + bc, err = repo.GetByChunkHash(ctx, "nonexistent") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if bc != nil { + t.Error("expected nil for non-existent chunk") + } +} + +func TestBlobChunkRepositoryMultipleBlobs(t *testing.T) { + db, cleanup := setupTestDB(t) + defer cleanup() + + ctx := context.Background() + repo := NewBlobChunkRepository(db) + + // Create chunks across multiple blobs + // Some chunks are shared between blobs (deduplication scenario) + blobChunks := []BlobChunk{ + {BlobHash: "blob1", ChunkHash: "chunk1", Offset: 0, Length: 1024}, + {BlobHash: "blob1", ChunkHash: "chunk2", Offset: 1024, Length: 1024}, + {BlobHash: "blob2", ChunkHash: "chunk2", Offset: 0, Length: 1024}, // chunk2 is shared + {BlobHash: "blob2", ChunkHash: "chunk3", Offset: 1024, Length: 1024}, + } + + for _, bc := range blobChunks { + err := repo.Create(ctx, nil, &bc) + if err != nil { + t.Fatalf("failed to create blob chunk: %v", err) + } + } + + // Verify blob1 chunks + chunks, err := repo.GetByBlobHash(ctx, "blob1") + if err != nil { + t.Fatalf("failed to get blob1 chunks: %v", err) + } + if len(chunks) != 2 { + t.Errorf("expected 2 chunks for blob1, got %d", len(chunks)) + } + + // Verify blob2 chunks + chunks, err = repo.GetByBlobHash(ctx, "blob2") + if err != nil { + t.Fatalf("failed to get blob2 chunks: %v", err) + } + if len(chunks) != 2 { + t.Errorf("expected 2 chunks for blob2, got %d", len(chunks)) + } + + // Verify shared chunk + bc, err := repo.GetByChunkHash(ctx, "chunk2") + if err != nil { + t.Fatalf("failed to get shared chunk: %v", err) + } + if bc == nil { + t.Fatal("expected shared chunk, got nil") + } + // GetByChunkHash returns first match, should be blob1 + if bc.BlobHash != "blob1" { + t.Errorf("expected blob1 for shared chunk, got %s", bc.BlobHash) + } +} diff --git a/internal/database/blobs.go b/internal/database/blobs.go index 372201f..e7b4cb3 100644 --- a/internal/database/blobs.go +++ b/internal/database/blobs.go @@ -25,7 +25,7 @@ func (r *BlobRepository) Create(ctx context.Context, tx *sql.Tx, blob *Blob) err if tx != nil { _, err = tx.ExecContext(ctx, query, blob.BlobHash, blob.CreatedTS.Unix()) } else { - _, err = r.db.conn.ExecContext(ctx, query, blob.BlobHash, blob.CreatedTS.Unix()) + _, err = r.db.ExecWithLock(ctx, query, blob.BlobHash, blob.CreatedTS.Unix()) } if err != nil { diff --git a/internal/database/blobs_test.go b/internal/database/blobs_test.go new file mode 100644 index 0000000..511a6e7 --- /dev/null +++ b/internal/database/blobs_test.go @@ -0,0 +1,100 @@ +package database + +import ( + "context" + "testing" + "time" +) + +func TestBlobRepository(t *testing.T) { + db, cleanup := setupTestDB(t) + defer cleanup() + + ctx := context.Background() + repo := NewBlobRepository(db) + + // Test Create + blob := &Blob{ + BlobHash: "blobhash123", + CreatedTS: time.Now().Truncate(time.Second), + } + + err := repo.Create(ctx, nil, blob) + if err != nil { + t.Fatalf("failed to create blob: %v", err) + } + + // Test GetByHash + retrieved, err := repo.GetByHash(ctx, blob.BlobHash) + if err != nil { + t.Fatalf("failed to get blob: %v", err) + } + if retrieved == nil { + t.Fatal("expected blob, got nil") + } + if retrieved.BlobHash != blob.BlobHash { + t.Errorf("blob hash mismatch: got %s, want %s", retrieved.BlobHash, blob.BlobHash) + } + if !retrieved.CreatedTS.Equal(blob.CreatedTS) { + t.Errorf("created timestamp mismatch: got %v, want %v", retrieved.CreatedTS, blob.CreatedTS) + } + + // Test List + blob2 := &Blob{ + BlobHash: "blobhash456", + CreatedTS: time.Now().Truncate(time.Second), + } + err = repo.Create(ctx, nil, blob2) + if err != nil { + t.Fatalf("failed to create second blob: %v", err) + } + + blobs, err := repo.List(ctx, 10, 0) + if err != nil { + t.Fatalf("failed to list blobs: %v", err) + } + if len(blobs) != 2 { + t.Errorf("expected 2 blobs, got %d", len(blobs)) + } + + // Test pagination + blobs, err = repo.List(ctx, 1, 0) + if err != nil { + t.Fatalf("failed to list blobs with limit: %v", err) + } + if len(blobs) != 1 { + t.Errorf("expected 1 blob with limit, got %d", len(blobs)) + } + + blobs, err = repo.List(ctx, 1, 1) + if err != nil { + t.Fatalf("failed to list blobs with offset: %v", err) + } + if len(blobs) != 1 { + t.Errorf("expected 1 blob with offset, got %d", len(blobs)) + } +} + +func TestBlobRepositoryDuplicate(t *testing.T) { + db, cleanup := setupTestDB(t) + defer cleanup() + + ctx := context.Background() + repo := NewBlobRepository(db) + + blob := &Blob{ + BlobHash: "duplicate_blob", + CreatedTS: time.Now().Truncate(time.Second), + } + + err := repo.Create(ctx, nil, blob) + if err != nil { + t.Fatalf("failed to create blob: %v", err) + } + + // Try to create duplicate - should fail due to unique constraint + err = repo.Create(ctx, nil, blob) + if err == nil { + t.Error("expected error for duplicate blob") + } +} diff --git a/internal/database/chunk_files.go b/internal/database/chunk_files.go index 56a4323..3dbf321 100644 --- a/internal/database/chunk_files.go +++ b/internal/database/chunk_files.go @@ -25,7 +25,7 @@ func (r *ChunkFileRepository) Create(ctx context.Context, tx *sql.Tx, cf *ChunkF if tx != nil { _, err = tx.ExecContext(ctx, query, cf.ChunkHash, cf.FilePath, cf.FileOffset, cf.Length) } else { - _, err = r.db.conn.ExecContext(ctx, query, cf.ChunkHash, cf.FilePath, cf.FileOffset, cf.Length) + _, err = r.db.ExecWithLock(ctx, query, cf.ChunkHash, cf.FilePath, cf.FileOffset, cf.Length) } if err != nil { diff --git a/internal/database/chunk_files_test.go b/internal/database/chunk_files_test.go new file mode 100644 index 0000000..992a65b --- /dev/null +++ b/internal/database/chunk_files_test.go @@ -0,0 +1,142 @@ +package database + +import ( + "context" + "testing" +) + +func TestChunkFileRepository(t *testing.T) { + db, cleanup := setupTestDB(t) + defer cleanup() + + ctx := context.Background() + repo := NewChunkFileRepository(db) + + // Test Create + cf1 := &ChunkFile{ + ChunkHash: "chunk1", + FilePath: "/file1.txt", + FileOffset: 0, + Length: 1024, + } + + err := repo.Create(ctx, nil, cf1) + if err != nil { + t.Fatalf("failed to create chunk file: %v", err) + } + + // Add same chunk in different file (deduplication scenario) + cf2 := &ChunkFile{ + ChunkHash: "chunk1", + FilePath: "/file2.txt", + FileOffset: 2048, + Length: 1024, + } + err = repo.Create(ctx, nil, cf2) + if err != nil { + t.Fatalf("failed to create second chunk file: %v", err) + } + + // Test GetByChunkHash + chunkFiles, err := repo.GetByChunkHash(ctx, "chunk1") + if err != nil { + t.Fatalf("failed to get chunk files: %v", err) + } + if len(chunkFiles) != 2 { + t.Errorf("expected 2 files for chunk, got %d", len(chunkFiles)) + } + + // Verify both files are returned + foundFile1 := false + foundFile2 := false + for _, cf := range chunkFiles { + if cf.FilePath == "/file1.txt" && cf.FileOffset == 0 { + foundFile1 = true + } + if cf.FilePath == "/file2.txt" && cf.FileOffset == 2048 { + foundFile2 = true + } + } + if !foundFile1 || !foundFile2 { + t.Error("not all expected files found") + } + + // Test GetByFilePath + chunkFiles, err = repo.GetByFilePath(ctx, "/file1.txt") + if err != nil { + t.Fatalf("failed to get chunks by file path: %v", err) + } + if len(chunkFiles) != 1 { + t.Errorf("expected 1 chunk for file, got %d", len(chunkFiles)) + } + if chunkFiles[0].ChunkHash != "chunk1" { + t.Errorf("wrong chunk hash: expected chunk1, got %s", chunkFiles[0].ChunkHash) + } + + // Test duplicate insert (should be idempotent) + err = repo.Create(ctx, nil, cf1) + if err != nil { + t.Fatalf("failed to create duplicate chunk file: %v", err) + } +} + +func TestChunkFileRepositoryComplexDeduplication(t *testing.T) { + db, cleanup := setupTestDB(t) + defer cleanup() + + ctx := context.Background() + repo := NewChunkFileRepository(db) + + // Simulate a scenario where multiple files share chunks + // File1: chunk1, chunk2, chunk3 + // File2: chunk2, chunk3, chunk4 + // File3: chunk1, chunk4 + + chunkFiles := []ChunkFile{ + // File1 + {ChunkHash: "chunk1", FilePath: "/file1.txt", FileOffset: 0, Length: 1024}, + {ChunkHash: "chunk2", FilePath: "/file1.txt", FileOffset: 1024, Length: 1024}, + {ChunkHash: "chunk3", FilePath: "/file1.txt", FileOffset: 2048, Length: 1024}, + // File2 + {ChunkHash: "chunk2", FilePath: "/file2.txt", FileOffset: 0, Length: 1024}, + {ChunkHash: "chunk3", FilePath: "/file2.txt", FileOffset: 1024, Length: 1024}, + {ChunkHash: "chunk4", FilePath: "/file2.txt", FileOffset: 2048, Length: 1024}, + // File3 + {ChunkHash: "chunk1", FilePath: "/file3.txt", FileOffset: 0, Length: 1024}, + {ChunkHash: "chunk4", FilePath: "/file3.txt", FileOffset: 1024, Length: 1024}, + } + + for _, cf := range chunkFiles { + err := repo.Create(ctx, nil, &cf) + if err != nil { + t.Fatalf("failed to create chunk file: %v", err) + } + } + + // Test chunk1 (used by file1 and file3) + files, err := repo.GetByChunkHash(ctx, "chunk1") + if err != nil { + t.Fatalf("failed to get files for chunk1: %v", err) + } + if len(files) != 2 { + t.Errorf("expected 2 files for chunk1, got %d", len(files)) + } + + // Test chunk2 (used by file1 and file2) + files, err = repo.GetByChunkHash(ctx, "chunk2") + if err != nil { + t.Fatalf("failed to get files for chunk2: %v", err) + } + if len(files) != 2 { + t.Errorf("expected 2 files for chunk2, got %d", len(files)) + } + + // Test file2 chunks + chunks, err := repo.GetByFilePath(ctx, "/file2.txt") + if err != nil { + t.Fatalf("failed to get chunks for file2: %v", err) + } + if len(chunks) != 3 { + t.Errorf("expected 3 chunks for file2, got %d", len(chunks)) + } +} diff --git a/internal/database/chunks.go b/internal/database/chunks.go index 409b16d..6d802e1 100644 --- a/internal/database/chunks.go +++ b/internal/database/chunks.go @@ -25,7 +25,7 @@ func (r *ChunkRepository) Create(ctx context.Context, tx *sql.Tx, chunk *Chunk) if tx != nil { _, err = tx.ExecContext(ctx, query, chunk.ChunkHash, chunk.SHA256, chunk.Size) } else { - _, err = r.db.conn.ExecContext(ctx, query, chunk.ChunkHash, chunk.SHA256, chunk.Size) + _, err = r.db.ExecWithLock(ctx, query, chunk.ChunkHash, chunk.SHA256, chunk.Size) } if err != nil { diff --git a/internal/database/chunks_test.go b/internal/database/chunks_test.go new file mode 100644 index 0000000..2991d8f --- /dev/null +++ b/internal/database/chunks_test.go @@ -0,0 +1,104 @@ +package database + +import ( + "context" + "testing" +) + +func TestChunkRepository(t *testing.T) { + db, cleanup := setupTestDB(t) + defer cleanup() + + ctx := context.Background() + repo := NewChunkRepository(db) + + // Test Create + chunk := &Chunk{ + ChunkHash: "chunkhash123", + SHA256: "sha256hash123", + Size: 4096, + } + + err := repo.Create(ctx, nil, chunk) + if err != nil { + t.Fatalf("failed to create chunk: %v", err) + } + + // Test GetByHash + retrieved, err := repo.GetByHash(ctx, chunk.ChunkHash) + if err != nil { + t.Fatalf("failed to get chunk: %v", err) + } + if retrieved == nil { + t.Fatal("expected chunk, got nil") + } + if retrieved.ChunkHash != chunk.ChunkHash { + t.Errorf("chunk hash mismatch: got %s, want %s", retrieved.ChunkHash, chunk.ChunkHash) + } + if retrieved.SHA256 != chunk.SHA256 { + t.Errorf("sha256 mismatch: got %s, want %s", retrieved.SHA256, chunk.SHA256) + } + if retrieved.Size != chunk.Size { + t.Errorf("size mismatch: got %d, want %d", retrieved.Size, chunk.Size) + } + + // Test duplicate insert (should be idempotent) + err = repo.Create(ctx, nil, chunk) + if err != nil { + t.Fatalf("failed to create duplicate chunk: %v", err) + } + + // Test GetByHashes + chunk2 := &Chunk{ + ChunkHash: "chunkhash456", + SHA256: "sha256hash456", + Size: 8192, + } + err = repo.Create(ctx, nil, chunk2) + if err != nil { + t.Fatalf("failed to create second chunk: %v", err) + } + + chunks, err := repo.GetByHashes(ctx, []string{chunk.ChunkHash, chunk2.ChunkHash}) + if err != nil { + t.Fatalf("failed to get chunks by hashes: %v", err) + } + if len(chunks) != 2 { + t.Errorf("expected 2 chunks, got %d", len(chunks)) + } + + // Test ListUnpacked + unpacked, err := repo.ListUnpacked(ctx, 10) + if err != nil { + t.Fatalf("failed to list unpacked chunks: %v", err) + } + if len(unpacked) != 2 { + t.Errorf("expected 2 unpacked chunks, got %d", len(unpacked)) + } +} + +func TestChunkRepositoryNotFound(t *testing.T) { + db, cleanup := setupTestDB(t) + defer cleanup() + + ctx := context.Background() + repo := NewChunkRepository(db) + + // Test GetByHash with non-existent hash + chunk, err := repo.GetByHash(ctx, "nonexistent") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if chunk != nil { + t.Error("expected nil for non-existent chunk") + } + + // Test GetByHashes with empty list + chunks, err := repo.GetByHashes(ctx, []string{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if chunks != nil { + t.Error("expected nil for empty hash list") + } +} diff --git a/internal/database/database.go b/internal/database/database.go index adef2a3..46aceea 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -4,16 +4,18 @@ import ( "context" "database/sql" "fmt" + "sync" - _ "github.com/mattn/go-sqlite3" + _ "modernc.org/sqlite" ) type DB struct { - conn *sql.DB + conn *sql.DB + writeLock sync.Mutex } func New(ctx context.Context, path string) (*DB, error) { - conn, err := sql.Open("sqlite3", path+"?_journal_mode=WAL&_synchronous=NORMAL&_busy_timeout=5000") + conn, err := sql.Open("sqlite", path+"?_journal_mode=WAL&_synchronous=NORMAL&_busy_timeout=5000") if err != nil { return nil, fmt.Errorf("opening database: %w", err) } @@ -51,6 +53,30 @@ func (db *DB) BeginTx(ctx context.Context, opts *sql.TxOptions) (*sql.Tx, error) return db.conn.BeginTx(ctx, opts) } +// LockForWrite acquires the write lock +func (db *DB) LockForWrite() { + db.writeLock.Lock() +} + +// UnlockWrite releases the write lock +func (db *DB) UnlockWrite() { + db.writeLock.Unlock() +} + +// ExecWithLock executes a write query with the write lock held +func (db *DB) ExecWithLock(ctx context.Context, query string, args ...interface{}) (sql.Result, error) { + db.writeLock.Lock() + defer db.writeLock.Unlock() + return db.conn.ExecContext(ctx, query, args...) +} + +// QueryRowWithLock executes a write query that returns a row with the write lock held +func (db *DB) QueryRowWithLock(ctx context.Context, query string, args ...interface{}) *sql.Row { + db.writeLock.Lock() + defer db.writeLock.Unlock() + return db.conn.QueryRowContext(ctx, query, args...) +} + func (db *DB) createSchema(ctx context.Context) error { schema := ` CREATE TABLE IF NOT EXISTS files ( @@ -105,7 +131,10 @@ func (db *DB) createSchema(ctx context.Context) error { created_ts INTEGER NOT NULL, file_count INTEGER NOT NULL, chunk_count INTEGER NOT NULL, - blob_count INTEGER NOT NULL + blob_count INTEGER NOT NULL, + total_size INTEGER NOT NULL, + blob_size INTEGER NOT NULL, + compression_ratio REAL NOT NULL ); ` diff --git a/internal/database/database_test.go b/internal/database/database_test.go new file mode 100644 index 0000000..cea865c --- /dev/null +++ b/internal/database/database_test.go @@ -0,0 +1,96 @@ +package database + +import ( + "context" + "fmt" + "path/filepath" + "testing" +) + +func TestDatabase(t *testing.T) { + ctx := context.Background() + dbPath := filepath.Join(t.TempDir(), "test.db") + + db, err := New(ctx, dbPath) + if err != nil { + t.Fatalf("failed to create database: %v", err) + } + defer func() { + if err := db.Close(); err != nil { + t.Errorf("failed to close database: %v", err) + } + }() + + // Test connection + if db.Conn() == nil { + t.Fatal("database connection is nil") + } + + // Test schema creation (already done in New) + // Verify tables exist + tables := []string{ + "files", "file_chunks", "chunks", "blobs", + "blob_chunks", "chunk_files", "snapshots", + } + + for _, table := range tables { + var name string + err := db.conn.QueryRow("SELECT name FROM sqlite_master WHERE type='table' AND name=?", table).Scan(&name) + if err != nil { + t.Errorf("table %s does not exist: %v", table, err) + } + } +} + +func TestDatabaseInvalidPath(t *testing.T) { + ctx := context.Background() + + // Test with invalid path + _, err := New(ctx, "/invalid/path/that/does/not/exist/test.db") + if err == nil { + t.Fatal("expected error for invalid path") + } +} + +func TestDatabaseConcurrentAccess(t *testing.T) { + ctx := context.Background() + dbPath := filepath.Join(t.TempDir(), "test.db") + + db, err := New(ctx, dbPath) + if err != nil { + t.Fatalf("failed to create database: %v", err) + } + defer func() { + if err := db.Close(); err != nil { + t.Errorf("failed to close database: %v", err) + } + }() + + // Test concurrent writes + done := make(chan bool, 10) + for i := 0; i < 10; i++ { + go func(i int) { + _, err := db.ExecWithLock(ctx, "INSERT INTO chunks (chunk_hash, sha256, size) VALUES (?, ?, ?)", + fmt.Sprintf("hash%d", i), fmt.Sprintf("sha%d", i), i*1024) + if err != nil { + t.Errorf("concurrent insert failed: %v", err) + } + done <- true + }(i) + } + + // Wait for all goroutines + for i := 0; i < 10; i++ { + <-done + } + + // Verify all inserts succeeded + var count int + err = db.conn.QueryRowContext(ctx, "SELECT COUNT(*) FROM chunks").Scan(&count) + if err != nil { + t.Fatalf("failed to count chunks: %v", err) + } + if count != 10 { + t.Errorf("expected 10 chunks, got %d", count) + } +} diff --git a/internal/database/file_chunks.go b/internal/database/file_chunks.go index 05b6ab1..cf9a946 100644 --- a/internal/database/file_chunks.go +++ b/internal/database/file_chunks.go @@ -25,7 +25,7 @@ func (r *FileChunkRepository) Create(ctx context.Context, tx *sql.Tx, fc *FileCh if tx != nil { _, err = tx.ExecContext(ctx, query, fc.Path, fc.Idx, fc.ChunkHash) } else { - _, err = r.db.conn.ExecContext(ctx, query, fc.Path, fc.Idx, fc.ChunkHash) + _, err = r.db.ExecWithLock(ctx, query, fc.Path, fc.Idx, fc.ChunkHash) } if err != nil { @@ -69,7 +69,7 @@ func (r *FileChunkRepository) DeleteByPath(ctx context.Context, tx *sql.Tx, path if tx != nil { _, err = tx.ExecContext(ctx, query, path) } else { - _, err = r.db.conn.ExecContext(ctx, query, path) + _, err = r.db.ExecWithLock(ctx, query, path) } if err != nil { diff --git a/internal/database/file_chunks_test.go b/internal/database/file_chunks_test.go new file mode 100644 index 0000000..f72c351 --- /dev/null +++ b/internal/database/file_chunks_test.go @@ -0,0 +1,119 @@ +package database + +import ( + "context" + "fmt" + "testing" +) + +func TestFileChunkRepository(t *testing.T) { + db, cleanup := setupTestDB(t) + defer cleanup() + + ctx := context.Background() + repo := NewFileChunkRepository(db) + + // Test Create + fc1 := &FileChunk{ + Path: "/test/file.txt", + Idx: 0, + ChunkHash: "chunk1", + } + + err := repo.Create(ctx, nil, fc1) + if err != nil { + t.Fatalf("failed to create file chunk: %v", err) + } + + // Add more chunks for the same file + fc2 := &FileChunk{ + Path: "/test/file.txt", + Idx: 1, + ChunkHash: "chunk2", + } + err = repo.Create(ctx, nil, fc2) + if err != nil { + t.Fatalf("failed to create second file chunk: %v", err) + } + + fc3 := &FileChunk{ + Path: "/test/file.txt", + Idx: 2, + ChunkHash: "chunk3", + } + err = repo.Create(ctx, nil, fc3) + if err != nil { + t.Fatalf("failed to create third file chunk: %v", err) + } + + // Test GetByPath + chunks, err := repo.GetByPath(ctx, "/test/file.txt") + if err != nil { + t.Fatalf("failed to get file chunks: %v", err) + } + if len(chunks) != 3 { + t.Errorf("expected 3 chunks, got %d", len(chunks)) + } + + // Verify order + for i, chunk := range chunks { + if chunk.Idx != i { + t.Errorf("wrong chunk order: expected idx %d, got %d", i, chunk.Idx) + } + } + + // Test duplicate insert (should be idempotent) + err = repo.Create(ctx, nil, fc1) + if err != nil { + t.Fatalf("failed to create duplicate file chunk: %v", err) + } + + // Test DeleteByPath + err = repo.DeleteByPath(ctx, nil, "/test/file.txt") + if err != nil { + t.Fatalf("failed to delete file chunks: %v", err) + } + + chunks, err = repo.GetByPath(ctx, "/test/file.txt") + if err != nil { + t.Fatalf("failed to get deleted file chunks: %v", err) + } + if len(chunks) != 0 { + t.Errorf("expected 0 chunks after delete, got %d", len(chunks)) + } +} + +func TestFileChunkRepositoryMultipleFiles(t *testing.T) { + db, cleanup := setupTestDB(t) + defer cleanup() + + ctx := context.Background() + repo := NewFileChunkRepository(db) + + // Create chunks for multiple files + files := []string{"/file1.txt", "/file2.txt", "/file3.txt"} + for _, path := range files { + for i := 0; i < 2; i++ { + fc := &FileChunk{ + Path: path, + Idx: i, + ChunkHash: fmt.Sprintf("%s_chunk%d", path, i), + } + err := repo.Create(ctx, nil, fc) + if err != nil { + t.Fatalf("failed to create file chunk: %v", err) + } + } + } + + // Verify each file has correct chunks + for _, path := range files { + chunks, err := repo.GetByPath(ctx, path) + if err != nil { + t.Fatalf("failed to get chunks for %s: %v", path, err) + } + if len(chunks) != 2 { + t.Errorf("expected 2 chunks for %s, got %d", path, len(chunks)) + } + } +} diff --git a/internal/database/files.go b/internal/database/files.go index d6565e1..b8a5179 100644 --- a/internal/database/files.go +++ b/internal/database/files.go @@ -33,7 +33,7 @@ func (r *FileRepository) Create(ctx context.Context, tx *sql.Tx, file *File) err if tx != nil { _, err = tx.ExecContext(ctx, query, file.Path, file.MTime.Unix(), file.CTime.Unix(), file.Size, file.Mode, file.UID, file.GID, file.LinkTarget) } else { - _, err = r.db.conn.ExecContext(ctx, query, file.Path, file.MTime.Unix(), file.CTime.Unix(), file.Size, file.Mode, file.UID, file.GID, file.LinkTarget) + _, err = r.db.ExecWithLock(ctx, query, file.Path, file.MTime.Unix(), file.CTime.Unix(), file.Size, file.Mode, file.UID, file.GID, file.LinkTarget) } if err != nil { @@ -134,7 +134,7 @@ func (r *FileRepository) Delete(ctx context.Context, tx *sql.Tx, path string) er if tx != nil { _, err = tx.ExecContext(ctx, query, path) } else { - _, err = r.db.conn.ExecContext(ctx, query, path) + _, err = r.db.ExecWithLock(ctx, query, path) } if err != nil { diff --git a/internal/database/files_test.go b/internal/database/files_test.go new file mode 100644 index 0000000..f6e155e --- /dev/null +++ b/internal/database/files_test.go @@ -0,0 +1,191 @@ +package database + +import ( + "context" + "database/sql" + "fmt" + "os" + "path/filepath" + "testing" + "time" +) + +func setupTestDB(t *testing.T) (*DB, func()) { + ctx := context.Background() + dbPath := filepath.Join(t.TempDir(), "test.db") + + db, err := New(ctx, dbPath) + if err != nil { + t.Fatalf("failed to create database: %v", err) + } + + cleanup := func() { + if err := db.Close(); err != nil { + t.Errorf("failed to close database: %v", err) + } + } + + return db, cleanup +} + +func TestFileRepository(t *testing.T) { + db, cleanup := setupTestDB(t) + defer cleanup() + + ctx := context.Background() + repo := NewFileRepository(db) + + // Test Create + file := &File{ + Path: "/test/file.txt", + MTime: time.Now().Truncate(time.Second), + CTime: time.Now().Truncate(time.Second), + Size: 1024, + Mode: 0644, + UID: 1000, + GID: 1000, + LinkTarget: "", + } + + err := repo.Create(ctx, nil, file) + if err != nil { + t.Fatalf("failed to create file: %v", err) + } + + // Test GetByPath + retrieved, err := repo.GetByPath(ctx, file.Path) + if err != nil { + t.Fatalf("failed to get file: %v", err) + } + if retrieved == nil { + t.Fatal("expected file, got nil") + } + if retrieved.Path != file.Path { + t.Errorf("path mismatch: got %s, want %s", retrieved.Path, file.Path) + } + if !retrieved.MTime.Equal(file.MTime) { + t.Errorf("mtime mismatch: got %v, want %v", retrieved.MTime, file.MTime) + } + if retrieved.Size != file.Size { + t.Errorf("size mismatch: got %d, want %d", retrieved.Size, file.Size) + } + if retrieved.Mode != file.Mode { + t.Errorf("mode mismatch: got %o, want %o", retrieved.Mode, file.Mode) + } + + // Test Update (upsert) + file.Size = 2048 + file.MTime = time.Now().Truncate(time.Second) + err = repo.Create(ctx, nil, file) + if err != nil { + t.Fatalf("failed to update file: %v", err) + } + + retrieved, err = repo.GetByPath(ctx, file.Path) + if err != nil { + t.Fatalf("failed to get updated file: %v", err) + } + if retrieved.Size != 2048 { + t.Errorf("size not updated: got %d, want %d", retrieved.Size, 2048) + } + + // Test ListModifiedSince + files, err := repo.ListModifiedSince(ctx, time.Now().Add(-1*time.Hour)) + if err != nil { + t.Fatalf("failed to list files: %v", err) + } + if len(files) != 1 { + t.Errorf("expected 1 file, got %d", len(files)) + } + + // Test Delete + err = repo.Delete(ctx, nil, file.Path) + if err != nil { + t.Fatalf("failed to delete file: %v", err) + } + + retrieved, err = repo.GetByPath(ctx, file.Path) + if err != nil { + t.Fatalf("error getting deleted file: %v", err) + } + if retrieved != nil { + t.Error("expected nil for deleted file") + } +} + +func TestFileRepositorySymlink(t *testing.T) { + db, cleanup := setupTestDB(t) + defer cleanup() + + ctx := context.Background() + repo := NewFileRepository(db) + + // Test symlink + symlink := &File{ + Path: "/test/link", + MTime: time.Now().Truncate(time.Second), + CTime: time.Now().Truncate(time.Second), + Size: 0, + Mode: uint32(0777 | os.ModeSymlink), + UID: 1000, + GID: 1000, + LinkTarget: "/test/target", + } + + err := repo.Create(ctx, nil, symlink) + if err != nil { + t.Fatalf("failed to create symlink: %v", err) + } + + retrieved, err := repo.GetByPath(ctx, symlink.Path) + if err != nil { + t.Fatalf("failed to get symlink: %v", err) + } + if !retrieved.IsSymlink() { + t.Error("expected IsSymlink() to be true") + } + if retrieved.LinkTarget != symlink.LinkTarget { + t.Errorf("link target mismatch: got %s, want %s", retrieved.LinkTarget, symlink.LinkTarget) + } +} + +func TestFileRepositoryTransaction(t *testing.T) { + db, cleanup := setupTestDB(t) + defer cleanup() + + ctx := context.Background() + repos := NewRepositories(db) + + // Test transaction rollback + err := repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { + file := &File{ + Path: "/test/tx_file.txt", + MTime: time.Now().Truncate(time.Second), + CTime: time.Now().Truncate(time.Second), + Size: 1024, + Mode: 0644, + UID: 1000, + GID: 1000, + } + + if err := repos.Files.Create(ctx, tx, file); err != nil { + return err + } + + // Return error to trigger rollback + return fmt.Errorf("test rollback") + }) + + if err == nil || err.Error() != "test rollback" { + t.Fatalf("expected rollback error, got: %v", err) + } + + // Verify file was not created + retrieved, err := repos.Files.GetByPath(ctx, "/test/tx_file.txt") + if err != nil { + t.Fatalf("error checking for file: %v", err) + } + if retrieved != nil { + t.Error("file should not exist after rollback") + } +} diff --git a/internal/database/models.go b/internal/database/models.go index 60eec75..1e24c5f 100644 --- a/internal/database/models.go +++ b/internal/database/models.go @@ -57,11 +57,14 @@ type ChunkFile struct { // Snapshot represents a snapshot record in the database type Snapshot struct { - ID string - Hostname string - VaultikVersion string - CreatedTS time.Time - FileCount int64 - ChunkCount int64 - BlobCount int64 + ID string + Hostname string + VaultikVersion string + CreatedTS time.Time + FileCount int64 + ChunkCount int64 + BlobCount int64 + TotalSize int64 // Total size of all referenced files + BlobSize int64 // Total size of all referenced blobs (compressed and encrypted) + CompressionRatio float64 // Compression ratio (BlobSize / TotalSize) } diff --git a/internal/database/repositories.go b/internal/database/repositories.go index d033e3b..c6db312 100644 --- a/internal/database/repositories.go +++ b/internal/database/repositories.go @@ -33,6 +33,10 @@ func NewRepositories(db *DB) *Repositories { type TxFunc func(ctx context.Context, tx *sql.Tx) error func (r *Repositories) WithTx(ctx context.Context, fn TxFunc) error { + // Acquire write lock for the entire transaction + r.db.LockForWrite() + defer r.db.UnlockWrite() + tx, err := r.db.BeginTx(ctx, nil) if err != nil { return fmt.Errorf("beginning transaction: %w", err) diff --git a/internal/database/repositories_test.go b/internal/database/repositories_test.go new file mode 100644 index 0000000..94f6170 --- /dev/null +++ b/internal/database/repositories_test.go @@ -0,0 +1,247 @@ +package database + +import ( + "context" + "database/sql" + "fmt" + "testing" + "time" +) + +func TestRepositoriesTransaction(t *testing.T) { + db, cleanup := setupTestDB(t) + defer cleanup() + + ctx := context.Background() + repos := NewRepositories(db) + + // Test successful transaction with multiple operations + err := repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { + // Create a file + file := &File{ + Path: "/test/tx_file.txt", + MTime: time.Now().Truncate(time.Second), + CTime: time.Now().Truncate(time.Second), + Size: 1024, + Mode: 0644, + UID: 1000, + GID: 1000, + } + if err := repos.Files.Create(ctx, tx, file); err != nil { + return err + } + + // Create chunks + chunk1 := &Chunk{ + ChunkHash: "tx_chunk1", + SHA256: "tx_sha1", + Size: 512, + } + if err := repos.Chunks.Create(ctx, tx, chunk1); err != nil { + return err + } + + chunk2 := &Chunk{ + ChunkHash: "tx_chunk2", + SHA256: "tx_sha2", + Size: 512, + } + if err := repos.Chunks.Create(ctx, tx, chunk2); err != nil { + return err + } + + // Map chunks to file + fc1 := &FileChunk{ + Path: file.Path, + Idx: 0, + ChunkHash: chunk1.ChunkHash, + } + if err := repos.FileChunks.Create(ctx, tx, fc1); err != nil { + return err + } + + fc2 := &FileChunk{ + Path: file.Path, + Idx: 1, + ChunkHash: chunk2.ChunkHash, + } + if err := repos.FileChunks.Create(ctx, tx, fc2); err != nil { + return err + } + + // Create blob + blob := &Blob{ + BlobHash: "tx_blob1", + CreatedTS: time.Now().Truncate(time.Second), + } + if err := repos.Blobs.Create(ctx, tx, blob); err != nil { + return err + } + + // Map chunks to blob + bc1 := &BlobChunk{ + BlobHash: blob.BlobHash, + ChunkHash: chunk1.ChunkHash, + Offset: 0, + Length: 512, + } + if err := repos.BlobChunks.Create(ctx, tx, bc1); err != nil { + return err + } + + bc2 := &BlobChunk{ + BlobHash: blob.BlobHash, + ChunkHash: chunk2.ChunkHash, + Offset: 512, + Length: 512, + } + if err := repos.BlobChunks.Create(ctx, tx, bc2); err != nil { + return err + } + + return nil + }) + + if err != nil { + t.Fatalf("transaction failed: %v", err) + } + + // Verify all data was committed + file, err := repos.Files.GetByPath(ctx, "/test/tx_file.txt") + if err != nil { + t.Fatalf("failed to get file: %v", err) + } + if file == nil { + t.Error("expected file after transaction") + } + + chunks, err := repos.FileChunks.GetByPath(ctx, "/test/tx_file.txt") + if err != nil { + t.Fatalf("failed to get file chunks: %v", err) + } + if len(chunks) != 2 { + t.Errorf("expected 2 file chunks, got %d", len(chunks)) + } + + blob, err := repos.Blobs.GetByHash(ctx, "tx_blob1") + if err != nil { + t.Fatalf("failed to get blob: %v", err) + } + if blob == nil { + t.Error("expected blob after transaction") + } +} + +func TestRepositoriesTransactionRollback(t *testing.T) { + db, cleanup := setupTestDB(t) + defer cleanup() + + ctx := context.Background() + repos := NewRepositories(db) + + // Test transaction rollback + err := repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { + // Create a file + file := &File{ + Path: "/test/rollback_file.txt", + MTime: time.Now().Truncate(time.Second), + CTime: time.Now().Truncate(time.Second), + Size: 1024, + Mode: 0644, + UID: 1000, + GID: 1000, + } + if err := repos.Files.Create(ctx, tx, file); err != nil { + return err + } + + // Create a chunk + chunk := &Chunk{ + ChunkHash: "rollback_chunk", + SHA256: "rollback_sha", + Size: 1024, + } + if err := repos.Chunks.Create(ctx, tx, chunk); err != nil { + return err + } + + // Return error to trigger rollback + return fmt.Errorf("intentional rollback") + }) + + if err == nil || err.Error() != "intentional rollback" { + t.Fatalf("expected rollback error, got: %v", err) + } + + // Verify nothing was committed + file, err := repos.Files.GetByPath(ctx, "/test/rollback_file.txt") + if err != nil { + t.Fatalf("error checking for file: %v", err) + } + if file != nil { + t.Error("file should not exist after rollback") + } + + chunk, err := repos.Chunks.GetByHash(ctx, "rollback_chunk") + if err != nil { + t.Fatalf("error checking for chunk: %v", err) + } + if chunk != nil { + t.Error("chunk should not exist after rollback") + } +} + +func TestRepositoriesReadTransaction(t *testing.T) { + db, cleanup := setupTestDB(t) + defer cleanup() + + ctx := context.Background() + repos := NewRepositories(db) + + // First, create some data + file := &File{ + Path: "/test/read_file.txt", + MTime: time.Now().Truncate(time.Second), + CTime: time.Now().Truncate(time.Second), + Size: 1024, + Mode: 0644, + UID: 1000, + GID: 1000, + } + err := repos.Files.Create(ctx, nil, file) + if err != nil { + t.Fatalf("failed to create file: %v", err) + } + + // Test read-only transaction + var retrievedFile *File + err = repos.WithReadTx(ctx, func(ctx context.Context, tx *sql.Tx) error { + var err error + retrievedFile, err = repos.Files.GetByPath(ctx, "/test/read_file.txt") + if err != nil { + return err + } + + // Try to write in read-only transaction (should fail) + _ = repos.Files.Create(ctx, tx, &File{ + Path: "/test/should_fail.txt", + MTime: time.Now(), + CTime: time.Now(), + Size: 0, + Mode: 0644, + UID: 1000, + GID: 1000, + }) + // SQLite might not enforce read-only at this level, but we test the pattern + + return nil + }) + + if err != nil { + t.Fatalf("read transaction failed: %v", err) + } + + if retrievedFile == nil { + t.Error("expected to retrieve file in read transaction") + } +} diff --git a/internal/database/snapshots.go b/internal/database/snapshots.go index ea5e8f6..0b3e0b8 100644 --- a/internal/database/snapshots.go +++ b/internal/database/snapshots.go @@ -17,15 +17,17 @@ func NewSnapshotRepository(db *DB) *SnapshotRepository { func (r *SnapshotRepository) Create(ctx context.Context, tx *sql.Tx, snapshot *Snapshot) error { query := ` - INSERT INTO snapshots (id, hostname, vaultik_version, created_ts, file_count, chunk_count, blob_count) - VALUES (?, ?, ?, ?, ?, ?, ?) + INSERT INTO snapshots (id, hostname, vaultik_version, created_ts, file_count, chunk_count, blob_count, total_size, blob_size, compression_ratio) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ` var err error if tx != nil { - _, err = tx.ExecContext(ctx, query, snapshot.ID, snapshot.Hostname, snapshot.VaultikVersion, snapshot.CreatedTS.Unix(), snapshot.FileCount, snapshot.ChunkCount, snapshot.BlobCount) + _, err = tx.ExecContext(ctx, query, snapshot.ID, snapshot.Hostname, snapshot.VaultikVersion, snapshot.CreatedTS.Unix(), + snapshot.FileCount, snapshot.ChunkCount, snapshot.BlobCount, snapshot.TotalSize, snapshot.BlobSize, snapshot.CompressionRatio) } else { - _, err = r.db.conn.ExecContext(ctx, query, snapshot.ID, snapshot.Hostname, snapshot.VaultikVersion, snapshot.CreatedTS.Unix(), snapshot.FileCount, snapshot.ChunkCount, snapshot.BlobCount) + _, err = r.db.ExecWithLock(ctx, query, snapshot.ID, snapshot.Hostname, snapshot.VaultikVersion, snapshot.CreatedTS.Unix(), + snapshot.FileCount, snapshot.ChunkCount, snapshot.BlobCount, snapshot.TotalSize, snapshot.BlobSize, snapshot.CompressionRatio) } if err != nil { @@ -35,20 +37,28 @@ func (r *SnapshotRepository) Create(ctx context.Context, tx *sql.Tx, snapshot *S return nil } -func (r *SnapshotRepository) UpdateCounts(ctx context.Context, tx *sql.Tx, snapshotID string, fileCount, chunkCount, blobCount int64) error { +func (r *SnapshotRepository) UpdateCounts(ctx context.Context, tx *sql.Tx, snapshotID string, fileCount, chunkCount, blobCount, totalSize, blobSize int64) error { + compressionRatio := 1.0 + if totalSize > 0 { + compressionRatio = float64(blobSize) / float64(totalSize) + } + query := ` UPDATE snapshots SET file_count = ?, chunk_count = ?, - blob_count = ? + blob_count = ?, + total_size = ?, + blob_size = ?, + compression_ratio = ? WHERE id = ? ` var err error if tx != nil { - _, err = tx.ExecContext(ctx, query, fileCount, chunkCount, blobCount, snapshotID) + _, err = tx.ExecContext(ctx, query, fileCount, chunkCount, blobCount, totalSize, blobSize, compressionRatio, snapshotID) } else { - _, err = r.db.conn.ExecContext(ctx, query, fileCount, chunkCount, blobCount, snapshotID) + _, err = r.db.ExecWithLock(ctx, query, fileCount, chunkCount, blobCount, totalSize, blobSize, compressionRatio, snapshotID) } if err != nil { @@ -60,7 +70,7 @@ func (r *SnapshotRepository) UpdateCounts(ctx context.Context, tx *sql.Tx, snaps func (r *SnapshotRepository) GetByID(ctx context.Context, snapshotID string) (*Snapshot, error) { query := ` - SELECT id, hostname, vaultik_version, created_ts, file_count, chunk_count, blob_count + SELECT id, hostname, vaultik_version, created_ts, file_count, chunk_count, blob_count, total_size, blob_size, compression_ratio FROM snapshots WHERE id = ? ` @@ -76,6 +86,9 @@ func (r *SnapshotRepository) GetByID(ctx context.Context, snapshotID string) (*S &snapshot.FileCount, &snapshot.ChunkCount, &snapshot.BlobCount, + &snapshot.TotalSize, + &snapshot.BlobSize, + &snapshot.CompressionRatio, ) if err == sql.ErrNoRows { @@ -92,7 +105,7 @@ func (r *SnapshotRepository) GetByID(ctx context.Context, snapshotID string) (*S func (r *SnapshotRepository) ListRecent(ctx context.Context, limit int) ([]*Snapshot, error) { query := ` - SELECT id, hostname, vaultik_version, created_ts, file_count, chunk_count, blob_count + SELECT id, hostname, vaultik_version, created_ts, file_count, chunk_count, blob_count, total_size, blob_size, compression_ratio FROM snapshots ORDER BY created_ts DESC LIMIT ? @@ -117,6 +130,9 @@ func (r *SnapshotRepository) ListRecent(ctx context.Context, limit int) ([]*Snap &snapshot.FileCount, &snapshot.ChunkCount, &snapshot.BlobCount, + &snapshot.TotalSize, + &snapshot.BlobSize, + &snapshot.CompressionRatio, ) if err != nil { return nil, fmt.Errorf("scanning snapshot: %w", err) diff --git a/internal/database/snapshots_test.go b/internal/database/snapshots_test.go new file mode 100644 index 0000000..8a77020 --- /dev/null +++ b/internal/database/snapshots_test.go @@ -0,0 +1,181 @@ +package database + +import ( + "context" + "fmt" + "math" + "testing" + "time" +) + +const ( + Mebibyte = 1024 * 1024 + oneHundredMebibytes = 100 * Mebibyte + fortyMebibytes = 40 * Mebibyte + sixtyMebibytes = 60 * Mebibyte + twoHundredMebibytes = 200 * Mebibyte + compressionRatioPoint4 = 0.4 + compressionRatioPoint3 = 0.3 +) + +func TestSnapshotRepository(t *testing.T) { + db, cleanup := setupTestDB(t) + defer cleanup() + + ctx := context.Background() + repo := NewSnapshotRepository(db) + + // Test Create + snapshot := &Snapshot{ + ID: "2024-01-01T12:00:00Z", + Hostname: "test-host", + VaultikVersion: "1.0.0", + CreatedTS: time.Now().Truncate(time.Second), + FileCount: 100, + ChunkCount: 500, + BlobCount: 10, + TotalSize: oneHundredMebibytes, + BlobSize: fortyMebibytes, + CompressionRatio: compressionRatioPoint4, // 40MB / 100MB + } + + err := repo.Create(ctx, nil, snapshot) + if err != nil { + t.Fatalf("failed to create snapshot: %v", err) + } + + // Test GetByID + retrieved, err := repo.GetByID(ctx, snapshot.ID) + if err != nil { + t.Fatalf("failed to get snapshot: %v", err) + } + if retrieved == nil { + t.Fatal("expected snapshot, got nil") + } + if retrieved.ID != snapshot.ID { + t.Errorf("ID mismatch: got %s, want %s", retrieved.ID, snapshot.ID) + } + if retrieved.Hostname != snapshot.Hostname { + t.Errorf("hostname mismatch: got %s, want %s", retrieved.Hostname, snapshot.Hostname) + } + if retrieved.FileCount != snapshot.FileCount { + t.Errorf("file count mismatch: got %d, want %d", retrieved.FileCount, snapshot.FileCount) + } + + // Test UpdateCounts + err = repo.UpdateCounts(ctx, nil, snapshot.ID, 200, 1000, 20, twoHundredMebibytes, sixtyMebibytes) + if err != nil { + t.Fatalf("failed to update counts: %v", err) + } + + retrieved, err = repo.GetByID(ctx, snapshot.ID) + if err != nil { + t.Fatalf("failed to get updated snapshot: %v", err) + } + if retrieved.FileCount != 200 { + t.Errorf("file count not updated: got %d, want %d", retrieved.FileCount, 200) + } + if retrieved.ChunkCount != 1000 { + t.Errorf("chunk count not updated: got %d, want %d", retrieved.ChunkCount, 1000) + } + if retrieved.BlobCount != 20 { + t.Errorf("blob count not updated: got %d, want %d", retrieved.BlobCount, 20) + } + if retrieved.TotalSize != twoHundredMebibytes { + t.Errorf("total size not updated: got %d, want %d", retrieved.TotalSize, twoHundredMebibytes) + } + if retrieved.BlobSize != sixtyMebibytes { + t.Errorf("blob size not updated: got %d, want %d", retrieved.BlobSize, sixtyMebibytes) + } + expectedRatio := compressionRatioPoint3 // 0.3 + if math.Abs(retrieved.CompressionRatio-expectedRatio) > 0.001 { + t.Errorf("compression ratio not updated: got %f, want %f", retrieved.CompressionRatio, expectedRatio) + } + + // Test ListRecent + // Add more snapshots + for i := 2; i <= 5; i++ { + s := &Snapshot{ + ID: fmt.Sprintf("2024-01-0%dT12:00:00Z", i), + Hostname: "test-host", + VaultikVersion: "1.0.0", + CreatedTS: time.Now().Add(time.Duration(i) * time.Hour).Truncate(time.Second), + FileCount: int64(100 * i), + ChunkCount: int64(500 * i), + BlobCount: int64(10 * i), + } + err := repo.Create(ctx, nil, s) + if err != nil { + t.Fatalf("failed to create snapshot %d: %v", i, err) + } + } + + // Test listing with limit + recent, err := repo.ListRecent(ctx, 3) + if err != nil { + t.Fatalf("failed to list recent snapshots: %v", err) + } + if len(recent) != 3 { + t.Errorf("expected 3 recent snapshots, got %d", len(recent)) + } + + // Verify order (most recent first) + for i := 0; i < len(recent)-1; i++ { + if recent[i].CreatedTS.Before(recent[i+1].CreatedTS) { + t.Error("snapshots not in descending order") + } + } +} + +func TestSnapshotRepositoryNotFound(t *testing.T) { + db, cleanup := setupTestDB(t) + defer cleanup() + + ctx := context.Background() + repo := NewSnapshotRepository(db) + + // Test GetByID with non-existent ID + snapshot, err := repo.GetByID(ctx, "nonexistent") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if snapshot != nil { + t.Error("expected nil for non-existent snapshot") + } + + // Test UpdateCounts on non-existent snapshot + err = repo.UpdateCounts(ctx, nil, "nonexistent", 100, 200, 10, oneHundredMebibytes, fortyMebibytes) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + // No error expected, but no rows should be affected +} + +func TestSnapshotRepositoryDuplicate(t *testing.T) { + db, cleanup := setupTestDB(t) + defer cleanup() + + ctx := context.Background() + repo := NewSnapshotRepository(db) + + snapshot := &Snapshot{ + ID: "2024-01-01T12:00:00Z", + Hostname: "test-host", + VaultikVersion: "1.0.0", + CreatedTS: time.Now().Truncate(time.Second), + FileCount: 100, + ChunkCount: 500, + BlobCount: 10, + } + + err := repo.Create(ctx, nil, snapshot) + if err != nil { + t.Fatalf("failed to create snapshot: %v", err) + } + + // Try to create duplicate - should fail due to primary key constraint + err = repo.Create(ctx, nil, snapshot) + if err == nil { + t.Error("expected error for duplicate snapshot") + } +}