Implement SQLite index database layer
- Add pure Go SQLite driver (modernc.org/sqlite) to avoid CGO dependency - Implement database connection management with WAL mode - Add write mutex for serializing concurrent writes - Create schema for all tables matching DESIGN.md specifications - Implement repository pattern for all database entities: - Files, FileChunks, Chunks, Blobs, BlobChunks, ChunkFiles, Snapshots - Add transaction support with proper rollback handling - Add fatal error handling for database integrity issues - Add snapshot fields for tracking file sizes and compression ratios - Make index path configurable via VAULTIK_INDEX_PATH environment variable - Add comprehensive test coverage for all repositories - Add format check to Makefile to ensure code formatting
This commit is contained in:
142
internal/database/chunk_files_test.go
Normal file
142
internal/database/chunk_files_test.go
Normal file
@@ -0,0 +1,142 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestChunkFileRepository(t *testing.T) {
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
repo := NewChunkFileRepository(db)
|
||||
|
||||
// Test Create
|
||||
cf1 := &ChunkFile{
|
||||
ChunkHash: "chunk1",
|
||||
FilePath: "/file1.txt",
|
||||
FileOffset: 0,
|
||||
Length: 1024,
|
||||
}
|
||||
|
||||
err := repo.Create(ctx, nil, cf1)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create chunk file: %v", err)
|
||||
}
|
||||
|
||||
// Add same chunk in different file (deduplication scenario)
|
||||
cf2 := &ChunkFile{
|
||||
ChunkHash: "chunk1",
|
||||
FilePath: "/file2.txt",
|
||||
FileOffset: 2048,
|
||||
Length: 1024,
|
||||
}
|
||||
err = repo.Create(ctx, nil, cf2)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create second chunk file: %v", err)
|
||||
}
|
||||
|
||||
// Test GetByChunkHash
|
||||
chunkFiles, err := repo.GetByChunkHash(ctx, "chunk1")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get chunk files: %v", err)
|
||||
}
|
||||
if len(chunkFiles) != 2 {
|
||||
t.Errorf("expected 2 files for chunk, got %d", len(chunkFiles))
|
||||
}
|
||||
|
||||
// Verify both files are returned
|
||||
foundFile1 := false
|
||||
foundFile2 := false
|
||||
for _, cf := range chunkFiles {
|
||||
if cf.FilePath == "/file1.txt" && cf.FileOffset == 0 {
|
||||
foundFile1 = true
|
||||
}
|
||||
if cf.FilePath == "/file2.txt" && cf.FileOffset == 2048 {
|
||||
foundFile2 = true
|
||||
}
|
||||
}
|
||||
if !foundFile1 || !foundFile2 {
|
||||
t.Error("not all expected files found")
|
||||
}
|
||||
|
||||
// Test GetByFilePath
|
||||
chunkFiles, err = repo.GetByFilePath(ctx, "/file1.txt")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get chunks by file path: %v", err)
|
||||
}
|
||||
if len(chunkFiles) != 1 {
|
||||
t.Errorf("expected 1 chunk for file, got %d", len(chunkFiles))
|
||||
}
|
||||
if chunkFiles[0].ChunkHash != "chunk1" {
|
||||
t.Errorf("wrong chunk hash: expected chunk1, got %s", chunkFiles[0].ChunkHash)
|
||||
}
|
||||
|
||||
// Test duplicate insert (should be idempotent)
|
||||
err = repo.Create(ctx, nil, cf1)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create duplicate chunk file: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChunkFileRepositoryComplexDeduplication(t *testing.T) {
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
repo := NewChunkFileRepository(db)
|
||||
|
||||
// Simulate a scenario where multiple files share chunks
|
||||
// File1: chunk1, chunk2, chunk3
|
||||
// File2: chunk2, chunk3, chunk4
|
||||
// File3: chunk1, chunk4
|
||||
|
||||
chunkFiles := []ChunkFile{
|
||||
// File1
|
||||
{ChunkHash: "chunk1", FilePath: "/file1.txt", FileOffset: 0, Length: 1024},
|
||||
{ChunkHash: "chunk2", FilePath: "/file1.txt", FileOffset: 1024, Length: 1024},
|
||||
{ChunkHash: "chunk3", FilePath: "/file1.txt", FileOffset: 2048, Length: 1024},
|
||||
// File2
|
||||
{ChunkHash: "chunk2", FilePath: "/file2.txt", FileOffset: 0, Length: 1024},
|
||||
{ChunkHash: "chunk3", FilePath: "/file2.txt", FileOffset: 1024, Length: 1024},
|
||||
{ChunkHash: "chunk4", FilePath: "/file2.txt", FileOffset: 2048, Length: 1024},
|
||||
// File3
|
||||
{ChunkHash: "chunk1", FilePath: "/file3.txt", FileOffset: 0, Length: 1024},
|
||||
{ChunkHash: "chunk4", FilePath: "/file3.txt", FileOffset: 1024, Length: 1024},
|
||||
}
|
||||
|
||||
for _, cf := range chunkFiles {
|
||||
err := repo.Create(ctx, nil, &cf)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create chunk file: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Test chunk1 (used by file1 and file3)
|
||||
files, err := repo.GetByChunkHash(ctx, "chunk1")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get files for chunk1: %v", err)
|
||||
}
|
||||
if len(files) != 2 {
|
||||
t.Errorf("expected 2 files for chunk1, got %d", len(files))
|
||||
}
|
||||
|
||||
// Test chunk2 (used by file1 and file2)
|
||||
files, err = repo.GetByChunkHash(ctx, "chunk2")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get files for chunk2: %v", err)
|
||||
}
|
||||
if len(files) != 2 {
|
||||
t.Errorf("expected 2 files for chunk2, got %d", len(files))
|
||||
}
|
||||
|
||||
// Test file2 chunks
|
||||
chunks, err := repo.GetByFilePath(ctx, "/file2.txt")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get chunks for file2: %v", err)
|
||||
}
|
||||
if len(chunks) != 3 {
|
||||
t.Errorf("expected 3 chunks for file2, got %d", len(chunks))
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user