vaultik/internal/database/chunks.go
sneak 78af626759 Major refactoring: UUID-based storage, streaming architecture, and CLI improvements
This commit represents a significant architectural overhaul of vaultik:

Database Schema Changes:
- Switch files table to use UUID primary keys instead of path-based keys
- Add UUID primary keys to blobs table for immediate chunk association
- Update all foreign key relationships to use UUIDs
- Add comprehensive schema documentation in DATAMODEL.md
- Add SQLite busy timeout handling for concurrent operations

Streaming and Performance Improvements:
- Implement true streaming blob packing without intermediate storage
- Add streaming chunk processing to reduce memory usage
- Improve progress reporting with real-time metrics
- Add upload metrics tracking in new uploads table

CLI Refactoring:
- Restructure CLI to use subcommands: snapshot create/list/purge/verify
- Add store info command for S3 configuration display
- Add custom duration parser supporting days/weeks/months/years
- Remove old backup.go in favor of enhanced snapshot.go
- Add --cron flag for silent operation

Configuration Changes:
- Remove unused index_prefix configuration option
- Add support for snapshot pruning retention policies
- Improve configuration validation and error messages

Testing Improvements:
- Add comprehensive repository tests with edge cases
- Add cascade delete debugging tests
- Fix concurrent operation tests to use SQLite busy timeout
- Remove tolerance for SQLITE_BUSY errors in tests

Documentation:
- Add MIT LICENSE file
- Update README with new command structure
- Add comprehensive DATAMODEL.md explaining database schema
- Update DESIGN.md with UUID-based architecture

Other Changes:
- Add test-config.yml for testing
- Update Makefile with better test output formatting
- Fix various race conditions in concurrent operations
- Improve error handling throughout
2025-07-22 14:56:44 +02:00

167 lines
3.2 KiB
Go

package database
import (
"context"
"database/sql"
"fmt"
"git.eeqj.de/sneak/vaultik/internal/log"
)
type ChunkRepository struct {
db *DB
}
func NewChunkRepository(db *DB) *ChunkRepository {
return &ChunkRepository{db: db}
}
func (r *ChunkRepository) Create(ctx context.Context, tx *sql.Tx, chunk *Chunk) error {
query := `
INSERT INTO chunks (chunk_hash, sha256, size)
VALUES (?, ?, ?)
ON CONFLICT(chunk_hash) DO NOTHING
`
var err error
if tx != nil {
_, err = tx.ExecContext(ctx, query, chunk.ChunkHash, chunk.SHA256, chunk.Size)
} else {
_, err = r.db.ExecWithLog(ctx, query, chunk.ChunkHash, chunk.SHA256, chunk.Size)
}
if err != nil {
return fmt.Errorf("inserting chunk: %w", err)
}
return nil
}
func (r *ChunkRepository) GetByHash(ctx context.Context, hash string) (*Chunk, error) {
query := `
SELECT chunk_hash, sha256, size
FROM chunks
WHERE chunk_hash = ?
`
var chunk Chunk
err := r.db.conn.QueryRowContext(ctx, query, hash).Scan(
&chunk.ChunkHash,
&chunk.SHA256,
&chunk.Size,
)
if err == sql.ErrNoRows {
return nil, nil
}
if err != nil {
return nil, fmt.Errorf("querying chunk: %w", err)
}
return &chunk, nil
}
func (r *ChunkRepository) GetByHashes(ctx context.Context, hashes []string) ([]*Chunk, error) {
if len(hashes) == 0 {
return nil, nil
}
query := `
SELECT chunk_hash, sha256, size
FROM chunks
WHERE chunk_hash IN (`
args := make([]interface{}, len(hashes))
for i, hash := range hashes {
if i > 0 {
query += ", "
}
query += "?"
args[i] = hash
}
query += ") ORDER BY chunk_hash"
rows, err := r.db.conn.QueryContext(ctx, query, args...)
if err != nil {
return nil, fmt.Errorf("querying chunks: %w", err)
}
defer CloseRows(rows)
var chunks []*Chunk
for rows.Next() {
var chunk Chunk
err := rows.Scan(
&chunk.ChunkHash,
&chunk.SHA256,
&chunk.Size,
)
if err != nil {
return nil, fmt.Errorf("scanning chunk: %w", err)
}
chunks = append(chunks, &chunk)
}
return chunks, rows.Err()
}
func (r *ChunkRepository) ListUnpacked(ctx context.Context, limit int) ([]*Chunk, error) {
query := `
SELECT c.chunk_hash, c.sha256, c.size
FROM chunks c
LEFT JOIN blob_chunks bc ON c.chunk_hash = bc.chunk_hash
WHERE bc.chunk_hash IS NULL
ORDER BY c.chunk_hash
LIMIT ?
`
rows, err := r.db.conn.QueryContext(ctx, query, limit)
if err != nil {
return nil, fmt.Errorf("querying unpacked chunks: %w", err)
}
defer CloseRows(rows)
var chunks []*Chunk
for rows.Next() {
var chunk Chunk
err := rows.Scan(
&chunk.ChunkHash,
&chunk.SHA256,
&chunk.Size,
)
if err != nil {
return nil, fmt.Errorf("scanning chunk: %w", err)
}
chunks = append(chunks, &chunk)
}
return chunks, rows.Err()
}
// DeleteOrphaned deletes chunks that are not referenced by any file
func (r *ChunkRepository) DeleteOrphaned(ctx context.Context) error {
query := `
DELETE FROM chunks
WHERE NOT EXISTS (
SELECT 1 FROM file_chunks
WHERE file_chunks.chunk_hash = chunks.chunk_hash
)
`
result, err := r.db.ExecWithLog(ctx, query)
if err != nil {
return fmt.Errorf("deleting orphaned chunks: %w", err)
}
rowsAffected, _ := result.RowsAffected()
if rowsAffected > 0 {
log.Debug("Deleted orphaned chunks", "count", rowsAffected)
}
return nil
}