This commit represents a significant architectural overhaul of vaultik: Database Schema Changes: - Switch files table to use UUID primary keys instead of path-based keys - Add UUID primary keys to blobs table for immediate chunk association - Update all foreign key relationships to use UUIDs - Add comprehensive schema documentation in DATAMODEL.md - Add SQLite busy timeout handling for concurrent operations Streaming and Performance Improvements: - Implement true streaming blob packing without intermediate storage - Add streaming chunk processing to reduce memory usage - Improve progress reporting with real-time metrics - Add upload metrics tracking in new uploads table CLI Refactoring: - Restructure CLI to use subcommands: snapshot create/list/purge/verify - Add store info command for S3 configuration display - Add custom duration parser supporting days/weeks/months/years - Remove old backup.go in favor of enhanced snapshot.go - Add --cron flag for silent operation Configuration Changes: - Remove unused index_prefix configuration option - Add support for snapshot pruning retention policies - Improve configuration validation and error messages Testing Improvements: - Add comprehensive repository tests with edge cases - Add cascade delete debugging tests - Fix concurrent operation tests to use SQLite busy timeout - Remove tolerance for SQLITE_BUSY errors in tests Documentation: - Add MIT LICENSE file - Update README with new command structure - Add comprehensive DATAMODEL.md explaining database schema - Update DESIGN.md with UUID-based architecture Other Changes: - Add test-config.yml for testing - Update Makefile with better test output formatting - Fix various race conditions in concurrent operations - Improve error handling throughout
166 lines
3.9 KiB
Go
166 lines
3.9 KiB
Go
package database
|
|
|
|
import (
|
|
"context"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// TestOrphanedFileCleanupDebug tests orphaned file cleanup with debug output
|
|
func TestOrphanedFileCleanupDebug(t *testing.T) {
|
|
db, cleanup := setupTestDB(t)
|
|
defer cleanup()
|
|
|
|
ctx := context.Background()
|
|
repos := NewRepositories(db)
|
|
|
|
// Create files
|
|
file1 := &File{
|
|
Path: "/orphaned.txt",
|
|
MTime: time.Now().Truncate(time.Second),
|
|
CTime: time.Now().Truncate(time.Second),
|
|
Size: 1024,
|
|
Mode: 0644,
|
|
UID: 1000,
|
|
GID: 1000,
|
|
}
|
|
file2 := &File{
|
|
Path: "/referenced.txt",
|
|
MTime: time.Now().Truncate(time.Second),
|
|
CTime: time.Now().Truncate(time.Second),
|
|
Size: 2048,
|
|
Mode: 0644,
|
|
UID: 1000,
|
|
GID: 1000,
|
|
}
|
|
|
|
err := repos.Files.Create(ctx, nil, file1)
|
|
if err != nil {
|
|
t.Fatalf("failed to create file1: %v", err)
|
|
}
|
|
t.Logf("Created file1 with ID: %s", file1.ID)
|
|
|
|
err = repos.Files.Create(ctx, nil, file2)
|
|
if err != nil {
|
|
t.Fatalf("failed to create file2: %v", err)
|
|
}
|
|
t.Logf("Created file2 with ID: %s", file2.ID)
|
|
|
|
// Create a snapshot and reference only file2
|
|
snapshot := &Snapshot{
|
|
ID: "test-snapshot",
|
|
Hostname: "test-host",
|
|
StartedAt: time.Now(),
|
|
}
|
|
err = repos.Snapshots.Create(ctx, nil, snapshot)
|
|
if err != nil {
|
|
t.Fatalf("failed to create snapshot: %v", err)
|
|
}
|
|
t.Logf("Created snapshot: %s", snapshot.ID)
|
|
|
|
// Check snapshot_files before adding
|
|
var count int
|
|
err = db.conn.QueryRow("SELECT COUNT(*) FROM snapshot_files").Scan(&count)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Logf("snapshot_files count before add: %d", count)
|
|
|
|
// Add file2 to snapshot
|
|
err = repos.Snapshots.AddFileByID(ctx, nil, snapshot.ID, file2.ID)
|
|
if err != nil {
|
|
t.Fatalf("failed to add file to snapshot: %v", err)
|
|
}
|
|
t.Logf("Added file2 to snapshot")
|
|
|
|
// Check snapshot_files after adding
|
|
err = db.conn.QueryRow("SELECT COUNT(*) FROM snapshot_files").Scan(&count)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Logf("snapshot_files count after add: %d", count)
|
|
|
|
// Check which files are referenced
|
|
rows, err := db.conn.Query("SELECT file_id FROM snapshot_files")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer func() {
|
|
if err := rows.Close(); err != nil {
|
|
t.Logf("failed to close rows: %v", err)
|
|
}
|
|
}()
|
|
t.Log("Files in snapshot_files:")
|
|
for rows.Next() {
|
|
var fileID string
|
|
if err := rows.Scan(&fileID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Logf(" - %s", fileID)
|
|
}
|
|
|
|
// Check files before cleanup
|
|
err = db.conn.QueryRow("SELECT COUNT(*) FROM files").Scan(&count)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Logf("Files count before cleanup: %d", count)
|
|
|
|
// Run orphaned cleanup
|
|
err = repos.Files.DeleteOrphaned(ctx)
|
|
if err != nil {
|
|
t.Fatalf("failed to delete orphaned files: %v", err)
|
|
}
|
|
t.Log("Ran orphaned cleanup")
|
|
|
|
// Check files after cleanup
|
|
err = db.conn.QueryRow("SELECT COUNT(*) FROM files").Scan(&count)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Logf("Files count after cleanup: %d", count)
|
|
|
|
// List remaining files
|
|
files, err := repos.Files.ListByPrefix(ctx, "/")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Log("Remaining files:")
|
|
for _, f := range files {
|
|
t.Logf(" - ID: %s, Path: %s", f.ID, f.Path)
|
|
}
|
|
|
|
// Check that orphaned file is gone
|
|
orphanedFile, err := repos.Files.GetByID(ctx, file1.ID)
|
|
if err != nil {
|
|
t.Fatalf("error getting file: %v", err)
|
|
}
|
|
if orphanedFile != nil {
|
|
t.Error("orphaned file should have been deleted")
|
|
// Let's check why it wasn't deleted
|
|
var exists bool
|
|
err = db.conn.QueryRow(`
|
|
SELECT EXISTS(
|
|
SELECT 1 FROM snapshot_files
|
|
WHERE file_id = ?
|
|
)`, file1.ID).Scan(&exists)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Logf("File1 exists in snapshot_files: %v", exists)
|
|
} else {
|
|
t.Log("Orphaned file was correctly deleted")
|
|
}
|
|
|
|
// Check that referenced file still exists
|
|
referencedFile, err := repos.Files.GetByID(ctx, file2.ID)
|
|
if err != nil {
|
|
t.Fatalf("error getting file: %v", err)
|
|
}
|
|
if referencedFile == nil {
|
|
t.Error("referenced file should not have been deleted")
|
|
} else {
|
|
t.Log("Referenced file correctly remains")
|
|
}
|
|
}
|