Major refactoring: UUID-based storage, streaming architecture, and CLI improvements

This commit represents a significant architectural overhaul of vaultik:

Database Schema Changes:
- Switch files table to use UUID primary keys instead of path-based keys
- Add UUID primary keys to blobs table for immediate chunk association
- Update all foreign key relationships to use UUIDs
- Add comprehensive schema documentation in DATAMODEL.md
- Add SQLite busy timeout handling for concurrent operations

Streaming and Performance Improvements:
- Implement true streaming blob packing without intermediate storage
- Add streaming chunk processing to reduce memory usage
- Improve progress reporting with real-time metrics
- Add upload metrics tracking in new uploads table

CLI Refactoring:
- Restructure CLI to use subcommands: snapshot create/list/purge/verify
- Add store info command for S3 configuration display
- Add custom duration parser supporting days/weeks/months/years
- Remove old backup.go in favor of enhanced snapshot.go
- Add --cron flag for silent operation

Configuration Changes:
- Remove unused index_prefix configuration option
- Add support for snapshot pruning retention policies
- Improve configuration validation and error messages

Testing Improvements:
- Add comprehensive repository tests with edge cases
- Add cascade delete debugging tests
- Fix concurrent operation tests to use SQLite busy timeout
- Remove tolerance for SQLITE_BUSY errors in tests

Documentation:
- Add MIT LICENSE file
- Update README with new command structure
- Add comprehensive DATAMODEL.md explaining database schema
- Update DESIGN.md with UUID-based architecture

Other Changes:
- Add test-config.yml for testing
- Update Makefile with better test output formatting
- Fix various race conditions in concurrent operations
- Improve error handling throughout

internal/database/repository_debug_test.go

@@ -0,0 +1,165 @@
+package database
+
+import (
+	"context"
+	"testing"
+	"time"
+)
+
+// TestOrphanedFileCleanupDebug tests orphaned file cleanup with debug output
+func TestOrphanedFileCleanupDebug(t *testing.T) {
+	db, cleanup := setupTestDB(t)
+	defer cleanup()
+
+	ctx := context.Background()
+	repos := NewRepositories(db)
+
+	// Create files
+	file1 := &File{
+		Path:  "/orphaned.txt",
+		MTime: time.Now().Truncate(time.Second),
+		CTime: time.Now().Truncate(time.Second),
+		Size:  1024,
+		Mode:  0644,
+		UID:   1000,
+		GID:   1000,
+	}
+	file2 := &File{
+		Path:  "/referenced.txt",
+		MTime: time.Now().Truncate(time.Second),
+		CTime: time.Now().Truncate(time.Second),
+		Size:  2048,
+		Mode:  0644,
+		UID:   1000,
+		GID:   1000,
+	}
+
+	err := repos.Files.Create(ctx, nil, file1)
+	if err != nil {
+		t.Fatalf("failed to create file1: %v", err)
+	}
+	t.Logf("Created file1 with ID: %s", file1.ID)
+
+	err = repos.Files.Create(ctx, nil, file2)
+	if err != nil {
+		t.Fatalf("failed to create file2: %v", err)
+	}
+	t.Logf("Created file2 with ID: %s", file2.ID)
+
+	// Create a snapshot and reference only file2
+	snapshot := &Snapshot{
+		ID:        "test-snapshot",
+		Hostname:  "test-host",
+		StartedAt: time.Now(),
+	}
+	err = repos.Snapshots.Create(ctx, nil, snapshot)
+	if err != nil {
+		t.Fatalf("failed to create snapshot: %v", err)
+	}
+	t.Logf("Created snapshot: %s", snapshot.ID)
+
+	// Check snapshot_files before adding
+	var count int
+	err = db.conn.QueryRow("SELECT COUNT(*) FROM snapshot_files").Scan(&count)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Logf("snapshot_files count before add: %d", count)
+
+	// Add file2 to snapshot
+	err = repos.Snapshots.AddFileByID(ctx, nil, snapshot.ID, file2.ID)
+	if err != nil {
+		t.Fatalf("failed to add file to snapshot: %v", err)
+	}
+	t.Logf("Added file2 to snapshot")
+
+	// Check snapshot_files after adding
+	err = db.conn.QueryRow("SELECT COUNT(*) FROM snapshot_files").Scan(&count)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Logf("snapshot_files count after add: %d", count)
+
+	// Check which files are referenced
+	rows, err := db.conn.Query("SELECT file_id FROM snapshot_files")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() {
+		if err := rows.Close(); err != nil {
+			t.Logf("failed to close rows: %v", err)
+		}
+	}()
+	t.Log("Files in snapshot_files:")
+	for rows.Next() {
+		var fileID string
+		if err := rows.Scan(&fileID); err != nil {
+			t.Fatal(err)
+		}
+		t.Logf("  - %s", fileID)
+	}
+
+	// Check files before cleanup
+	err = db.conn.QueryRow("SELECT COUNT(*) FROM files").Scan(&count)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Logf("Files count before cleanup: %d", count)
+
+	// Run orphaned cleanup
+	err = repos.Files.DeleteOrphaned(ctx)
+	if err != nil {
+		t.Fatalf("failed to delete orphaned files: %v", err)
+	}
+	t.Log("Ran orphaned cleanup")
+
+	// Check files after cleanup
+	err = db.conn.QueryRow("SELECT COUNT(*) FROM files").Scan(&count)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Logf("Files count after cleanup: %d", count)
+
+	// List remaining files
+	files, err := repos.Files.ListByPrefix(ctx, "/")
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log("Remaining files:")
+	for _, f := range files {
+		t.Logf("  - ID: %s, Path: %s", f.ID, f.Path)
+	}
+
+	// Check that orphaned file is gone
+	orphanedFile, err := repos.Files.GetByID(ctx, file1.ID)
+	if err != nil {
+		t.Fatalf("error getting file: %v", err)
+	}
+	if orphanedFile != nil {
+		t.Error("orphaned file should have been deleted")
+		// Let's check why it wasn't deleted
+		var exists bool
+		err = db.conn.QueryRow(`
+			SELECT EXISTS(
+				SELECT 1 FROM snapshot_files 
+				WHERE file_id = ?
+			)`, file1.ID).Scan(&exists)
+		if err != nil {
+			t.Fatal(err)
+		}
+		t.Logf("File1 exists in snapshot_files: %v", exists)
+	} else {
+		t.Log("Orphaned file was correctly deleted")
+	}
+
+	// Check that referenced file still exists
+	referencedFile, err := repos.Files.GetByID(ctx, file2.ID)
+	if err != nil {
+		t.Fatalf("error getting file: %v", err)
+	}
+	if referencedFile == nil {
+		t.Error("referenced file should not have been deleted")
+	} else {
+		t.Log("Referenced file correctly remains")
+	}
+}