Refactor blob storage to use UUID primary keys and implement streaming chunking

- Changed blob table to use ID (UUID) as primary key instead of hash
- Blob records are now created at packing start, enabling immediate chunk associations
- Implemented streaming chunking to process large files without memory exhaustion
- Fixed blob manifest generation to include all referenced blobs
- Updated all foreign key references from blob_hash to blob_id
- Added progress reporting and improved error handling
- Enforced encryption requirement for all blob packing
- Updated tests to use test encryption keys
- Added Cyrillic transliteration to README
This commit is contained in:
2025-07-22 07:43:39 +02:00
parent 26db096913
commit 86b533d6ee
49 changed files with 5709 additions and 324 deletions

View File

@@ -30,7 +30,8 @@ func TestSnapshotRepository(t *testing.T) {
ID: "2024-01-01T12:00:00Z",
Hostname: "test-host",
VaultikVersion: "1.0.0",
CreatedTS: time.Now().Truncate(time.Second),
StartedAt: time.Now().Truncate(time.Second),
CompletedAt: nil,
FileCount: 100,
ChunkCount: 500,
BlobCount: 10,
@@ -99,7 +100,8 @@ func TestSnapshotRepository(t *testing.T) {
ID: fmt.Sprintf("2024-01-0%dT12:00:00Z", i),
Hostname: "test-host",
VaultikVersion: "1.0.0",
CreatedTS: time.Now().Add(time.Duration(i) * time.Hour).Truncate(time.Second),
StartedAt: time.Now().Add(time.Duration(i) * time.Hour).Truncate(time.Second),
CompletedAt: nil,
FileCount: int64(100 * i),
ChunkCount: int64(500 * i),
BlobCount: int64(10 * i),
@@ -121,7 +123,7 @@ func TestSnapshotRepository(t *testing.T) {
// Verify order (most recent first)
for i := 0; i < len(recent)-1; i++ {
if recent[i].CreatedTS.Before(recent[i+1].CreatedTS) {
if recent[i].StartedAt.Before(recent[i+1].StartedAt) {
t.Error("snapshots not in descending order")
}
}
@@ -162,7 +164,8 @@ func TestSnapshotRepositoryDuplicate(t *testing.T) {
ID: "2024-01-01T12:00:00Z",
Hostname: "test-host",
VaultikVersion: "1.0.0",
CreatedTS: time.Now().Truncate(time.Second),
StartedAt: time.Now().Truncate(time.Second),
CompletedAt: nil,
FileCount: 100,
ChunkCount: 500,
BlobCount: 10,