Refactor blob storage to use UUID primary keys and implement streaming chunking

- Changed blob table to use ID (UUID) as primary key instead of hash
- Blob records are now created at packing start, enabling immediate chunk associations
- Implemented streaming chunking to process large files without memory exhaustion
- Fixed blob manifest generation to include all referenced blobs
- Updated all foreign key references from blob_hash to blob_id
- Added progress reporting and improved error handling
- Enforced encryption requirement for all blob packing
- Updated tests to use test encryption keys
- Added Cyrillic transliteration to README

internal/crypto/encryption.go

@@ -0,0 +1,125 @@
+package crypto
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"sync"
+
+	"filippo.io/age"
+)
+
+// Encryptor provides thread-safe encryption using age
+type Encryptor struct {
+	recipients []age.Recipient
+	mu         sync.RWMutex
+}
+
+// NewEncryptor creates a new encryptor with the given age public keys
+func NewEncryptor(publicKeys []string) (*Encryptor, error) {
+	if len(publicKeys) == 0 {
+		return nil, fmt.Errorf("at least one recipient is required")
+	}
+
+	recipients := make([]age.Recipient, 0, len(publicKeys))
+	for _, key := range publicKeys {
+		recipient, err := age.ParseX25519Recipient(key)
+		if err != nil {
+			return nil, fmt.Errorf("parsing age recipient %s: %w", key, err)
+		}
+		recipients = append(recipients, recipient)
+	}
+
+	return &Encryptor{
+		recipients: recipients,
+	}, nil
+}
+
+// Encrypt encrypts data using age encryption
+func (e *Encryptor) Encrypt(data []byte) ([]byte, error) {
+	e.mu.RLock()
+	recipients := e.recipients
+	e.mu.RUnlock()
+
+	var buf bytes.Buffer
+
+	// Create encrypted writer for all recipients
+	w, err := age.Encrypt(&buf, recipients...)
+	if err != nil {
+		return nil, fmt.Errorf("creating encrypted writer: %w", err)
+	}
+
+	// Write data
+	if _, err := w.Write(data); err != nil {
+		return nil, fmt.Errorf("writing encrypted data: %w", err)
+	}
+
+	// Close to flush
+	if err := w.Close(); err != nil {
+		return nil, fmt.Errorf("closing encrypted writer: %w", err)
+	}
+
+	return buf.Bytes(), nil
+}
+
+// EncryptStream encrypts data from reader to writer
+func (e *Encryptor) EncryptStream(dst io.Writer, src io.Reader) error {
+	e.mu.RLock()
+	recipients := e.recipients
+	e.mu.RUnlock()
+
+	// Create encrypted writer for all recipients
+	w, err := age.Encrypt(dst, recipients...)
+	if err != nil {
+		return fmt.Errorf("creating encrypted writer: %w", err)
+	}
+
+	// Copy data
+	if _, err := io.Copy(w, src); err != nil {
+		return fmt.Errorf("copying encrypted data: %w", err)
+	}
+
+	// Close to flush
+	if err := w.Close(); err != nil {
+		return fmt.Errorf("closing encrypted writer: %w", err)
+	}
+
+	return nil
+}
+
+// EncryptWriter creates a writer that encrypts data written to it
+func (e *Encryptor) EncryptWriter(dst io.Writer) (io.WriteCloser, error) {
+	e.mu.RLock()
+	recipients := e.recipients
+	e.mu.RUnlock()
+
+	// Create encrypted writer for all recipients
+	w, err := age.Encrypt(dst, recipients...)
+	if err != nil {
+		return nil, fmt.Errorf("creating encrypted writer: %w", err)
+	}
+
+	return w, nil
+}
+
+// UpdateRecipients updates the recipients (thread-safe)
+func (e *Encryptor) UpdateRecipients(publicKeys []string) error {
+	if len(publicKeys) == 0 {
+		return fmt.Errorf("at least one recipient is required")
+	}
+
+	recipients := make([]age.Recipient, 0, len(publicKeys))
+	for _, key := range publicKeys {
+		recipient, err := age.ParseX25519Recipient(key)
+		if err != nil {
+			return fmt.Errorf("parsing age recipient %s: %w", key, err)
+		}
+		recipients = append(recipients, recipient)
+	}
+
+	e.mu.Lock()
+	e.recipients = recipients
+	e.mu.Unlock()
+
+	return nil
+}

internal/crypto/encryption_test.go

@@ -0,0 +1,157 @@
+package crypto
+
+import (
+	"bytes"
+	"testing"
+
+	"filippo.io/age"
+)
+
+func TestEncryptor(t *testing.T) {
+	// Generate a test key pair
+	identity, err := age.GenerateX25519Identity()
+	if err != nil {
+		t.Fatalf("failed to generate identity: %v", err)
+	}
+
+	publicKey := identity.Recipient().String()
+
+	// Create encryptor
+	enc, err := NewEncryptor([]string{publicKey})
+	if err != nil {
+		t.Fatalf("failed to create encryptor: %v", err)
+	}
+
+	// Test data
+	plaintext := []byte("Hello, World! This is a test message.")
+
+	// Encrypt
+	ciphertext, err := enc.Encrypt(plaintext)
+	if err != nil {
+		t.Fatalf("failed to encrypt: %v", err)
+	}
+
+	// Verify it's actually encrypted (should be larger and different)
+	if bytes.Equal(plaintext, ciphertext) {
+		t.Error("ciphertext equals plaintext")
+	}
+
+	// Decrypt to verify
+	r, err := age.Decrypt(bytes.NewReader(ciphertext), identity)
+	if err != nil {
+		t.Fatalf("failed to decrypt: %v", err)
+	}
+
+	var decrypted bytes.Buffer
+	if _, err := decrypted.ReadFrom(r); err != nil {
+		t.Fatalf("failed to read decrypted data: %v", err)
+	}
+
+	if !bytes.Equal(plaintext, decrypted.Bytes()) {
+		t.Error("decrypted data doesn't match original")
+	}
+}
+
+func TestEncryptorMultipleRecipients(t *testing.T) {
+	// Generate three test key pairs
+	identity1, err := age.GenerateX25519Identity()
+	if err != nil {
+		t.Fatalf("failed to generate identity1: %v", err)
+	}
+	identity2, err := age.GenerateX25519Identity()
+	if err != nil {
+		t.Fatalf("failed to generate identity2: %v", err)
+	}
+	identity3, err := age.GenerateX25519Identity()
+	if err != nil {
+		t.Fatalf("failed to generate identity3: %v", err)
+	}
+
+	publicKeys := []string{
+		identity1.Recipient().String(),
+		identity2.Recipient().String(),
+		identity3.Recipient().String(),
+	}
+
+	// Create encryptor with multiple recipients
+	enc, err := NewEncryptor(publicKeys)
+	if err != nil {
+		t.Fatalf("failed to create encryptor: %v", err)
+	}
+
+	// Test data
+	plaintext := []byte("Secret message for multiple recipients")
+
+	// Encrypt
+	ciphertext, err := enc.Encrypt(plaintext)
+	if err != nil {
+		t.Fatalf("failed to encrypt: %v", err)
+	}
+
+	// Verify each recipient can decrypt
+	identities := []age.Identity{identity1, identity2, identity3}
+	for i, identity := range identities {
+		r, err := age.Decrypt(bytes.NewReader(ciphertext), identity)
+		if err != nil {
+			t.Fatalf("recipient %d failed to decrypt: %v", i+1, err)
+		}
+
+		var decrypted bytes.Buffer
+		if _, err := decrypted.ReadFrom(r); err != nil {
+			t.Fatalf("recipient %d failed to read decrypted data: %v", i+1, err)
+		}
+
+		if !bytes.Equal(plaintext, decrypted.Bytes()) {
+			t.Errorf("recipient %d: decrypted data doesn't match original", i+1)
+		}
+	}
+}
+
+func TestEncryptorUpdateRecipients(t *testing.T) {
+	// Generate two identities
+	identity1, _ := age.GenerateX25519Identity()
+	identity2, _ := age.GenerateX25519Identity()
+
+	publicKey1 := identity1.Recipient().String()
+	publicKey2 := identity2.Recipient().String()
+
+	// Create encryptor with first key
+	enc, err := NewEncryptor([]string{publicKey1})
+	if err != nil {
+		t.Fatalf("failed to create encryptor: %v", err)
+	}
+
+	// Encrypt with first key
+	plaintext := []byte("test data")
+	ciphertext1, err := enc.Encrypt(plaintext)
+	if err != nil {
+		t.Fatalf("failed to encrypt: %v", err)
+	}
+
+	// Update to second key
+	if err := enc.UpdateRecipients([]string{publicKey2}); err != nil {
+		t.Fatalf("failed to update recipients: %v", err)
+	}
+
+	// Encrypt with second key
+	ciphertext2, err := enc.Encrypt(plaintext)
+	if err != nil {
+		t.Fatalf("failed to encrypt: %v", err)
+	}
+
+	// First ciphertext should only decrypt with first identity
+	if _, err := age.Decrypt(bytes.NewReader(ciphertext1), identity1); err != nil {
+		t.Error("failed to decrypt with identity1")
+	}
+	if _, err := age.Decrypt(bytes.NewReader(ciphertext1), identity2); err == nil {
+		t.Error("should not decrypt with identity2")
+	}
+
+	// Second ciphertext should only decrypt with second identity
+	if _, err := age.Decrypt(bytes.NewReader(ciphertext2), identity2); err != nil {
+		t.Error("failed to decrypt with identity2")
+	}
+	if _, err := age.Decrypt(bytes.NewReader(ciphertext2), identity1); err == nil {
+		t.Error("should not decrypt with identity1")
+	}
+}