Fix foreign key constraints and improve snapshot tracking
- Add unified compression/encryption package in internal/blobgen - Update DATAMODEL.md to reflect current schema implementation - Refactor snapshot cleanup into well-named methods for clarity - Add snapshot_id to uploads table to track new blobs per snapshot - Fix blob count reporting for incremental backups - Add DeleteOrphaned method to BlobChunkRepository - Fix cleanup order to respect foreign key constraints - Update tests to reflect schema changes
This commit is contained in:
67
internal/blobgen/compress.go
Normal file
67
internal/blobgen/compress.go
Normal file
@@ -0,0 +1,67 @@
|
||||
package blobgen
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
)
|
||||
|
||||
// CompressResult contains the results of compression
|
||||
type CompressResult struct {
|
||||
Data []byte
|
||||
UncompressedSize int64
|
||||
CompressedSize int64
|
||||
SHA256 string
|
||||
}
|
||||
|
||||
// CompressData compresses and encrypts data, returning the result with hash
|
||||
func CompressData(data []byte, compressionLevel int, recipients []string) (*CompressResult, error) {
|
||||
var buf bytes.Buffer
|
||||
|
||||
// Create writer
|
||||
w, err := NewWriter(&buf, compressionLevel, recipients)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("creating writer: %w", err)
|
||||
}
|
||||
|
||||
// Write data
|
||||
if _, err := w.Write(data); err != nil {
|
||||
_ = w.Close()
|
||||
return nil, fmt.Errorf("writing data: %w", err)
|
||||
}
|
||||
|
||||
// Close to flush
|
||||
if err := w.Close(); err != nil {
|
||||
return nil, fmt.Errorf("closing writer: %w", err)
|
||||
}
|
||||
|
||||
return &CompressResult{
|
||||
Data: buf.Bytes(),
|
||||
UncompressedSize: int64(len(data)),
|
||||
CompressedSize: int64(buf.Len()),
|
||||
SHA256: hex.EncodeToString(w.Sum256()),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// CompressStream compresses and encrypts from reader to writer, returning hash
|
||||
func CompressStream(dst io.Writer, src io.Reader, compressionLevel int, recipients []string) (written int64, hash string, err error) {
|
||||
// Create writer
|
||||
w, err := NewWriter(dst, compressionLevel, recipients)
|
||||
if err != nil {
|
||||
return 0, "", fmt.Errorf("creating writer: %w", err)
|
||||
}
|
||||
defer func() { _ = w.Close() }()
|
||||
|
||||
// Copy data
|
||||
if _, err := io.Copy(w, src); err != nil {
|
||||
return 0, "", fmt.Errorf("copying data: %w", err)
|
||||
}
|
||||
|
||||
// Close to flush
|
||||
if err := w.Close(); err != nil {
|
||||
return 0, "", fmt.Errorf("closing writer: %w", err)
|
||||
}
|
||||
|
||||
return w.BytesWritten(), hex.EncodeToString(w.Sum256()), nil
|
||||
}
|
||||
73
internal/blobgen/reader.go
Normal file
73
internal/blobgen/reader.go
Normal file
@@ -0,0 +1,73 @@
|
||||
package blobgen
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"hash"
|
||||
"io"
|
||||
|
||||
"filippo.io/age"
|
||||
"github.com/klauspost/compress/zstd"
|
||||
)
|
||||
|
||||
// Reader wraps decompression and decryption with SHA256 verification
|
||||
type Reader struct {
|
||||
reader io.Reader
|
||||
decompressor *zstd.Decoder
|
||||
decryptor io.Reader
|
||||
hasher hash.Hash
|
||||
teeReader io.Reader
|
||||
bytesRead int64
|
||||
}
|
||||
|
||||
// NewReader creates a new Reader that decrypts, decompresses, and verifies data
|
||||
func NewReader(r io.Reader, identity age.Identity) (*Reader, error) {
|
||||
// Create decryption reader
|
||||
decReader, err := age.Decrypt(r, identity)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("creating decryption reader: %w", err)
|
||||
}
|
||||
|
||||
// Create decompression reader
|
||||
decompressor, err := zstd.NewReader(decReader)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("creating decompression reader: %w", err)
|
||||
}
|
||||
|
||||
// Create SHA256 hasher
|
||||
hasher := sha256.New()
|
||||
|
||||
// Create tee reader that reads from decompressor and writes to hasher
|
||||
teeReader := io.TeeReader(decompressor, hasher)
|
||||
|
||||
return &Reader{
|
||||
reader: r,
|
||||
decompressor: decompressor,
|
||||
decryptor: decReader,
|
||||
hasher: hasher,
|
||||
teeReader: teeReader,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Read implements io.Reader
|
||||
func (r *Reader) Read(p []byte) (n int, err error) {
|
||||
n, err = r.teeReader.Read(p)
|
||||
r.bytesRead += int64(n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
// Close closes the decompressor
|
||||
func (r *Reader) Close() error {
|
||||
r.decompressor.Close()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Sum256 returns the SHA256 hash of all data read
|
||||
func (r *Reader) Sum256() []byte {
|
||||
return r.hasher.Sum(nil)
|
||||
}
|
||||
|
||||
// BytesRead returns the number of uncompressed bytes read
|
||||
func (r *Reader) BytesRead() int64 {
|
||||
return r.bytesRead
|
||||
}
|
||||
112
internal/blobgen/writer.go
Normal file
112
internal/blobgen/writer.go
Normal file
@@ -0,0 +1,112 @@
|
||||
package blobgen
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"hash"
|
||||
"io"
|
||||
|
||||
"filippo.io/age"
|
||||
"github.com/klauspost/compress/zstd"
|
||||
)
|
||||
|
||||
// Writer wraps compression and encryption with SHA256 hashing
|
||||
type Writer struct {
|
||||
writer io.Writer // Final destination
|
||||
compressor *zstd.Encoder // Compression layer
|
||||
encryptor io.WriteCloser // Encryption layer
|
||||
hasher hash.Hash // SHA256 hasher
|
||||
teeWriter io.Writer // Tees data to hasher
|
||||
compressionLevel int
|
||||
bytesWritten int64
|
||||
}
|
||||
|
||||
// NewWriter creates a new Writer that compresses, encrypts, and hashes data
|
||||
func NewWriter(w io.Writer, compressionLevel int, recipients []string) (*Writer, error) {
|
||||
// Validate compression level
|
||||
if err := validateCompressionLevel(compressionLevel); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Create SHA256 hasher
|
||||
hasher := sha256.New()
|
||||
|
||||
// Parse recipients
|
||||
var ageRecipients []age.Recipient
|
||||
for _, recipient := range recipients {
|
||||
r, err := age.ParseX25519Recipient(recipient)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parsing recipient %s: %w", recipient, err)
|
||||
}
|
||||
ageRecipients = append(ageRecipients, r)
|
||||
}
|
||||
|
||||
// Create encryption writer
|
||||
encWriter, err := age.Encrypt(w, ageRecipients...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("creating encryption writer: %w", err)
|
||||
}
|
||||
|
||||
// Create compression writer with encryption as destination
|
||||
compressor, err := zstd.NewWriter(encWriter,
|
||||
zstd.WithEncoderLevel(zstd.EncoderLevelFromZstd(compressionLevel)),
|
||||
zstd.WithEncoderConcurrency(1), // Use single thread for streaming
|
||||
)
|
||||
if err != nil {
|
||||
_ = encWriter.Close()
|
||||
return nil, fmt.Errorf("creating compression writer: %w", err)
|
||||
}
|
||||
|
||||
// Create tee writer that writes to both compressor and hasher
|
||||
teeWriter := io.MultiWriter(compressor, hasher)
|
||||
|
||||
return &Writer{
|
||||
writer: w,
|
||||
compressor: compressor,
|
||||
encryptor: encWriter,
|
||||
hasher: hasher,
|
||||
teeWriter: teeWriter,
|
||||
compressionLevel: compressionLevel,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Write implements io.Writer
|
||||
func (w *Writer) Write(p []byte) (n int, err error) {
|
||||
n, err = w.teeWriter.Write(p)
|
||||
w.bytesWritten += int64(n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
// Close closes all layers and returns any errors
|
||||
func (w *Writer) Close() error {
|
||||
// Close compressor first
|
||||
if err := w.compressor.Close(); err != nil {
|
||||
return fmt.Errorf("closing compressor: %w", err)
|
||||
}
|
||||
|
||||
// Then close encryptor
|
||||
if err := w.encryptor.Close(); err != nil {
|
||||
return fmt.Errorf("closing encryptor: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Sum256 returns the SHA256 hash of all data written
|
||||
func (w *Writer) Sum256() []byte {
|
||||
return w.hasher.Sum(nil)
|
||||
}
|
||||
|
||||
// BytesWritten returns the number of uncompressed bytes written
|
||||
func (w *Writer) BytesWritten() int64 {
|
||||
return w.bytesWritten
|
||||
}
|
||||
|
||||
func validateCompressionLevel(level int) error {
|
||||
// Zstd compression levels: 1-19 (default is 3)
|
||||
// SpeedFastest = 1, SpeedDefault = 3, SpeedBetterCompression = 7, SpeedBestCompression = 11
|
||||
if level < 1 || level > 19 {
|
||||
return fmt.Errorf("invalid compression level %d: must be between 1 and 19", level)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user