Fix manifest generation to not encrypt manifests
- Manifests are now only compressed (not encrypted) so pruning operations can work without private keys - Updated generateBlobManifest to use zstd compression directly - Updated prune command to handle unencrypted manifests - Updated snapshot list command to handle new manifest format - Updated documentation to reflect manifest.json.zst (not .age) - Removed unnecessary VAULTIK_PRIVATE_KEY check from prune command
This commit is contained in:
@@ -56,6 +56,7 @@ import (
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"git.eeqj.de/sneak/vaultik/internal/s3"
|
||||
"github.com/dustin/go-humanize"
|
||||
"github.com/klauspost/compress/zstd"
|
||||
"go.uber.org/fx"
|
||||
)
|
||||
|
||||
@@ -277,8 +278,8 @@ func (sm *SnapshotManager) ExportSnapshotMetadata(ctx context.Context, dbPath st
|
||||
"duration", dbUploadDuration,
|
||||
"speed", humanize.SI(dbUploadSpeed, "bps"))
|
||||
|
||||
// Upload blob manifest (compressed and encrypted)
|
||||
manifestKey := fmt.Sprintf("metadata/%s/manifest.json.zst.age", snapshotID)
|
||||
// Upload blob manifest (compressed only, not encrypted)
|
||||
manifestKey := fmt.Sprintf("metadata/%s/manifest.json.zst", snapshotID)
|
||||
log.Debug("Uploading blob manifest to S3", "key", manifestKey, "size", len(blobManifest))
|
||||
manifestUploadStart := time.Now()
|
||||
if err := sm.s3Client.PutObject(ctx, manifestKey, bytes.NewReader(blobManifest)); err != nil {
|
||||
@@ -566,25 +567,33 @@ func (sm *SnapshotManager) generateBlobManifest(ctx context.Context, dbPath stri
|
||||
}
|
||||
log.Debug("JSON manifest created", "size", len(jsonData))
|
||||
|
||||
// Compress and encrypt with blobgen
|
||||
log.Debug("Compressing and encrypting manifest")
|
||||
// Compress only (no encryption) - manifests must be readable without private keys for pruning
|
||||
log.Debug("Compressing manifest")
|
||||
|
||||
result, err := blobgen.CompressData(jsonData, sm.config.CompressionLevel, sm.config.AgeRecipients)
|
||||
var compressedBuf bytes.Buffer
|
||||
writer, err := zstd.NewWriter(&compressedBuf, zstd.WithEncoderLevel(zstd.EncoderLevelFromZstd(sm.config.CompressionLevel)))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("compressing manifest: %w", err)
|
||||
return nil, fmt.Errorf("creating zstd writer: %w", err)
|
||||
}
|
||||
log.Debug("Manifest compressed and encrypted",
|
||||
if _, err := writer.Write(jsonData); err != nil {
|
||||
_ = writer.Close()
|
||||
return nil, fmt.Errorf("writing compressed data: %w", err)
|
||||
}
|
||||
if err := writer.Close(); err != nil {
|
||||
return nil, fmt.Errorf("closing zstd writer: %w", err)
|
||||
}
|
||||
|
||||
log.Debug("Manifest compressed",
|
||||
"original_size", len(jsonData),
|
||||
"compressed_size", result.CompressedSize,
|
||||
"hash", result.SHA256)
|
||||
"compressed_size", compressedBuf.Len())
|
||||
|
||||
log.Info("Generated blob manifest",
|
||||
"snapshot_id", snapshotID,
|
||||
"blob_count", len(blobs),
|
||||
"json_size", len(jsonData),
|
||||
"compressed_size", result.CompressedSize)
|
||||
"compressed_size", compressedBuf.Len())
|
||||
|
||||
return result.Data, nil
|
||||
return compressedBuf.Bytes(), nil
|
||||
}
|
||||
|
||||
// compressData compresses data using zstd
|
||||
|
||||
Reference in New Issue
Block a user