Fix manifest generation to not encrypt manifests

- Manifests are now only compressed (not encrypted) so pruning operations can work without private keys
- Updated generateBlobManifest to use zstd compression directly
- Updated prune command to handle unencrypted manifests
- Updated snapshot list command to handle new manifest format
- Updated documentation to reflect manifest.json.zst (not .age)
- Removed unnecessary VAULTIK_PRIVATE_KEY check from prune command
This commit is contained in:
2025-07-26 02:54:52 +02:00
parent 1d027bde57
commit fb220685a2
4 changed files with 352 additions and 34 deletions

View File

@@ -56,6 +56,7 @@ import (
"git.eeqj.de/sneak/vaultik/internal/log"
"git.eeqj.de/sneak/vaultik/internal/s3"
"github.com/dustin/go-humanize"
"github.com/klauspost/compress/zstd"
"go.uber.org/fx"
)
@@ -277,8 +278,8 @@ func (sm *SnapshotManager) ExportSnapshotMetadata(ctx context.Context, dbPath st
"duration", dbUploadDuration,
"speed", humanize.SI(dbUploadSpeed, "bps"))
// Upload blob manifest (compressed and encrypted)
manifestKey := fmt.Sprintf("metadata/%s/manifest.json.zst.age", snapshotID)
// Upload blob manifest (compressed only, not encrypted)
manifestKey := fmt.Sprintf("metadata/%s/manifest.json.zst", snapshotID)
log.Debug("Uploading blob manifest to S3", "key", manifestKey, "size", len(blobManifest))
manifestUploadStart := time.Now()
if err := sm.s3Client.PutObject(ctx, manifestKey, bytes.NewReader(blobManifest)); err != nil {
@@ -566,25 +567,33 @@ func (sm *SnapshotManager) generateBlobManifest(ctx context.Context, dbPath stri
}
log.Debug("JSON manifest created", "size", len(jsonData))
// Compress and encrypt with blobgen
log.Debug("Compressing and encrypting manifest")
// Compress only (no encryption) - manifests must be readable without private keys for pruning
log.Debug("Compressing manifest")
result, err := blobgen.CompressData(jsonData, sm.config.CompressionLevel, sm.config.AgeRecipients)
var compressedBuf bytes.Buffer
writer, err := zstd.NewWriter(&compressedBuf, zstd.WithEncoderLevel(zstd.EncoderLevelFromZstd(sm.config.CompressionLevel)))
if err != nil {
return nil, fmt.Errorf("compressing manifest: %w", err)
return nil, fmt.Errorf("creating zstd writer: %w", err)
}
log.Debug("Manifest compressed and encrypted",
if _, err := writer.Write(jsonData); err != nil {
_ = writer.Close()
return nil, fmt.Errorf("writing compressed data: %w", err)
}
if err := writer.Close(); err != nil {
return nil, fmt.Errorf("closing zstd writer: %w", err)
}
log.Debug("Manifest compressed",
"original_size", len(jsonData),
"compressed_size", result.CompressedSize,
"hash", result.SHA256)
"compressed_size", compressedBuf.Len())
log.Info("Generated blob manifest",
"snapshot_id", snapshotID,
"blob_count", len(blobs),
"json_size", len(jsonData),
"compressed_size", result.CompressedSize)
"compressed_size", compressedBuf.Len())
return result.Data, nil
return compressedBuf.Bytes(), nil
}
// compressData compresses data using zstd