Add pluggable storage backend, PID locking, and improved scan progress
Storage backend: - Add internal/storage package with Storer interface - Implement FileStorer for local filesystem storage (file:// URLs) - Implement S3Storer wrapping existing s3.Client - Support storage_url config field (s3:// or file://) - Migrate all consumers to use storage.Storer interface PID locking: - Add internal/pidlock package to prevent concurrent instances - Acquire lock before app start, release on exit - Detect stale locks from crashed processes Scan progress improvements: - Add fast file enumeration pass before stat() phase - Use enumerated set for deletion detection (no extra filesystem access) - Show progress with percentage, files/sec, elapsed time, and ETA - Change "changed" to "changed/new" for clarity Config improvements: - Add tilde expansion for paths (~/) - Use xdg library for platform-specific default index path
This commit is contained in:
@@ -52,7 +52,7 @@ import (
|
||||
"git.eeqj.de/sneak/vaultik/internal/config"
|
||||
"git.eeqj.de/sneak/vaultik/internal/database"
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"git.eeqj.de/sneak/vaultik/internal/s3"
|
||||
"git.eeqj.de/sneak/vaultik/internal/storage"
|
||||
"github.com/dustin/go-humanize"
|
||||
"github.com/spf13/afero"
|
||||
"go.uber.org/fx"
|
||||
@@ -60,27 +60,27 @@ import (
|
||||
|
||||
// SnapshotManager handles snapshot creation and metadata export
|
||||
type SnapshotManager struct {
|
||||
repos *database.Repositories
|
||||
s3Client S3Client
|
||||
config *config.Config
|
||||
fs afero.Fs
|
||||
repos *database.Repositories
|
||||
storage storage.Storer
|
||||
config *config.Config
|
||||
fs afero.Fs
|
||||
}
|
||||
|
||||
// SnapshotManagerParams holds dependencies for NewSnapshotManager
|
||||
type SnapshotManagerParams struct {
|
||||
fx.In
|
||||
|
||||
Repos *database.Repositories
|
||||
S3Client *s3.Client
|
||||
Config *config.Config
|
||||
Repos *database.Repositories
|
||||
Storage storage.Storer
|
||||
Config *config.Config
|
||||
}
|
||||
|
||||
// NewSnapshotManager creates a new snapshot manager for dependency injection
|
||||
func NewSnapshotManager(params SnapshotManagerParams) *SnapshotManager {
|
||||
return &SnapshotManager{
|
||||
repos: params.Repos,
|
||||
s3Client: params.S3Client,
|
||||
config: params.Config,
|
||||
repos: params.Repos,
|
||||
storage: params.Storage,
|
||||
config: params.Config,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -268,7 +268,7 @@ func (sm *SnapshotManager) ExportSnapshotMetadata(ctx context.Context, dbPath st
|
||||
dbKey := fmt.Sprintf("metadata/%s/db.zst.age", snapshotID)
|
||||
|
||||
dbUploadStart := time.Now()
|
||||
if err := sm.s3Client.PutObject(ctx, dbKey, bytes.NewReader(finalData)); err != nil {
|
||||
if err := sm.storage.Put(ctx, dbKey, bytes.NewReader(finalData)); err != nil {
|
||||
return fmt.Errorf("uploading snapshot database: %w", err)
|
||||
}
|
||||
dbUploadDuration := time.Since(dbUploadStart)
|
||||
@@ -282,7 +282,7 @@ func (sm *SnapshotManager) ExportSnapshotMetadata(ctx context.Context, dbPath st
|
||||
// Upload blob manifest (compressed only, not encrypted)
|
||||
manifestKey := fmt.Sprintf("metadata/%s/manifest.json.zst", snapshotID)
|
||||
manifestUploadStart := time.Now()
|
||||
if err := sm.s3Client.PutObject(ctx, manifestKey, bytes.NewReader(blobManifest)); err != nil {
|
||||
if err := sm.storage.Put(ctx, manifestKey, bytes.NewReader(blobManifest)); err != nil {
|
||||
return fmt.Errorf("uploading blob manifest: %w", err)
|
||||
}
|
||||
manifestUploadDuration := time.Since(manifestUploadStart)
|
||||
@@ -635,11 +635,11 @@ func (sm *SnapshotManager) CleanupIncompleteSnapshots(ctx context.Context, hostn
|
||||
|
||||
log.Info("Found incomplete snapshots", "count", len(incompleteSnapshots))
|
||||
|
||||
// Check each incomplete snapshot for metadata in S3
|
||||
// Check each incomplete snapshot for metadata in storage
|
||||
for _, snapshot := range incompleteSnapshots {
|
||||
// Check if metadata exists in S3
|
||||
// Check if metadata exists in storage
|
||||
metadataKey := fmt.Sprintf("metadata/%s/db.zst", snapshot.ID)
|
||||
_, err := sm.s3Client.StatObject(ctx, metadataKey)
|
||||
_, err := sm.storage.Stat(ctx, metadataKey)
|
||||
|
||||
if err != nil {
|
||||
// Metadata doesn't exist in S3 - this is an incomplete snapshot
|
||||
|
||||
Reference in New Issue
Block a user