Add pluggable storage backend, PID locking, and improved scan progress
Storage backend: - Add internal/storage package with Storer interface - Implement FileStorer for local filesystem storage (file:// URLs) - Implement S3Storer wrapping existing s3.Client - Support storage_url config field (s3:// or file://) - Migrate all consumers to use storage.Storer interface PID locking: - Add internal/pidlock package to prevent concurrent instances - Acquire lock before app start, release on exit - Detect stale locks from crashed processes Scan progress improvements: - Add fast file enumeration pass before stat() phase - Use enumerated set for deletion detection (no extra filesystem access) - Show progress with percentage, files/sec, elapsed time, and ETA - Change "changed" to "changed/new" for clarity Config improvements: - Add tilde expansion for paths (~/) - Use xdg library for platform-specific default index path
This commit is contained in:
74
internal/storage/storer.go
Normal file
74
internal/storage/storer.go
Normal file
@@ -0,0 +1,74 @@
|
||||
// Package storage provides a unified interface for storage backends.
|
||||
// It supports both S3-compatible object storage and local filesystem storage,
|
||||
// allowing Vaultik to store backups in either location with the same API.
|
||||
//
|
||||
// Storage backends are selected via URL:
|
||||
// - s3://bucket/prefix?endpoint=host®ion=r - S3-compatible storage
|
||||
// - file:///path/to/backup - Local filesystem storage
|
||||
//
|
||||
// Both backends implement the Storer interface and support progress reporting
|
||||
// during upload/write operations.
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
)
|
||||
|
||||
// ErrNotFound is returned when an object does not exist.
|
||||
var ErrNotFound = errors.New("object not found")
|
||||
|
||||
// ProgressCallback is called during storage operations with bytes transferred so far.
|
||||
// Return an error to cancel the operation.
|
||||
type ProgressCallback func(bytesTransferred int64) error
|
||||
|
||||
// ObjectInfo contains metadata about a stored object.
|
||||
type ObjectInfo struct {
|
||||
Key string // Object key/path
|
||||
Size int64 // Size in bytes
|
||||
Err error // Error for streaming results (nil on success)
|
||||
}
|
||||
|
||||
// StorageInfo provides human-readable storage configuration.
|
||||
type StorageInfo struct {
|
||||
Type string // "s3" or "file"
|
||||
Location string // endpoint/bucket for S3, base path for filesystem
|
||||
}
|
||||
|
||||
// Storer defines the interface for storage backends.
|
||||
// All paths are relative to the storage root (bucket/prefix for S3, base directory for filesystem).
|
||||
type Storer interface {
|
||||
// Put stores data at the specified key.
|
||||
// Parent directories are created automatically for filesystem backends.
|
||||
Put(ctx context.Context, key string, data io.Reader) error
|
||||
|
||||
// PutWithProgress stores data with progress reporting.
|
||||
// Size must be the exact size of the data to store.
|
||||
// The progress callback is called periodically with bytes transferred.
|
||||
PutWithProgress(ctx context.Context, key string, data io.Reader, size int64, progress ProgressCallback) error
|
||||
|
||||
// Get retrieves data from the specified key.
|
||||
// The caller must close the returned ReadCloser.
|
||||
// Returns ErrNotFound if the object does not exist.
|
||||
Get(ctx context.Context, key string) (io.ReadCloser, error)
|
||||
|
||||
// Stat returns metadata about an object without retrieving its contents.
|
||||
// Returns ErrNotFound if the object does not exist.
|
||||
Stat(ctx context.Context, key string) (*ObjectInfo, error)
|
||||
|
||||
// Delete removes an object. No error is returned if the object doesn't exist.
|
||||
Delete(ctx context.Context, key string) error
|
||||
|
||||
// List returns all keys with the given prefix.
|
||||
// For large result sets, prefer ListStream.
|
||||
List(ctx context.Context, prefix string) ([]string, error)
|
||||
|
||||
// ListStream returns a channel of ObjectInfo for large result sets.
|
||||
// The channel is closed when listing completes.
|
||||
// If an error occurs during listing, the final item will have Err set.
|
||||
ListStream(ctx context.Context, prefix string) <-chan ObjectInfo
|
||||
|
||||
// Info returns human-readable storage location information.
|
||||
Info() StorageInfo
|
||||
}
|
||||
Reference in New Issue
Block a user