Major refactoring: UUID-based storage, streaming architecture, and CLI improvements
This commit represents a significant architectural overhaul of vaultik: Database Schema Changes: - Switch files table to use UUID primary keys instead of path-based keys - Add UUID primary keys to blobs table for immediate chunk association - Update all foreign key relationships to use UUIDs - Add comprehensive schema documentation in DATAMODEL.md - Add SQLite busy timeout handling for concurrent operations Streaming and Performance Improvements: - Implement true streaming blob packing without intermediate storage - Add streaming chunk processing to reduce memory usage - Improve progress reporting with real-time metrics - Add upload metrics tracking in new uploads table CLI Refactoring: - Restructure CLI to use subcommands: snapshot create/list/purge/verify - Add store info command for S3 configuration display - Add custom duration parser supporting days/weeks/months/years - Remove old backup.go in favor of enhanced snapshot.go - Add --cron flag for silent operation Configuration Changes: - Remove unused index_prefix configuration option - Add support for snapshot pruning retention policies - Improve configuration validation and error messages Testing Improvements: - Add comprehensive repository tests with edge cases - Add cascade delete debugging tests - Fix concurrent operation tests to use SQLite busy timeout - Remove tolerance for SQLITE_BUSY errors in tests Documentation: - Add MIT LICENSE file - Update README with new command structure - Add comprehensive DATAMODEL.md explaining database schema - Update DESIGN.md with UUID-based architecture Other Changes: - Add test-config.yml for testing - Update Makefile with better test output formatting - Fix various race conditions in concurrent operations - Improve error handling throughout
This commit is contained in:
159
internal/cli/store.go
Normal file
159
internal/cli/store.go
Normal file
@@ -0,0 +1,159 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"git.eeqj.de/sneak/vaultik/internal/s3"
|
||||
"github.com/spf13/cobra"
|
||||
"go.uber.org/fx"
|
||||
)
|
||||
|
||||
// StoreApp contains dependencies for store commands
|
||||
type StoreApp struct {
|
||||
S3Client *s3.Client
|
||||
Shutdowner fx.Shutdowner
|
||||
}
|
||||
|
||||
// NewStoreCommand creates the store command and subcommands
|
||||
func NewStoreCommand() *cobra.Command {
|
||||
cmd := &cobra.Command{
|
||||
Use: "store",
|
||||
Short: "Storage information commands",
|
||||
Long: "Commands for viewing information about the S3 storage backend",
|
||||
}
|
||||
|
||||
// Add subcommands
|
||||
cmd.AddCommand(newStoreInfoCommand())
|
||||
|
||||
return cmd
|
||||
}
|
||||
|
||||
// newStoreInfoCommand creates the 'store info' subcommand
|
||||
func newStoreInfoCommand() *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "info",
|
||||
Short: "Display storage information",
|
||||
Long: "Shows S3 bucket configuration and storage statistics including snapshots and blobs",
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
return runWithApp(cmd.Context(), func(app *StoreApp) error {
|
||||
return app.Info(cmd.Context())
|
||||
})
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Info displays storage information
|
||||
func (app *StoreApp) Info(ctx context.Context) error {
|
||||
// Get bucket info
|
||||
bucketName := app.S3Client.BucketName()
|
||||
endpoint := app.S3Client.Endpoint()
|
||||
|
||||
fmt.Printf("Storage Information\n")
|
||||
fmt.Printf("==================\n\n")
|
||||
fmt.Printf("S3 Configuration:\n")
|
||||
fmt.Printf(" Endpoint: %s\n", endpoint)
|
||||
fmt.Printf(" Bucket: %s\n\n", bucketName)
|
||||
|
||||
// Count snapshots by listing metadata/ prefix
|
||||
snapshotCount := 0
|
||||
snapshotCh := app.S3Client.ListObjectsStream(ctx, "metadata/", true)
|
||||
snapshotDirs := make(map[string]bool)
|
||||
|
||||
for object := range snapshotCh {
|
||||
if object.Err != nil {
|
||||
return fmt.Errorf("listing snapshots: %w", object.Err)
|
||||
}
|
||||
// Extract snapshot ID from path like metadata/2024-01-15-143052-hostname/
|
||||
parts := strings.Split(object.Key, "/")
|
||||
if len(parts) >= 2 && parts[0] == "metadata" && parts[1] != "" {
|
||||
snapshotDirs[parts[1]] = true
|
||||
}
|
||||
}
|
||||
snapshotCount = len(snapshotDirs)
|
||||
|
||||
// Count blobs and calculate total size by listing blobs/ prefix
|
||||
blobCount := 0
|
||||
var totalSize int64
|
||||
|
||||
blobCh := app.S3Client.ListObjectsStream(ctx, "blobs/", false)
|
||||
for object := range blobCh {
|
||||
if object.Err != nil {
|
||||
return fmt.Errorf("listing blobs: %w", object.Err)
|
||||
}
|
||||
if !strings.HasSuffix(object.Key, "/") { // Skip directories
|
||||
blobCount++
|
||||
totalSize += object.Size
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Printf("Storage Statistics:\n")
|
||||
fmt.Printf(" Snapshots: %d\n", snapshotCount)
|
||||
fmt.Printf(" Blobs: %d\n", blobCount)
|
||||
fmt.Printf(" Total Size: %s\n", formatBytes(totalSize))
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// formatBytes formats bytes into human-readable format
|
||||
func formatBytes(bytes int64) string {
|
||||
const unit = 1024
|
||||
if bytes < unit {
|
||||
return fmt.Sprintf("%d B", bytes)
|
||||
}
|
||||
div, exp := int64(unit), 0
|
||||
for n := bytes / unit; n >= unit; n /= unit {
|
||||
div *= unit
|
||||
exp++
|
||||
}
|
||||
return fmt.Sprintf("%.1f %cB", float64(bytes)/float64(div), "KMGTPE"[exp])
|
||||
}
|
||||
|
||||
// runWithApp creates the FX app and runs the given function
|
||||
func runWithApp(ctx context.Context, fn func(*StoreApp) error) error {
|
||||
var result error
|
||||
rootFlags := GetRootFlags()
|
||||
|
||||
// Use unified config resolution
|
||||
configPath, err := ResolveConfigPath()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = RunWithApp(ctx, AppOptions{
|
||||
ConfigPath: configPath,
|
||||
LogOptions: log.LogOptions{
|
||||
Verbose: rootFlags.Verbose,
|
||||
Debug: rootFlags.Debug,
|
||||
},
|
||||
Modules: []fx.Option{
|
||||
s3.Module,
|
||||
fx.Provide(func(s3Client *s3.Client, shutdowner fx.Shutdowner) *StoreApp {
|
||||
return &StoreApp{
|
||||
S3Client: s3Client,
|
||||
Shutdowner: shutdowner,
|
||||
}
|
||||
}),
|
||||
},
|
||||
Invokes: []fx.Option{
|
||||
fx.Invoke(func(app *StoreApp, shutdowner fx.Shutdowner) {
|
||||
result = fn(app)
|
||||
// Shutdown after command completes
|
||||
go func() {
|
||||
time.Sleep(100 * time.Millisecond) // Brief delay to ensure clean shutdown
|
||||
if err := shutdowner.Shutdown(); err != nil {
|
||||
log.Error("Failed to shutdown", "error", err)
|
||||
}
|
||||
}()
|
||||
}),
|
||||
},
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return result
|
||||
}
|
||||
Reference in New Issue
Block a user