This commit represents a significant architectural overhaul of vaultik: Database Schema Changes: - Switch files table to use UUID primary keys instead of path-based keys - Add UUID primary keys to blobs table for immediate chunk association - Update all foreign key relationships to use UUIDs - Add comprehensive schema documentation in DATAMODEL.md - Add SQLite busy timeout handling for concurrent operations Streaming and Performance Improvements: - Implement true streaming blob packing without intermediate storage - Add streaming chunk processing to reduce memory usage - Improve progress reporting with real-time metrics - Add upload metrics tracking in new uploads table CLI Refactoring: - Restructure CLI to use subcommands: snapshot create/list/purge/verify - Add store info command for S3 configuration display - Add custom duration parser supporting days/weeks/months/years - Remove old backup.go in favor of enhanced snapshot.go - Add --cron flag for silent operation Configuration Changes: - Remove unused index_prefix configuration option - Add support for snapshot pruning retention policies - Improve configuration validation and error messages Testing Improvements: - Add comprehensive repository tests with edge cases - Add cascade delete debugging tests - Fix concurrent operation tests to use SQLite busy timeout - Remove tolerance for SQLITE_BUSY errors in tests Documentation: - Add MIT LICENSE file - Update README with new command structure - Add comprehensive DATAMODEL.md explaining database schema - Update DESIGN.md with UUID-based architecture Other Changes: - Add test-config.yml for testing - Update Makefile with better test output formatting - Fix various race conditions in concurrent operations - Improve error handling throughout
160 lines
4.0 KiB
Go
160 lines
4.0 KiB
Go
package cli
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.eeqj.de/sneak/vaultik/internal/log"
|
|
"git.eeqj.de/sneak/vaultik/internal/s3"
|
|
"github.com/spf13/cobra"
|
|
"go.uber.org/fx"
|
|
)
|
|
|
|
// StoreApp contains dependencies for store commands
|
|
type StoreApp struct {
|
|
S3Client *s3.Client
|
|
Shutdowner fx.Shutdowner
|
|
}
|
|
|
|
// NewStoreCommand creates the store command and subcommands
|
|
func NewStoreCommand() *cobra.Command {
|
|
cmd := &cobra.Command{
|
|
Use: "store",
|
|
Short: "Storage information commands",
|
|
Long: "Commands for viewing information about the S3 storage backend",
|
|
}
|
|
|
|
// Add subcommands
|
|
cmd.AddCommand(newStoreInfoCommand())
|
|
|
|
return cmd
|
|
}
|
|
|
|
// newStoreInfoCommand creates the 'store info' subcommand
|
|
func newStoreInfoCommand() *cobra.Command {
|
|
return &cobra.Command{
|
|
Use: "info",
|
|
Short: "Display storage information",
|
|
Long: "Shows S3 bucket configuration and storage statistics including snapshots and blobs",
|
|
RunE: func(cmd *cobra.Command, args []string) error {
|
|
return runWithApp(cmd.Context(), func(app *StoreApp) error {
|
|
return app.Info(cmd.Context())
|
|
})
|
|
},
|
|
}
|
|
}
|
|
|
|
// Info displays storage information
|
|
func (app *StoreApp) Info(ctx context.Context) error {
|
|
// Get bucket info
|
|
bucketName := app.S3Client.BucketName()
|
|
endpoint := app.S3Client.Endpoint()
|
|
|
|
fmt.Printf("Storage Information\n")
|
|
fmt.Printf("==================\n\n")
|
|
fmt.Printf("S3 Configuration:\n")
|
|
fmt.Printf(" Endpoint: %s\n", endpoint)
|
|
fmt.Printf(" Bucket: %s\n\n", bucketName)
|
|
|
|
// Count snapshots by listing metadata/ prefix
|
|
snapshotCount := 0
|
|
snapshotCh := app.S3Client.ListObjectsStream(ctx, "metadata/", true)
|
|
snapshotDirs := make(map[string]bool)
|
|
|
|
for object := range snapshotCh {
|
|
if object.Err != nil {
|
|
return fmt.Errorf("listing snapshots: %w", object.Err)
|
|
}
|
|
// Extract snapshot ID from path like metadata/2024-01-15-143052-hostname/
|
|
parts := strings.Split(object.Key, "/")
|
|
if len(parts) >= 2 && parts[0] == "metadata" && parts[1] != "" {
|
|
snapshotDirs[parts[1]] = true
|
|
}
|
|
}
|
|
snapshotCount = len(snapshotDirs)
|
|
|
|
// Count blobs and calculate total size by listing blobs/ prefix
|
|
blobCount := 0
|
|
var totalSize int64
|
|
|
|
blobCh := app.S3Client.ListObjectsStream(ctx, "blobs/", false)
|
|
for object := range blobCh {
|
|
if object.Err != nil {
|
|
return fmt.Errorf("listing blobs: %w", object.Err)
|
|
}
|
|
if !strings.HasSuffix(object.Key, "/") { // Skip directories
|
|
blobCount++
|
|
totalSize += object.Size
|
|
}
|
|
}
|
|
|
|
fmt.Printf("Storage Statistics:\n")
|
|
fmt.Printf(" Snapshots: %d\n", snapshotCount)
|
|
fmt.Printf(" Blobs: %d\n", blobCount)
|
|
fmt.Printf(" Total Size: %s\n", formatBytes(totalSize))
|
|
|
|
return nil
|
|
}
|
|
|
|
// formatBytes formats bytes into human-readable format
|
|
func formatBytes(bytes int64) string {
|
|
const unit = 1024
|
|
if bytes < unit {
|
|
return fmt.Sprintf("%d B", bytes)
|
|
}
|
|
div, exp := int64(unit), 0
|
|
for n := bytes / unit; n >= unit; n /= unit {
|
|
div *= unit
|
|
exp++
|
|
}
|
|
return fmt.Sprintf("%.1f %cB", float64(bytes)/float64(div), "KMGTPE"[exp])
|
|
}
|
|
|
|
// runWithApp creates the FX app and runs the given function
|
|
func runWithApp(ctx context.Context, fn func(*StoreApp) error) error {
|
|
var result error
|
|
rootFlags := GetRootFlags()
|
|
|
|
// Use unified config resolution
|
|
configPath, err := ResolveConfigPath()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
err = RunWithApp(ctx, AppOptions{
|
|
ConfigPath: configPath,
|
|
LogOptions: log.LogOptions{
|
|
Verbose: rootFlags.Verbose,
|
|
Debug: rootFlags.Debug,
|
|
},
|
|
Modules: []fx.Option{
|
|
s3.Module,
|
|
fx.Provide(func(s3Client *s3.Client, shutdowner fx.Shutdowner) *StoreApp {
|
|
return &StoreApp{
|
|
S3Client: s3Client,
|
|
Shutdowner: shutdowner,
|
|
}
|
|
}),
|
|
},
|
|
Invokes: []fx.Option{
|
|
fx.Invoke(func(app *StoreApp, shutdowner fx.Shutdowner) {
|
|
result = fn(app)
|
|
// Shutdown after command completes
|
|
go func() {
|
|
time.Sleep(100 * time.Millisecond) // Brief delay to ensure clean shutdown
|
|
if err := shutdowner.Shutdown(); err != nil {
|
|
log.Error("Failed to shutdown", "error", err)
|
|
}
|
|
}()
|
|
}),
|
|
},
|
|
})
|
|
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return result
|
|
}
|