vaultik/internal/config/config.go
sneak 86b533d6ee Refactor blob storage to use UUID primary keys and implement streaming chunking
- Changed blob table to use ID (UUID) as primary key instead of hash
- Blob records are now created at packing start, enabling immediate chunk associations
- Implemented streaming chunking to process large files without memory exhaustion
- Fixed blob manifest generation to include all referenced blobs
- Updated all foreign key references from blob_hash to blob_id
- Added progress reporting and improved error handling
- Enforced encryption requirement for all blob packing
- Updated tests to use test encryption keys
- Added Cyrillic transliteration to README
2025-07-22 07:43:39 +02:00

155 lines
4.0 KiB
Go

package config
import (
"fmt"
"os"
"time"
"go.uber.org/fx"
"gopkg.in/yaml.v3"
)
// Config represents the application configuration
type Config struct {
AgeRecipients []string `yaml:"age_recipients"`
BackupInterval time.Duration `yaml:"backup_interval"`
BlobSizeLimit Size `yaml:"blob_size_limit"`
ChunkSize Size `yaml:"chunk_size"`
Exclude []string `yaml:"exclude"`
FullScanInterval time.Duration `yaml:"full_scan_interval"`
Hostname string `yaml:"hostname"`
IndexPath string `yaml:"index_path"`
IndexPrefix string `yaml:"index_prefix"`
MinTimeBetweenRun time.Duration `yaml:"min_time_between_run"`
S3 S3Config `yaml:"s3"`
SourceDirs []string `yaml:"source_dirs"`
CompressionLevel int `yaml:"compression_level"`
}
// S3Config represents S3 storage configuration
type S3Config struct {
Endpoint string `yaml:"endpoint"`
Bucket string `yaml:"bucket"`
Prefix string `yaml:"prefix"`
AccessKeyID string `yaml:"access_key_id"`
SecretAccessKey string `yaml:"secret_access_key"`
Region string `yaml:"region"`
UseSSL bool `yaml:"use_ssl"`
PartSize Size `yaml:"part_size"`
}
// ConfigPath wraps the config file path for fx injection
type ConfigPath string
// New creates a new Config instance
func New(path ConfigPath) (*Config, error) {
if path == "" {
return nil, fmt.Errorf("config path not provided")
}
cfg, err := Load(string(path))
if err != nil {
return nil, fmt.Errorf("failed to load config: %w", err)
}
return cfg, nil
}
// Load reads and parses the configuration file
func Load(path string) (*Config, error) {
data, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("failed to read config file: %w", err)
}
cfg := &Config{
// Set defaults
BlobSizeLimit: Size(10 * 1024 * 1024 * 1024), // 10GB
ChunkSize: Size(10 * 1024 * 1024), // 10MB
BackupInterval: 1 * time.Hour,
FullScanInterval: 24 * time.Hour,
MinTimeBetweenRun: 15 * time.Minute,
IndexPath: "/var/lib/vaultik/index.sqlite",
IndexPrefix: "index/",
CompressionLevel: 3,
}
if err := yaml.Unmarshal(data, cfg); err != nil {
return nil, fmt.Errorf("failed to parse config: %w", err)
}
// Check for environment variable override for IndexPath
if envIndexPath := os.Getenv("VAULTIK_INDEX_PATH"); envIndexPath != "" {
cfg.IndexPath = envIndexPath
}
// Get hostname if not set
if cfg.Hostname == "" {
hostname, err := os.Hostname()
if err != nil {
return nil, fmt.Errorf("failed to get hostname: %w", err)
}
cfg.Hostname = hostname
}
// Set default S3 settings
if cfg.S3.Region == "" {
cfg.S3.Region = "us-east-1"
}
if cfg.S3.PartSize == 0 {
cfg.S3.PartSize = Size(5 * 1024 * 1024) // 5MB
}
if err := cfg.Validate(); err != nil {
return nil, fmt.Errorf("invalid config: %w", err)
}
return cfg, nil
}
// Validate checks if the configuration is valid
func (c *Config) Validate() error {
if len(c.AgeRecipients) == 0 {
return fmt.Errorf("at least one age_recipient is required")
}
if len(c.SourceDirs) == 0 {
return fmt.Errorf("at least one source directory is required")
}
if c.S3.Endpoint == "" {
return fmt.Errorf("s3.endpoint is required")
}
if c.S3.Bucket == "" {
return fmt.Errorf("s3.bucket is required")
}
if c.S3.AccessKeyID == "" {
return fmt.Errorf("s3.access_key_id is required")
}
if c.S3.SecretAccessKey == "" {
return fmt.Errorf("s3.secret_access_key is required")
}
if c.ChunkSize.Int64() < 1024*1024 { // 1MB minimum
return fmt.Errorf("chunk_size must be at least 1MB")
}
if c.BlobSizeLimit.Int64() < c.ChunkSize.Int64() {
return fmt.Errorf("blob_size_limit must be at least chunk_size")
}
if c.CompressionLevel < 1 || c.CompressionLevel > 19 {
return fmt.Errorf("compression_level must be between 1 and 19")
}
return nil
}
// Module exports the config module for fx
var Module = fx.Module("config",
fx.Provide(New),
)