Refactor blob storage to use UUID primary keys and implement streaming chunking
- Changed blob table to use ID (UUID) as primary key instead of hash - Blob records are now created at packing start, enabling immediate chunk associations - Implemented streaming chunking to process large files without memory exhaustion - Fixed blob manifest generation to include all referenced blobs - Updated all foreign key references from blob_hash to blob_id - Added progress reporting and improved error handling - Enforced encryption requirement for all blob packing - Updated tests to use test encryption keys - Added Cyrillic transliteration to README
This commit is contained in:
175
internal/log/log.go
Normal file
175
internal/log/log.go
Normal file
@@ -0,0 +1,175 @@
|
||||
package log
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
|
||||
"golang.org/x/term"
|
||||
)
|
||||
|
||||
// LogLevel represents the logging level
|
||||
type LogLevel int
|
||||
|
||||
const (
|
||||
LevelFatal LogLevel = iota
|
||||
LevelError
|
||||
LevelWarn
|
||||
LevelNotice
|
||||
LevelInfo
|
||||
LevelDebug
|
||||
)
|
||||
|
||||
// Logger configuration
|
||||
type Config struct {
|
||||
Verbose bool
|
||||
Debug bool
|
||||
Cron bool
|
||||
}
|
||||
|
||||
var logger *slog.Logger
|
||||
|
||||
// Initialize sets up the global logger based on the provided configuration
|
||||
func Initialize(cfg Config) {
|
||||
// Determine log level based on configuration
|
||||
var level slog.Level
|
||||
|
||||
if cfg.Cron {
|
||||
// In cron mode, only show fatal errors (which we'll handle specially)
|
||||
level = slog.LevelError
|
||||
} else if cfg.Debug || strings.Contains(os.Getenv("GODEBUG"), "vaultik") {
|
||||
level = slog.LevelDebug
|
||||
} else if cfg.Verbose {
|
||||
level = slog.LevelInfo
|
||||
} else {
|
||||
level = slog.LevelWarn
|
||||
}
|
||||
|
||||
// Create handler with appropriate level
|
||||
opts := &slog.HandlerOptions{
|
||||
Level: level,
|
||||
}
|
||||
|
||||
// Check if stdout is a TTY
|
||||
if term.IsTerminal(int(os.Stdout.Fd())) {
|
||||
// Use colorized TTY handler
|
||||
logger = slog.New(NewTTYHandler(os.Stdout, opts))
|
||||
} else {
|
||||
// Use JSON format for non-TTY output
|
||||
logger = slog.New(slog.NewJSONHandler(os.Stdout, opts))
|
||||
}
|
||||
|
||||
// Set as default logger
|
||||
slog.SetDefault(logger)
|
||||
}
|
||||
|
||||
// getCaller returns the caller information as a string
|
||||
func getCaller(skip int) string {
|
||||
_, file, line, ok := runtime.Caller(skip)
|
||||
if !ok {
|
||||
return "unknown"
|
||||
}
|
||||
return fmt.Sprintf("%s:%d", filepath.Base(file), line)
|
||||
}
|
||||
|
||||
// Fatal logs a fatal error and exits
|
||||
func Fatal(msg string, args ...any) {
|
||||
if logger != nil {
|
||||
// Add caller info to args
|
||||
args = append(args, "caller", getCaller(2))
|
||||
logger.Error(msg, args...)
|
||||
}
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Fatalf logs a formatted fatal error and exits
|
||||
func Fatalf(format string, args ...any) {
|
||||
Fatal(fmt.Sprintf(format, args...))
|
||||
}
|
||||
|
||||
// Error logs an error
|
||||
func Error(msg string, args ...any) {
|
||||
if logger != nil {
|
||||
args = append(args, "caller", getCaller(2))
|
||||
logger.Error(msg, args...)
|
||||
}
|
||||
}
|
||||
|
||||
// Errorf logs a formatted error
|
||||
func Errorf(format string, args ...any) {
|
||||
Error(fmt.Sprintf(format, args...))
|
||||
}
|
||||
|
||||
// Warn logs a warning
|
||||
func Warn(msg string, args ...any) {
|
||||
if logger != nil {
|
||||
args = append(args, "caller", getCaller(2))
|
||||
logger.Warn(msg, args...)
|
||||
}
|
||||
}
|
||||
|
||||
// Warnf logs a formatted warning
|
||||
func Warnf(format string, args ...any) {
|
||||
Warn(fmt.Sprintf(format, args...))
|
||||
}
|
||||
|
||||
// Notice logs a notice (mapped to Info level)
|
||||
func Notice(msg string, args ...any) {
|
||||
if logger != nil {
|
||||
args = append(args, "caller", getCaller(2))
|
||||
logger.Info(msg, args...)
|
||||
}
|
||||
}
|
||||
|
||||
// Noticef logs a formatted notice
|
||||
func Noticef(format string, args ...any) {
|
||||
Notice(fmt.Sprintf(format, args...))
|
||||
}
|
||||
|
||||
// Info logs an info message
|
||||
func Info(msg string, args ...any) {
|
||||
if logger != nil {
|
||||
args = append(args, "caller", getCaller(2))
|
||||
logger.Info(msg, args...)
|
||||
}
|
||||
}
|
||||
|
||||
// Infof logs a formatted info message
|
||||
func Infof(format string, args ...any) {
|
||||
Info(fmt.Sprintf(format, args...))
|
||||
}
|
||||
|
||||
// Debug logs a debug message
|
||||
func Debug(msg string, args ...any) {
|
||||
if logger != nil {
|
||||
args = append(args, "caller", getCaller(2))
|
||||
logger.Debug(msg, args...)
|
||||
}
|
||||
}
|
||||
|
||||
// Debugf logs a formatted debug message
|
||||
func Debugf(format string, args ...any) {
|
||||
Debug(fmt.Sprintf(format, args...))
|
||||
}
|
||||
|
||||
// With returns a logger with additional context
|
||||
func With(args ...any) *slog.Logger {
|
||||
if logger != nil {
|
||||
return logger.With(args...)
|
||||
}
|
||||
return slog.Default()
|
||||
}
|
||||
|
||||
// WithContext returns a logger with context
|
||||
func WithContext(ctx context.Context) *slog.Logger {
|
||||
return logger
|
||||
}
|
||||
|
||||
// Logger returns the underlying slog.Logger
|
||||
func Logger() *slog.Logger {
|
||||
return logger
|
||||
}
|
||||
24
internal/log/module.go
Normal file
24
internal/log/module.go
Normal file
@@ -0,0 +1,24 @@
|
||||
package log
|
||||
|
||||
import (
|
||||
"go.uber.org/fx"
|
||||
)
|
||||
|
||||
// Module exports logging functionality
|
||||
var Module = fx.Module("log",
|
||||
fx.Invoke(func(cfg Config) {
|
||||
Initialize(cfg)
|
||||
}),
|
||||
)
|
||||
|
||||
// New creates a new logger configuration from provided options
|
||||
func New(opts LogOptions) Config {
|
||||
return Config(opts)
|
||||
}
|
||||
|
||||
// LogOptions are provided by the CLI
|
||||
type LogOptions struct {
|
||||
Verbose bool
|
||||
Debug bool
|
||||
Cron bool
|
||||
}
|
||||
140
internal/log/tty_handler.go
Normal file
140
internal/log/tty_handler.go
Normal file
@@ -0,0 +1,140 @@
|
||||
package log
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ANSI color codes
|
||||
const (
|
||||
colorReset = "\033[0m"
|
||||
colorRed = "\033[31m"
|
||||
colorYellow = "\033[33m"
|
||||
colorBlue = "\033[34m"
|
||||
colorGray = "\033[90m"
|
||||
colorGreen = "\033[32m"
|
||||
colorCyan = "\033[36m"
|
||||
colorBold = "\033[1m"
|
||||
)
|
||||
|
||||
// TTYHandler is a custom handler for TTY output with colors
|
||||
type TTYHandler struct {
|
||||
opts slog.HandlerOptions
|
||||
mu sync.Mutex
|
||||
out io.Writer
|
||||
}
|
||||
|
||||
// NewTTYHandler creates a new TTY handler
|
||||
func NewTTYHandler(out io.Writer, opts *slog.HandlerOptions) *TTYHandler {
|
||||
if opts == nil {
|
||||
opts = &slog.HandlerOptions{}
|
||||
}
|
||||
return &TTYHandler{
|
||||
out: out,
|
||||
opts: *opts,
|
||||
}
|
||||
}
|
||||
|
||||
// Enabled reports whether the handler handles records at the given level
|
||||
func (h *TTYHandler) Enabled(_ context.Context, level slog.Level) bool {
|
||||
return level >= h.opts.Level.Level()
|
||||
}
|
||||
|
||||
// Handle writes the log record
|
||||
func (h *TTYHandler) Handle(_ context.Context, r slog.Record) error {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
|
||||
// Format timestamp
|
||||
timestamp := r.Time.Format("15:04:05")
|
||||
|
||||
// Level and color
|
||||
level := r.Level.String()
|
||||
var levelColor string
|
||||
switch r.Level {
|
||||
case slog.LevelDebug:
|
||||
levelColor = colorGray
|
||||
level = "DEBUG"
|
||||
case slog.LevelInfo:
|
||||
levelColor = colorGreen
|
||||
level = "INFO "
|
||||
case slog.LevelWarn:
|
||||
levelColor = colorYellow
|
||||
level = "WARN "
|
||||
case slog.LevelError:
|
||||
levelColor = colorRed
|
||||
level = "ERROR"
|
||||
default:
|
||||
levelColor = colorReset
|
||||
}
|
||||
|
||||
// Print main message
|
||||
_, _ = fmt.Fprintf(h.out, "%s%s%s %s%s%s %s%s%s",
|
||||
colorGray, timestamp, colorReset,
|
||||
levelColor, level, colorReset,
|
||||
colorBold, r.Message, colorReset)
|
||||
|
||||
// Print attributes
|
||||
r.Attrs(func(a slog.Attr) bool {
|
||||
value := a.Value.String()
|
||||
// Special handling for certain attribute types
|
||||
switch a.Value.Kind() {
|
||||
case slog.KindDuration:
|
||||
if d, ok := a.Value.Any().(time.Duration); ok {
|
||||
value = formatDuration(d)
|
||||
}
|
||||
case slog.KindInt64:
|
||||
if a.Key == "bytes" {
|
||||
value = formatBytes(a.Value.Int64())
|
||||
}
|
||||
}
|
||||
|
||||
_, _ = fmt.Fprintf(h.out, " %s%s%s=%s%s%s",
|
||||
colorCyan, a.Key, colorReset,
|
||||
colorBlue, value, colorReset)
|
||||
return true
|
||||
})
|
||||
|
||||
_, _ = fmt.Fprintln(h.out)
|
||||
return nil
|
||||
}
|
||||
|
||||
// WithAttrs returns a new handler with the given attributes
|
||||
func (h *TTYHandler) WithAttrs(attrs []slog.Attr) slog.Handler {
|
||||
return h // Simplified for now
|
||||
}
|
||||
|
||||
// WithGroup returns a new handler with the given group name
|
||||
func (h *TTYHandler) WithGroup(name string) slog.Handler {
|
||||
return h // Simplified for now
|
||||
}
|
||||
|
||||
// formatDuration formats a duration in a human-readable way
|
||||
func formatDuration(d time.Duration) string {
|
||||
if d < time.Millisecond {
|
||||
return fmt.Sprintf("%dµs", d.Microseconds())
|
||||
} else if d < time.Second {
|
||||
return fmt.Sprintf("%dms", d.Milliseconds())
|
||||
} else if d < time.Minute {
|
||||
return fmt.Sprintf("%.1fs", d.Seconds())
|
||||
}
|
||||
return d.String()
|
||||
}
|
||||
|
||||
// formatBytes formats bytes in a human-readable way
|
||||
func formatBytes(b int64) string {
|
||||
const unit = 1024
|
||||
if b < unit {
|
||||
return fmt.Sprintf("%d B", b)
|
||||
}
|
||||
div, exp := int64(unit), 0
|
||||
for n := b / unit; n >= unit; n /= unit {
|
||||
div *= unit
|
||||
exp++
|
||||
}
|
||||
return fmt.Sprintf("%.1f %cB", float64(b)/float64(div), "KMGTPE"[exp])
|
||||
}
|
||||
Reference in New Issue
Block a user