- Implement exclude patterns with anchored pattern support: - Patterns starting with / only match from root of source dir - Unanchored patterns match anywhere in path - Support for glob patterns (*.log, .*, **/*.pack) - Directory patterns skip entire subtrees - Add gobwas/glob dependency for pattern matching - Add 16 comprehensive tests for exclude functionality - Add snapshot prune command to clean orphaned data: - Removes incomplete snapshots from database - Cleans orphaned files, chunks, and blobs - Runs automatically at backup start for consistency - Add snapshot remove command for deleting snapshots - Add VAULTIK_AGE_SECRET_KEY environment variable support - Fix duplicate fx module provider in restore command - Change snapshot ID format to hostname_YYYY-MM-DDTHH:MM:SSZ
376 lines
8.2 KiB
Go
376 lines
8.2 KiB
Go
package database
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"fmt"
|
|
"time"
|
|
|
|
"git.eeqj.de/sneak/vaultik/internal/log"
|
|
"github.com/google/uuid"
|
|
)
|
|
|
|
type FileRepository struct {
|
|
db *DB
|
|
}
|
|
|
|
func NewFileRepository(db *DB) *FileRepository {
|
|
return &FileRepository{db: db}
|
|
}
|
|
|
|
func (r *FileRepository) Create(ctx context.Context, tx *sql.Tx, file *File) error {
|
|
// Generate UUID if not provided
|
|
if file.ID == "" {
|
|
file.ID = uuid.New().String()
|
|
}
|
|
|
|
query := `
|
|
INSERT INTO files (id, path, mtime, ctime, size, mode, uid, gid, link_target)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
ON CONFLICT(path) DO UPDATE SET
|
|
mtime = excluded.mtime,
|
|
ctime = excluded.ctime,
|
|
size = excluded.size,
|
|
mode = excluded.mode,
|
|
uid = excluded.uid,
|
|
gid = excluded.gid,
|
|
link_target = excluded.link_target
|
|
RETURNING id
|
|
`
|
|
|
|
var err error
|
|
if tx != nil {
|
|
LogSQL("Execute", query, file.ID, file.Path, file.MTime.Unix(), file.CTime.Unix(), file.Size, file.Mode, file.UID, file.GID, file.LinkTarget)
|
|
err = tx.QueryRowContext(ctx, query, file.ID, file.Path, file.MTime.Unix(), file.CTime.Unix(), file.Size, file.Mode, file.UID, file.GID, file.LinkTarget).Scan(&file.ID)
|
|
} else {
|
|
err = r.db.QueryRowWithLog(ctx, query, file.ID, file.Path, file.MTime.Unix(), file.CTime.Unix(), file.Size, file.Mode, file.UID, file.GID, file.LinkTarget).Scan(&file.ID)
|
|
}
|
|
|
|
if err != nil {
|
|
return fmt.Errorf("inserting file: %w", err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (r *FileRepository) GetByPath(ctx context.Context, path string) (*File, error) {
|
|
query := `
|
|
SELECT id, path, mtime, ctime, size, mode, uid, gid, link_target
|
|
FROM files
|
|
WHERE path = ?
|
|
`
|
|
|
|
var file File
|
|
var mtimeUnix, ctimeUnix int64
|
|
var linkTarget sql.NullString
|
|
|
|
err := r.db.conn.QueryRowContext(ctx, query, path).Scan(
|
|
&file.ID,
|
|
&file.Path,
|
|
&mtimeUnix,
|
|
&ctimeUnix,
|
|
&file.Size,
|
|
&file.Mode,
|
|
&file.UID,
|
|
&file.GID,
|
|
&linkTarget,
|
|
)
|
|
|
|
if err == sql.ErrNoRows {
|
|
return nil, nil
|
|
}
|
|
if err != nil {
|
|
return nil, fmt.Errorf("querying file: %w", err)
|
|
}
|
|
|
|
file.MTime = time.Unix(mtimeUnix, 0).UTC()
|
|
file.CTime = time.Unix(ctimeUnix, 0).UTC()
|
|
if linkTarget.Valid {
|
|
file.LinkTarget = linkTarget.String
|
|
}
|
|
|
|
return &file, nil
|
|
}
|
|
|
|
// GetByID retrieves a file by its UUID
|
|
func (r *FileRepository) GetByID(ctx context.Context, id string) (*File, error) {
|
|
query := `
|
|
SELECT id, path, mtime, ctime, size, mode, uid, gid, link_target
|
|
FROM files
|
|
WHERE id = ?
|
|
`
|
|
|
|
var file File
|
|
var mtimeUnix, ctimeUnix int64
|
|
var linkTarget sql.NullString
|
|
|
|
err := r.db.conn.QueryRowContext(ctx, query, id).Scan(
|
|
&file.ID,
|
|
&file.Path,
|
|
&mtimeUnix,
|
|
&ctimeUnix,
|
|
&file.Size,
|
|
&file.Mode,
|
|
&file.UID,
|
|
&file.GID,
|
|
&linkTarget,
|
|
)
|
|
|
|
if err == sql.ErrNoRows {
|
|
return nil, nil
|
|
}
|
|
if err != nil {
|
|
return nil, fmt.Errorf("querying file: %w", err)
|
|
}
|
|
|
|
file.MTime = time.Unix(mtimeUnix, 0).UTC()
|
|
file.CTime = time.Unix(ctimeUnix, 0).UTC()
|
|
if linkTarget.Valid {
|
|
file.LinkTarget = linkTarget.String
|
|
}
|
|
|
|
return &file, nil
|
|
}
|
|
|
|
func (r *FileRepository) GetByPathTx(ctx context.Context, tx *sql.Tx, path string) (*File, error) {
|
|
query := `
|
|
SELECT id, path, mtime, ctime, size, mode, uid, gid, link_target
|
|
FROM files
|
|
WHERE path = ?
|
|
`
|
|
|
|
var file File
|
|
var mtimeUnix, ctimeUnix int64
|
|
var linkTarget sql.NullString
|
|
|
|
LogSQL("GetByPathTx QueryRowContext", query, path)
|
|
err := tx.QueryRowContext(ctx, query, path).Scan(
|
|
&file.ID,
|
|
&file.Path,
|
|
&mtimeUnix,
|
|
&ctimeUnix,
|
|
&file.Size,
|
|
&file.Mode,
|
|
&file.UID,
|
|
&file.GID,
|
|
&linkTarget,
|
|
)
|
|
LogSQL("GetByPathTx Scan complete", query, path)
|
|
|
|
if err == sql.ErrNoRows {
|
|
return nil, nil
|
|
}
|
|
if err != nil {
|
|
return nil, fmt.Errorf("querying file: %w", err)
|
|
}
|
|
|
|
file.MTime = time.Unix(mtimeUnix, 0).UTC()
|
|
file.CTime = time.Unix(ctimeUnix, 0).UTC()
|
|
if linkTarget.Valid {
|
|
file.LinkTarget = linkTarget.String
|
|
}
|
|
|
|
return &file, nil
|
|
}
|
|
|
|
func (r *FileRepository) ListModifiedSince(ctx context.Context, since time.Time) ([]*File, error) {
|
|
query := `
|
|
SELECT id, path, mtime, ctime, size, mode, uid, gid, link_target
|
|
FROM files
|
|
WHERE mtime >= ?
|
|
ORDER BY path
|
|
`
|
|
|
|
rows, err := r.db.conn.QueryContext(ctx, query, since.Unix())
|
|
if err != nil {
|
|
return nil, fmt.Errorf("querying files: %w", err)
|
|
}
|
|
defer CloseRows(rows)
|
|
|
|
var files []*File
|
|
for rows.Next() {
|
|
var file File
|
|
var mtimeUnix, ctimeUnix int64
|
|
var linkTarget sql.NullString
|
|
|
|
err := rows.Scan(
|
|
&file.ID,
|
|
&file.Path,
|
|
&mtimeUnix,
|
|
&ctimeUnix,
|
|
&file.Size,
|
|
&file.Mode,
|
|
&file.UID,
|
|
&file.GID,
|
|
&linkTarget,
|
|
)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("scanning file: %w", err)
|
|
}
|
|
|
|
file.MTime = time.Unix(mtimeUnix, 0)
|
|
file.CTime = time.Unix(ctimeUnix, 0)
|
|
if linkTarget.Valid {
|
|
file.LinkTarget = linkTarget.String
|
|
}
|
|
|
|
files = append(files, &file)
|
|
}
|
|
|
|
return files, rows.Err()
|
|
}
|
|
|
|
func (r *FileRepository) Delete(ctx context.Context, tx *sql.Tx, path string) error {
|
|
query := `DELETE FROM files WHERE path = ?`
|
|
|
|
var err error
|
|
if tx != nil {
|
|
_, err = tx.ExecContext(ctx, query, path)
|
|
} else {
|
|
_, err = r.db.ExecWithLog(ctx, query, path)
|
|
}
|
|
|
|
if err != nil {
|
|
return fmt.Errorf("deleting file: %w", err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// DeleteByID deletes a file by its UUID
|
|
func (r *FileRepository) DeleteByID(ctx context.Context, tx *sql.Tx, id string) error {
|
|
query := `DELETE FROM files WHERE id = ?`
|
|
|
|
var err error
|
|
if tx != nil {
|
|
_, err = tx.ExecContext(ctx, query, id)
|
|
} else {
|
|
_, err = r.db.ExecWithLog(ctx, query, id)
|
|
}
|
|
|
|
if err != nil {
|
|
return fmt.Errorf("deleting file: %w", err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (r *FileRepository) ListByPrefix(ctx context.Context, prefix string) ([]*File, error) {
|
|
query := `
|
|
SELECT id, path, mtime, ctime, size, mode, uid, gid, link_target
|
|
FROM files
|
|
WHERE path LIKE ? || '%'
|
|
ORDER BY path
|
|
`
|
|
|
|
rows, err := r.db.conn.QueryContext(ctx, query, prefix)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("querying files: %w", err)
|
|
}
|
|
defer CloseRows(rows)
|
|
|
|
var files []*File
|
|
for rows.Next() {
|
|
var file File
|
|
var mtimeUnix, ctimeUnix int64
|
|
var linkTarget sql.NullString
|
|
|
|
err := rows.Scan(
|
|
&file.ID,
|
|
&file.Path,
|
|
&mtimeUnix,
|
|
&ctimeUnix,
|
|
&file.Size,
|
|
&file.Mode,
|
|
&file.UID,
|
|
&file.GID,
|
|
&linkTarget,
|
|
)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("scanning file: %w", err)
|
|
}
|
|
|
|
file.MTime = time.Unix(mtimeUnix, 0)
|
|
file.CTime = time.Unix(ctimeUnix, 0)
|
|
if linkTarget.Valid {
|
|
file.LinkTarget = linkTarget.String
|
|
}
|
|
|
|
files = append(files, &file)
|
|
}
|
|
|
|
return files, rows.Err()
|
|
}
|
|
|
|
// CreateBatch inserts or updates multiple files in a single statement for efficiency.
|
|
// File IDs must be pre-generated before calling this method.
|
|
func (r *FileRepository) CreateBatch(ctx context.Context, tx *sql.Tx, files []*File) error {
|
|
if len(files) == 0 {
|
|
return nil
|
|
}
|
|
|
|
// Each File has 9 values, so batch at 100 to be safe with SQLite's variable limit
|
|
const batchSize = 100
|
|
|
|
for i := 0; i < len(files); i += batchSize {
|
|
end := i + batchSize
|
|
if end > len(files) {
|
|
end = len(files)
|
|
}
|
|
batch := files[i:end]
|
|
|
|
query := `INSERT INTO files (id, path, mtime, ctime, size, mode, uid, gid, link_target) VALUES `
|
|
args := make([]interface{}, 0, len(batch)*9)
|
|
for j, f := range batch {
|
|
if j > 0 {
|
|
query += ", "
|
|
}
|
|
query += "(?, ?, ?, ?, ?, ?, ?, ?, ?)"
|
|
args = append(args, f.ID, f.Path, f.MTime.Unix(), f.CTime.Unix(), f.Size, f.Mode, f.UID, f.GID, f.LinkTarget)
|
|
}
|
|
query += ` ON CONFLICT(path) DO UPDATE SET
|
|
mtime = excluded.mtime,
|
|
ctime = excluded.ctime,
|
|
size = excluded.size,
|
|
mode = excluded.mode,
|
|
uid = excluded.uid,
|
|
gid = excluded.gid,
|
|
link_target = excluded.link_target`
|
|
|
|
var err error
|
|
if tx != nil {
|
|
_, err = tx.ExecContext(ctx, query, args...)
|
|
} else {
|
|
_, err = r.db.ExecWithLog(ctx, query, args...)
|
|
}
|
|
if err != nil {
|
|
return fmt.Errorf("batch inserting files: %w", err)
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// DeleteOrphaned deletes files that are not referenced by any snapshot
|
|
func (r *FileRepository) DeleteOrphaned(ctx context.Context) error {
|
|
query := `
|
|
DELETE FROM files
|
|
WHERE NOT EXISTS (
|
|
SELECT 1 FROM snapshot_files
|
|
WHERE snapshot_files.file_id = files.id
|
|
)
|
|
`
|
|
|
|
result, err := r.db.ExecWithLog(ctx, query)
|
|
if err != nil {
|
|
return fmt.Errorf("deleting orphaned files: %w", err)
|
|
}
|
|
|
|
rowsAffected, _ := result.RowsAffected()
|
|
if rowsAffected > 0 {
|
|
log.Debug("Deleted orphaned files", "count", rowsAffected)
|
|
}
|
|
|
|
return nil
|
|
}
|